├── .gitignore ├── examples ├── .gitignore ├── rlp_gateway │ ├── .gitignore │ └── main.go ├── v1_ingress │ └── main.go ├── rfc5424 │ └── main.go ├── runtime_stats │ └── main.go ├── go.mod ├── v2_ingress │ └── main.go ├── envelope_stream_connector │ └── main.go └── go.sum ├── CODEOWNERS ├── rpc └── loggregator_v2 │ ├── doc.go │ ├── loggregator_v2_suite_test.go │ ├── generate.sh │ ├── egress_grpc.pb.go │ ├── ingress_grpc.pb.go │ └── ingress.pb.go ├── rfc5424 ├── setup_test.go ├── rfc5424.go ├── README.md ├── stream.go ├── message_test.go ├── message.go ├── stream_test.go └── marshal.go ├── scripts ├── test └── generate-test-certs ├── v1 ├── v1_suite_test.go └── client.go ├── conversion ├── conversion_suite_test.go ├── benchmark │ ├── benchmark_suite_test.go │ └── benchmark_test.go ├── tov2_test.go ├── tov1_test.go ├── error_test.go ├── log_message_test.go ├── counter_event_test.go ├── value_metric_test.go ├── envelope_test.go ├── instance_id_test.go ├── tov2.go ├── container_metric_test.go ├── tov1.go └── http_test.go ├── runtimeemitter ├── runtimeemitter_suite_test.go ├── runtime_emitter_test.go └── runtime_emitter.go ├── .github ├── dependabot.yaml └── workflows │ └── go.yml ├── suite_test.go ├── pulseemitter ├── pulseemitter_suite_test.go ├── gauge_metric_test.go ├── gauge_metric.go ├── pulse_emitter_test.go ├── counter_metric.go ├── pulse_emitter.go └── counter_metric_test.go ├── NOTICE ├── fixtures_test.go ├── .golangci.yml ├── tls.go ├── doc.go ├── go.mod ├── one_to_one_envelope_batch_diode.go ├── README.md ├── servers_test.go ├── envelope_stream_connector.go ├── envelope_stream_connector_test.go ├── rlp_gateway_client.go └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | vendor/ -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | main 2 | -------------------------------------------------------------------------------- /examples/rlp_gateway/.gitignore: -------------------------------------------------------------------------------- 1 | rlp_gateway 2 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @cloudfoundry/wg-app-runtime-platform-logging-and-metrics-approvers 2 | -------------------------------------------------------------------------------- /rpc/loggregator_v2/doc.go: -------------------------------------------------------------------------------- 1 | //go:generate ./generate.sh 2 | package loggregator_v2 3 | -------------------------------------------------------------------------------- /rfc5424/setup_test.go: -------------------------------------------------------------------------------- 1 | package rfc5424 2 | 3 | import ( 4 | "testing" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | // Hook up gocheck into the "go test" runner. 10 | func Test(t *testing.T) { TestingT(t) } 11 | -------------------------------------------------------------------------------- /scripts/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | go get github.com/onsi/gomega 4 | go get github.com/cloudfoundry/dropsonde/... 5 | go get github.com/golang/protobuf 6 | go get golang.org/x/net 7 | go get golang.org/x/text 8 | 9 | set -ex 10 | ginkgo -r -race 11 | -------------------------------------------------------------------------------- /v1/v1_suite_test.go: -------------------------------------------------------------------------------- 1 | package v1_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestV1(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "V1 Suite") 13 | } 14 | -------------------------------------------------------------------------------- /conversion/conversion_suite_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestConversion(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Conversion Suite") 13 | } 14 | -------------------------------------------------------------------------------- /conversion/benchmark/benchmark_suite_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestBenchmark(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Benchmark Suite") 13 | } 14 | -------------------------------------------------------------------------------- /runtimeemitter/runtimeemitter_suite_test.go: -------------------------------------------------------------------------------- 1 | package runtimeemitter_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo/v2" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestRuntimeemitter(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Runtime Emitter Suite") 13 | } 14 | -------------------------------------------------------------------------------- /rpc/loggregator_v2/loggregator_v2_suite_test.go: -------------------------------------------------------------------------------- 1 | package loggregator_v2_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestLoggregatorV2(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "LoggregatorV2 Suite") 13 | } 14 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | 9 | - package-ecosystem: "gomod" 10 | directory: "/" 11 | schedule: 12 | interval: "weekly" 13 | 14 | - package-ecosystem: "gomod" 15 | directory: "/examples" 16 | schedule: 17 | interval: "weekly" 18 | -------------------------------------------------------------------------------- /suite_test.go: -------------------------------------------------------------------------------- 1 | package loggregator_test 2 | 3 | import ( 4 | "io" 5 | "log" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "google.golang.org/grpc/grpclog" 10 | 11 | "testing" 12 | ) 13 | 14 | func TestGoLoggregator(t *testing.T) { 15 | grpclog.SetLoggerV2(grpclog.NewLoggerV2(GinkgoWriter, io.Discard, io.Discard)) 16 | log.SetOutput(GinkgoWriter) 17 | 18 | RegisterFailHandler(Fail) 19 | RunSpecs(t, "GoLoggregator Suite") 20 | } 21 | -------------------------------------------------------------------------------- /rfc5424/rfc5424.go: -------------------------------------------------------------------------------- 1 | package rfc5424 2 | 3 | type Priority int 4 | 5 | const ( 6 | Emergency Priority = iota 7 | Alert 8 | Crit 9 | Error 10 | Warning 11 | Notice 12 | Info 13 | Debug 14 | ) 15 | 16 | const ( 17 | Kern Priority = iota << 3 18 | User 19 | Mail 20 | Daemon 21 | Auth 22 | Syslog 23 | Lpr 24 | News 25 | Uucp 26 | Cron 27 | Authpriv 28 | Ftp 29 | Local0 30 | Local1 31 | Local2 32 | Local3 33 | Local4 34 | Local5 35 | Local6 36 | Local7 37 | ) 38 | -------------------------------------------------------------------------------- /pulseemitter/pulseemitter_suite_test.go: -------------------------------------------------------------------------------- 1 | package pulseemitter_test 2 | 3 | import ( 4 | "io" 5 | "log" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "google.golang.org/grpc/grpclog" 10 | 11 | "testing" 12 | ) 13 | 14 | func TestMetricemitter(t *testing.T) { 15 | grpclog.SetLoggerV2(grpclog.NewLoggerV2(GinkgoWriter, io.Discard, io.Discard)) 16 | log.SetOutput(GinkgoWriter) 17 | 18 | RegisterFailHandler(Fail) 19 | RunSpecs(t, "Pulse Emitter Suite") 20 | } 21 | -------------------------------------------------------------------------------- /examples/v1_ingress/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/cloudfoundry/dropsonde" 8 | 9 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 10 | v1 "code.cloudfoundry.org/go-loggregator/v10/v1" 11 | ) 12 | 13 | func main() { 14 | dropsonde.Initialize("127.0.0.1:3457", "example-source") 15 | 16 | client, err := v1.NewClient() 17 | if err != nil { 18 | log.Fatal("Could not create client", err) 19 | } 20 | 21 | for { 22 | client.EmitLog("some log goes here", 23 | loggregator.WithSourceInfo("v1-example-source-id", "platform", "v1-example-source-instance"), 24 | ) 25 | time.Sleep(time.Second) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | go-loggregator 2 | 3 | Copyright (c) 2017-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | -------------------------------------------------------------------------------- /rpc/loggregator_v2/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eu 4 | 5 | SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]:-$0}"; )" &> /dev/null && pwd 2> /dev/null; )"; 6 | 7 | go install google.golang.org/protobuf/cmd/protoc-gen-go@latest 8 | go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest 9 | 10 | TMP_DIR=$(mktemp -d) 11 | git clone https://github.com/cloudfoundry/loggregator-api.git $TMP_DIR/loggregator-api 12 | 13 | pushd $SCRIPT_DIR/../.. 14 | protoc -I=$TMP_DIR --go_out=. --go-grpc_out=. $TMP_DIR/loggregator-api/v2/*.proto 15 | mv code.cloudfoundry.org/go-loggregator/v9/rpc/loggregator_v2/* rpc/loggregator_v2/ 16 | rm -rf code.cloudfoundry.org 17 | popd 18 | 19 | rm -rf $TMP_DIR 20 | -------------------------------------------------------------------------------- /examples/rfc5424/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "code.cloudfoundry.org/go-loggregator/v10/rfc5424" 9 | ) 10 | 11 | func writeMain() { 12 | m := rfc5424.Message{ 13 | Priority: rfc5424.Daemon | rfc5424.Info, 14 | Timestamp: time.Now(), 15 | Hostname: "myhostname", 16 | AppName: "someapp", 17 | Message: []byte("Hello, World!"), 18 | } 19 | m.AddDatum("foo@1234", "Revision", "1.2.3.4") 20 | m.WriteTo(os.Stdout) 21 | } 22 | 23 | func readMain() { 24 | m := rfc5424.Message{} 25 | _, err := m.ReadFrom(os.Stdin) 26 | if err != nil { 27 | fmt.Printf("%s\n", err) 28 | } 29 | fmt.Printf("%#v\n", m) 30 | } 31 | 32 | func main() { 33 | readMain() 34 | } 35 | -------------------------------------------------------------------------------- /fixtures_test.go: -------------------------------------------------------------------------------- 1 | package loggregator_test 2 | 3 | import ( 4 | "log" 5 | "os" 6 | ) 7 | 8 | //go:generate go get github.com/loggregator/go-bindata/... 9 | //go:generate scripts/generate-test-certs 10 | //go:generate go-bindata -nocompress -o bindata_test.go -pkg loggregator_test -prefix test-certs/ test-certs/ 11 | //go:generate rm -rf test-certs 12 | 13 | func fixture(filename string) string { 14 | contents := MustAsset(filename) 15 | 16 | tmpfile, err := os.CreateTemp("", "") 17 | if err != nil { 18 | log.Fatal(err) 19 | } 20 | 21 | if _, err := tmpfile.Write(contents); err != nil { 22 | log.Fatal(err) 23 | } 24 | if err := tmpfile.Close(); err != nil { 25 | log.Fatal(err) 26 | } 27 | 28 | return tmpfile.Name() 29 | } 30 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | enable: 4 | # Checks for non-ASCII identifiers 5 | - asciicheck 6 | # Computes and checks the cyclomatic complexity of functions. 7 | - gocyclo 8 | # Inspects source code for security problems. 9 | - gosec 10 | settings: 11 | gosec: 12 | excludes: 13 | - G115 14 | exclusions: 15 | generated: lax 16 | presets: 17 | - comments 18 | - common-false-positives 19 | - legacy 20 | - std-error-handling 21 | paths: 22 | - third_party$ 23 | - builtin$ 24 | - examples$ 25 | issues: 26 | # Disable max issues per linter. 27 | max-issues-per-linter: 0 28 | # Disable max same issues. 29 | max-same-issues: 0 30 | formatters: 31 | exclusions: 32 | generated: lax 33 | paths: 34 | - third_party$ 35 | - builtin$ 36 | - examples$ 37 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v6 16 | - uses: actions/setup-go@v6 17 | with: 18 | go-version-file: go.mod 19 | - run: go test -race ./... 20 | 21 | vet: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - uses: actions/checkout@v6 25 | - uses: actions/setup-go@v6 26 | with: 27 | go-version-file: go.mod 28 | - run: go vet ./... 29 | 30 | lint: 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v6 34 | - uses: actions/setup-go@v6 35 | with: 36 | go-version-file: go.mod 37 | - uses: golangci/golangci-lint-action@v9.1.0 38 | with: 39 | args: --config .golangci.yml 40 | -------------------------------------------------------------------------------- /examples/runtime_stats/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | 8 | "code.cloudfoundry.org/go-loggregator/v10" 9 | "code.cloudfoundry.org/go-loggregator/v10/runtimeemitter" 10 | ) 11 | 12 | func main() { 13 | tlsConfig, err := loggregator.NewIngressTLSConfig( 14 | os.Getenv("CA_CERT_PATH"), 15 | os.Getenv("CERT_PATH"), 16 | os.Getenv("KEY_PATH"), 17 | ) 18 | if err != nil { 19 | log.Fatal("Could not create TLS config", err) 20 | } 21 | 22 | client, err := loggregator.NewIngressClient( 23 | tlsConfig, 24 | loggregator.WithAddr("localhost:3458"), 25 | loggregator.WithLogger(log.New(os.Stdout, "", log.LstdFlags)), 26 | ) 27 | 28 | if err != nil { 29 | log.Fatal("Could not create client", err) 30 | } 31 | 32 | runtimeStats := runtimeemitter.New( 33 | client, 34 | runtimeemitter.WithInterval(10*time.Second), 35 | ) 36 | 37 | runtimeStats.Run() 38 | } 39 | -------------------------------------------------------------------------------- /rfc5424/README.md: -------------------------------------------------------------------------------- 1 | 2 | [![Build Status](https://travis-ci.org/crewjam/rfc5424.png)](https://travis-ci.org/crewjam/rfc5424) 3 | 4 | [![](https://godoc.org/github.com/crewjam/rfc5424?status.png)](http://godoc.org/github.com/crewjam/rfc5424) 5 | 6 | This is a Go library that can read and write RFC-5424 syslog messages: 7 | 8 | Example usage: 9 | 10 | m := rfc5424.Message{ 11 | Priority: rfc5424.Daemon | rfc5424.Info, 12 | Timestamp: time.Now(), 13 | Hostname: "myhostname", 14 | AppName: "someapp", 15 | Message: []byte("Hello, World!"), 16 | } 17 | m.AddDatum("foo@1234", "Revision", "1.2.3.4") 18 | m.WriteTo(os.Stdout) 19 | 20 | Produces output like: 21 | 22 | 107 <7>1 2016-02-28T09:57:10.804642398-05:00 myhostname someapp - - [foo@1234 Revision="1.2.3.4"] Hello, World! 23 | 24 | You can also use the library to parse syslog messages: 25 | 26 | m := rfc5424.Message{} 27 | _, err := m.ReadFrom(os.Stdin) 28 | fmt.Printf("%s\n", m.Message) 29 | -------------------------------------------------------------------------------- /tls.go: -------------------------------------------------------------------------------- 1 | package loggregator 2 | 3 | import ( 4 | "crypto/tls" 5 | 6 | "code.cloudfoundry.org/tlsconfig" 7 | ) 8 | 9 | // NewIngressTLSConfig provides a convenient means for creating a *tls.Config 10 | // which uses the CA, cert, and key for the ingress endpoint. 11 | func NewIngressTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) { 12 | return newTLSConfig(caPath, certPath, keyPath, "metron") 13 | } 14 | 15 | // NewEgressTLSConfig provides a convenient means for creating a *tls.Config 16 | // which uses the CA, cert, and key for the egress endpoint. 17 | func NewEgressTLSConfig(caPath, certPath, keyPath string) (*tls.Config, error) { 18 | return newTLSConfig(caPath, certPath, keyPath, "reverselogproxy") 19 | } 20 | 21 | func newTLSConfig(caPath, certPath, keyPath, cn string) (*tls.Config, error) { 22 | return tlsconfig.Build( 23 | tlsconfig.WithInternalServiceDefaults(), 24 | tlsconfig.WithIdentityFromFile(certPath, keyPath), 25 | ).Client( 26 | tlsconfig.WithAuthorityFromFile(caPath), 27 | tlsconfig.WithServerName(cn), 28 | ) 29 | } 30 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package loggregator provides clients to send data to the Loggregator v1 and 2 | // v2 API. 3 | // 4 | // The v2 API distinguishes itself from the v1 API on three counts: 5 | // 6 | // 1) it uses gRPC, 7 | // 2) it uses a streaming connection, and 8 | // 3) it supports batching to improve performance. 9 | // 10 | // The code here provides a generic interface into the two APIs. Clients who 11 | // prefer more fine grained control may generate their own code using the 12 | // protobuf and gRPC service definitions found at: 13 | // github.com/cloudfoundry/loggregator-api. 14 | // 15 | // Note that on account of the client using batching wherein multiple 16 | // messages may be sent at once, there is no meaningful error return value 17 | // available. Each of the methods below make a best-effort at message 18 | // delivery. Even in the event of a failed send, the client will not block 19 | // callers. 20 | // 21 | // In general, use IngressClient for communicating with Loggregator's v2 API. 22 | // For Loggregator's v1 API, see v1/client.go. 23 | package loggregator 24 | -------------------------------------------------------------------------------- /conversion/tov2_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "time" 5 | 6 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 7 | "github.com/cloudfoundry/sonde-go/events" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | "google.golang.org/protobuf/proto" 11 | ) 12 | 13 | var _ = Describe("ToV2", func() { 14 | It("doesn't modify the input", func() { 15 | tags := make(map[string]string) 16 | tags["origin"] = "not-doppler" 17 | 18 | v1e := &events.Envelope{ 19 | Origin: proto.String("doppler"), 20 | EventType: events.Envelope_LogMessage.Enum(), 21 | Timestamp: proto.Int64(time.Now().UnixNano()), 22 | LogMessage: &events.LogMessage{ 23 | Message: []byte("some-log-message"), 24 | MessageType: events.LogMessage_OUT.Enum(), 25 | Timestamp: proto.Int64(time.Now().UnixNano()), 26 | }, 27 | Tags: tags, 28 | } 29 | 30 | v2e := conversion.ToV2(v1e, true) 31 | 32 | Expect(v1e.GetTags()["origin"]).To(Equal("not-doppler")) 33 | Expect(v2e.GetTags()["origin"]).To(Equal("doppler")) 34 | }) 35 | }) 36 | -------------------------------------------------------------------------------- /examples/go.mod: -------------------------------------------------------------------------------- 1 | module code.cloudfoundry.org/go-loggregator/v10/examples 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | code.cloudfoundry.org/go-loggregator/v10 v10.0.0-00010101000000-000000000000 9 | github.com/cloudfoundry/dropsonde v1.1.0 10 | google.golang.org/protobuf v1.36.10 11 | ) 12 | 13 | require ( 14 | code.cloudfoundry.org/go-diodes v0.0.0-20180905200951-72629b5276e3 // indirect 15 | code.cloudfoundry.org/tlsconfig v0.35.0 // indirect 16 | github.com/cloudfoundry/sonde-go v0.0.0-20220627221915-ff36de9c3435 // indirect 17 | github.com/fsnotify/fsnotify v1.7.0 // indirect 18 | github.com/josharian/intern v1.0.0 // indirect 19 | github.com/mailru/easyjson v0.7.7 // indirect 20 | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect 21 | golang.org/x/net v0.44.0 // indirect 22 | golang.org/x/sys v0.36.0 // indirect 23 | golang.org/x/text v0.29.0 // indirect 24 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect 25 | google.golang.org/grpc v1.75.1 // indirect 26 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 27 | ) 28 | 29 | replace code.cloudfoundry.org/go-loggregator/v10 => ../ 30 | -------------------------------------------------------------------------------- /conversion/tov1_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | var _ = Describe("Tov1", func() { 12 | It("doesn't modify the input data", func() { 13 | tags := make(map[string]string) 14 | dTags := make(map[string]*loggregator_v2.Value) 15 | 16 | tags["foo"] = "bar" 17 | dTags["foo"] = &loggregator_v2.Value{Data: &loggregator_v2.Value_Text{Text: "baz"}} 18 | 19 | v2e := &loggregator_v2.Envelope{ 20 | Message: &loggregator_v2.Envelope_Log{ 21 | Log: &loggregator_v2.Log{ 22 | Payload: []byte("hello"), 23 | }, 24 | }, 25 | Tags: tags, 26 | DeprecatedTags: dTags, 27 | } 28 | 29 | conversion.ToV1(v2e) 30 | Expect(v2e.Tags["foo"]).To(Equal("bar")) 31 | }) 32 | 33 | It("sets the application id to nil when it's not parsable into a UUID", func() { 34 | v2e := &loggregator_v2.Envelope{ 35 | Message: &loggregator_v2.Envelope_Timer{ 36 | Timer: &loggregator_v2.Timer{}, 37 | }, 38 | SourceId: "some-id", 39 | } 40 | v1e := conversion.ToV1(v2e) 41 | Expect(v1e[0].HttpStartStop.ApplicationId).To(BeNil()) 42 | }) 43 | }) 44 | -------------------------------------------------------------------------------- /pulseemitter/gauge_metric_test.go: -------------------------------------------------------------------------------- 1 | package pulseemitter_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/pulseemitter" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var _ = Describe("GaugeMetric", func() { 11 | It("prepares the envelope for delivery", func() { 12 | g := pulseemitter.NewGaugeMetric( 13 | "some-gauge", 14 | "some-unit", 15 | "my-source-id", 16 | pulseemitter.WithVersion(1, 2), 17 | ) 18 | 19 | g.Set(10.21) 20 | 21 | spy := newSpyLogClient() 22 | g.Emit(spy) 23 | 24 | e := &loggregator_v2.Envelope{ 25 | Message: &loggregator_v2.Envelope_Gauge{ 26 | Gauge: &loggregator_v2.Gauge{ 27 | Metrics: make(map[string]*loggregator_v2.GaugeValue), 28 | }, 29 | }, 30 | Tags: make(map[string]string), 31 | } 32 | 33 | for _, o := range spy.GaugeOpts() { 34 | o(e) 35 | } 36 | Expect(e.GetGauge().GetMetrics()).To(HaveLen(1)) 37 | Expect(e.GetGauge().GetMetrics()).To(HaveKey("some-gauge")) 38 | Expect(e.GetGauge().GetMetrics()["some-gauge"].GetValue()).To(Equal(10.21)) 39 | Expect(e.GetGauge().GetMetrics()["some-gauge"].GetUnit()).To(Equal("some-unit")) 40 | 41 | Expect(e.GetTags()).To(HaveKey("metric_version")) 42 | Expect(e.GetTags()["metric_version"]).To(Equal("1.2")) 43 | }) 44 | }) 45 | -------------------------------------------------------------------------------- /examples/v2_ingress/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "os" 7 | "time" 8 | 9 | "code.cloudfoundry.org/go-loggregator/v10" 10 | ) 11 | 12 | func main() { 13 | tlsConfig, err := loggregator.NewIngressTLSConfig( 14 | os.Getenv("CA_CERT_PATH"), 15 | os.Getenv("CERT_PATH"), 16 | os.Getenv("KEY_PATH"), 17 | ) 18 | if err != nil { 19 | log.Fatal("Could not create TLS config", err) 20 | } 21 | 22 | client, err := loggregator.NewIngressClient( 23 | tlsConfig, 24 | loggregator.WithAddr("localhost:3458"), 25 | ) 26 | 27 | if err != nil { 28 | log.Fatal("Could not create client", err) 29 | } 30 | 31 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 32 | defer cancel() 33 | err = client.EmitEvent( 34 | ctx, 35 | "Starting sample V2 Client", 36 | "This sample V2 client is about to emit 50 log envelopes", 37 | ) 38 | if err != nil { 39 | log.Fatalf("Failed to emit event: %s", err) 40 | } 41 | 42 | for i := 0; i < 50; i++ { 43 | client.EmitLog("some log goes here", 44 | loggregator.WithSourceInfo("v2-example-source-id", "platform", "v2-example-source-instance"), 45 | ) 46 | 47 | time.Sleep(10 * time.Millisecond) 48 | } 49 | 50 | startTime := time.Now() 51 | for i := 0; i < 5; i++ { 52 | client.EmitTimer("loop_times", startTime, time.Now()) 53 | } 54 | 55 | client.CloseSend() 56 | } 57 | -------------------------------------------------------------------------------- /scripts/generate-test-certs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -ex 4 | 5 | fullpath() { 6 | ( 7 | cd $(dirname $1) 8 | echo $PWD/$(basename $1) 9 | ) 10 | } 11 | 12 | # Place keys and certificates here 13 | depot_path=$(fullpath ./test-certs) 14 | mkdir -p ${depot_path} 15 | 16 | # Install certstrap 17 | go get -v github.com/square/certstrap 18 | 19 | # CA to distribute to loggregator certs 20 | certstrap --depot-path ${depot_path} init --passphrase '' --common-name loggregatorCA --expires "25 years" 21 | mv -f ${depot_path}/loggregatorCA.crt ${depot_path}/CA.crt 22 | mv -f ${depot_path}/loggregatorCA.key ${depot_path}/CA.key 23 | mv -f ${depot_path}/loggregatorCA.crl ${depot_path}/CA.crl 24 | 25 | # Client certificate 26 | certstrap --depot-path ${depot_path} request-cert --passphrase '' --common-name metron --domain metron 27 | certstrap --depot-path ${depot_path} sign metron --CA CA --expires "25 years" 28 | mv -f ${depot_path}/metron.crt ${depot_path}/server.crt 29 | mv -f ${depot_path}/metron.key ${depot_path}/server.key 30 | 31 | # Server certificate 32 | certstrap --depot-path ${depot_path} request-cert --passphrase '' --common-name reverselogproxy --domain reverselogproxy 33 | certstrap --depot-path ${depot_path} sign reverselogproxy --CA CA --expires "25 years" 34 | mv -f ${depot_path}/reverselogproxy.crt ${depot_path}/client.crt 35 | mv -f ${depot_path}/reverselogproxy.key ${depot_path}/client.key 36 | 37 | # Create invalid cert 38 | echo "foobar" > ${depot_path}/invalid-ca.crt 39 | -------------------------------------------------------------------------------- /rfc5424/stream.go: -------------------------------------------------------------------------------- 1 | package rfc5424 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strconv" 7 | ) 8 | 9 | // WriteTo writes the message to a stream of messages in the style defined 10 | // by RFC-5425. (It does not implement the TLS stuff described in the RFC, just 11 | // the length delimiting. 12 | func (m Message) WriteTo(w io.Writer) (int64, error) { 13 | b, err := m.MarshalBinary() 14 | if err != nil { 15 | return 0, err 16 | } 17 | n, err := fmt.Fprintf(w, "%d %s", len(b), b) 18 | 19 | return int64(n), err 20 | } 21 | 22 | func readUntilSpace(r io.Reader) ([]byte, int, error) { 23 | buf := []byte{} 24 | nbytes := 0 25 | for { 26 | b := []byte{0} 27 | n, err := r.Read(b) 28 | nbytes += n 29 | if err != nil { 30 | return nil, nbytes, err 31 | } 32 | if b[0] == ' ' { 33 | return buf, nbytes, nil 34 | } 35 | buf = append(buf, b...) 36 | } 37 | } 38 | 39 | // ReadFrom reads a single record from an RFC-5425 style stream of messages 40 | func (m *Message) ReadFrom(r io.Reader) (int64, error) { 41 | lengthBuf, n1, err := readUntilSpace(r) 42 | if err != nil { 43 | return 0, err 44 | } 45 | length, err := strconv.Atoi(string(lengthBuf)) 46 | if err != nil { 47 | return 0, err 48 | } 49 | r2 := io.LimitReader(r, int64(length)) 50 | buf, err := io.ReadAll(r2) 51 | if err != nil { 52 | return int64(n1 + len(buf)), err 53 | } 54 | if len(buf) != int(length) { 55 | return int64(n1 + len(buf)), fmt.Errorf("expected to read %d bytes, got %d", length, len(buf)) 56 | } 57 | err = m.UnmarshalBinary(buf) 58 | if err != nil { 59 | return 0, err 60 | } 61 | return int64(n1 + len(buf)), err 62 | } 63 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module code.cloudfoundry.org/go-loggregator/v10 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | code.cloudfoundry.org/go-diodes v0.0.0-20180905200951-72629b5276e3 9 | code.cloudfoundry.org/tlsconfig v0.41.0 10 | github.com/cloudfoundry/dropsonde v1.1.0 11 | github.com/cloudfoundry/sonde-go v0.0.0-20220627221915-ff36de9c3435 12 | github.com/kr/pretty v0.3.1 // indirect 13 | github.com/onsi/ginkgo/v2 v2.27.2 14 | github.com/onsi/gomega v1.38.2 15 | golang.org/x/net v0.47.0 // indirect 16 | google.golang.org/grpc v1.77.0 17 | google.golang.org/protobuf v1.36.10 18 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 19 | ) 20 | 21 | require ( 22 | github.com/Masterminds/semver/v3 v3.4.0 // indirect 23 | github.com/go-logr/logr v1.4.3 // indirect 24 | github.com/go-task/slim-sprig/v3 v3.0.0 // indirect 25 | github.com/google/go-cmp v0.7.0 // indirect 26 | github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect 27 | github.com/josharian/intern v1.0.0 // indirect 28 | github.com/kr/text v0.2.0 // indirect 29 | github.com/mailru/easyjson v0.7.7 // indirect 30 | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect 31 | github.com/nxadm/tail v1.4.8 // indirect 32 | github.com/rogpeppe/go-internal v1.13.1 // indirect 33 | go.yaml.in/yaml/v3 v3.0.4 // indirect 34 | golang.org/x/mod v0.29.0 // indirect 35 | golang.org/x/sync v0.18.0 // indirect 36 | golang.org/x/sys v0.38.0 // indirect 37 | golang.org/x/text v0.31.0 // indirect 38 | golang.org/x/tools v0.38.0 // indirect 39 | google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /examples/rlp_gateway/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os" 9 | 10 | "code.cloudfoundry.org/go-loggregator/v10" 11 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 12 | "google.golang.org/protobuf/encoding/protojson" 13 | ) 14 | 15 | func main() { 16 | rlpAddr := os.Getenv("LOG_STREAM_ADDR") 17 | if rlpAddr == "" { 18 | log.Fatal("LOG_STREAM_ADDR is required") 19 | } 20 | 21 | token := os.Getenv("TOKEN") 22 | if token == "" { 23 | log.Fatalf("TOKEN is required") 24 | } 25 | 26 | sourceId := os.Getenv("SOURCE_ID") 27 | if sourceId == "" { 28 | log.Fatalf("SOURCE_ID is required") 29 | } 30 | 31 | c := loggregator.NewRLPGatewayClient( 32 | rlpAddr, 33 | loggregator.WithRLPGatewayClientLogger(log.New(os.Stderr, "", log.LstdFlags)), 34 | loggregator.WithRLPGatewayHTTPClient(&tokenAttacher{ 35 | token: token, 36 | }), 37 | ) 38 | 39 | es := c.Stream(context.Background(), &loggregator_v2.EgressBatchRequest{ 40 | Selectors: []*loggregator_v2.Selector{ 41 | { 42 | SourceId: sourceId, 43 | Message: &loggregator_v2.Selector_Log{ 44 | Log: &loggregator_v2.LogSelector{}, 45 | }, 46 | }, 47 | }, 48 | }) 49 | 50 | for { 51 | for _, e := range es() { 52 | b, err := protojson.Marshal(e) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | fmt.Fprint(os.Stdout, b) 57 | } 58 | } 59 | } 60 | 61 | type tokenAttacher struct { 62 | token string 63 | } 64 | 65 | func (a *tokenAttacher) Do(req *http.Request) (*http.Response, error) { 66 | req.Header.Set("Authorization", a.token) 67 | return http.DefaultClient.Do(req) 68 | } 69 | -------------------------------------------------------------------------------- /rfc5424/message_test.go: -------------------------------------------------------------------------------- 1 | package rfc5424 2 | 3 | import . "gopkg.in/check.v1" 4 | 5 | var _ = Suite(&MessageTest{}) 6 | 7 | type MessageTest struct { 8 | } 9 | 10 | func (s *MessageTest) TestAddDatum(c *C) { 11 | m := Message{} 12 | m.AddDatum("id", "name", "value") 13 | c.Assert(m, DeepEquals, Message{ 14 | StructuredData: []StructuredData{ 15 | { 16 | ID: "id", 17 | Parameters: []SDParam{ 18 | {"name", "value"}, 19 | }, 20 | }, 21 | }, 22 | }) 23 | 24 | m.AddDatum("id2", "name", "value") 25 | c.Assert(m, DeepEquals, Message{ 26 | StructuredData: []StructuredData{ 27 | { 28 | ID: "id", 29 | Parameters: []SDParam{ 30 | {"name", "value"}, 31 | }, 32 | }, 33 | { 34 | ID: "id2", 35 | Parameters: []SDParam{ 36 | {"name", "value"}, 37 | }, 38 | }, 39 | }, 40 | }) 41 | 42 | m.AddDatum("id", "name2", "value2") 43 | c.Assert(m, DeepEquals, Message{ 44 | StructuredData: []StructuredData{ 45 | { 46 | ID: "id", 47 | Parameters: []SDParam{ 48 | {"name", "value"}, 49 | {"name2", "value2"}, 50 | }, 51 | }, 52 | { 53 | ID: "id2", 54 | Parameters: []SDParam{ 55 | {"name", "value"}, 56 | }, 57 | }, 58 | }, 59 | }) 60 | 61 | m.AddDatum("id", "name", "value3") 62 | c.Assert(m, DeepEquals, Message{ 63 | StructuredData: []StructuredData{ 64 | { 65 | ID: "id", 66 | Parameters: []SDParam{ 67 | {"name", "value"}, 68 | {"name2", "value2"}, 69 | {"name", "value3"}, 70 | }, 71 | }, 72 | { 73 | ID: "id2", 74 | Parameters: []SDParam{ 75 | {"name", "value"}, 76 | }, 77 | }, 78 | }, 79 | }) 80 | } 81 | -------------------------------------------------------------------------------- /examples/envelope_stream_connector/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "os" 7 | 8 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 9 | 10 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 11 | ) 12 | 13 | var allSelectors = []*loggregator_v2.Selector{ 14 | { 15 | Message: &loggregator_v2.Selector_Log{ 16 | Log: &loggregator_v2.LogSelector{}, 17 | }, 18 | }, 19 | { 20 | Message: &loggregator_v2.Selector_Counter{ 21 | Counter: &loggregator_v2.CounterSelector{}, 22 | }, 23 | }, 24 | { 25 | Message: &loggregator_v2.Selector_Gauge{ 26 | Gauge: &loggregator_v2.GaugeSelector{}, 27 | }, 28 | }, 29 | { 30 | Message: &loggregator_v2.Selector_Timer{ 31 | Timer: &loggregator_v2.TimerSelector{}, 32 | }, 33 | }, 34 | { 35 | Message: &loggregator_v2.Selector_Event{ 36 | Event: &loggregator_v2.EventSelector{}, 37 | }, 38 | }, 39 | } 40 | 41 | func main() { 42 | tlsConfig, err := loggregator.NewEgressTLSConfig( 43 | os.Getenv("CA_CERT_PATH"), 44 | os.Getenv("CERT_PATH"), 45 | os.Getenv("KEY_PATH"), 46 | ) 47 | if err != nil { 48 | log.Fatal("Could not create TLS config", err) 49 | } 50 | 51 | loggr := log.New(os.Stderr, "[", log.LstdFlags) 52 | streamConnector := loggregator.NewEnvelopeStreamConnector( 53 | os.Getenv("LOGS_API_ADDR"), 54 | tlsConfig, 55 | loggregator.WithEnvelopeStreamLogger(loggr), 56 | ) 57 | 58 | rx := streamConnector.Stream(context.Background(), &loggregator_v2.EgressBatchRequest{ 59 | ShardId: os.Getenv("SHARD_ID"), 60 | Selectors: allSelectors, 61 | UsePreferredTags: true, 62 | }) 63 | 64 | for { 65 | batch := rx() 66 | 67 | for _, e := range batch { 68 | log.Printf("%+v\n", e) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /one_to_one_envelope_batch_diode.go: -------------------------------------------------------------------------------- 1 | package loggregator 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 5 | 6 | gendiodes "code.cloudfoundry.org/go-diodes" 7 | ) 8 | 9 | // OneToOneEnvelopeBatch diode is optimized for a single writer and a single reader 10 | type OneToOneEnvelopeBatch struct { 11 | d *gendiodes.Poller 12 | } 13 | 14 | // NewOneToOneEnvelopeBatch initializes a new one to one diode for envelope 15 | // batches of a given size and alerter. The alerter is called whenever data is 16 | // dropped with an integer representing the number of envelope batches that 17 | // were dropped. 18 | func NewOneToOneEnvelopeBatch(size int, alerter gendiodes.Alerter, opts ...gendiodes.PollerConfigOption) *OneToOneEnvelopeBatch { 19 | return &OneToOneEnvelopeBatch{ 20 | d: gendiodes.NewPoller(gendiodes.NewOneToOne(size, alerter), opts...), 21 | } 22 | } 23 | 24 | // Set inserts the given V2 envelope into the diode. 25 | func (d *OneToOneEnvelopeBatch) Set(data []*loggregator_v2.Envelope) { 26 | d.d.Set(gendiodes.GenericDataType(&data)) 27 | } 28 | 29 | // TryNext returns the next envelope batch to be read from the diode. If the 30 | // diode is empty it will return a nil envelope and false for the bool. 31 | func (d *OneToOneEnvelopeBatch) TryNext() ([]*loggregator_v2.Envelope, bool) { 32 | data, ok := d.d.TryNext() 33 | if !ok { 34 | return nil, ok 35 | } 36 | 37 | return *(*[]*loggregator_v2.Envelope)(data), true 38 | } 39 | 40 | // Next will return the next envelope batch to be read from the diode. If the 41 | // diode is empty this method will block until anenvelope is available to be 42 | // read. 43 | func (d *OneToOneEnvelopeBatch) Next() []*loggregator_v2.Envelope { 44 | data := d.d.Next() 45 | if data == nil { 46 | return nil 47 | } 48 | return *(*[]*loggregator_v2.Envelope)(data) 49 | } 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-loggregator 2 | [![GoDoc][go-doc-badge]][go-doc] 3 | 4 | This is a golang client library for [Loggregator][loggregator]. 5 | 6 | If you have any questions, or want to get attention for a PR or issue please reach out on the [#logging-and-metrics channel in the cloudfoundry slack](https://cloudfoundry.slack.com/archives/CUW93AF3M) 7 | 8 | ## Versions 9 | 10 | At present, Loggregator supports two API versions: v1 (UDP) and v2 (gRPC). 11 | This library provides clients for both versions. 12 | 13 | Note that this library is also versioned. Its versions have *no* relation to 14 | the Loggregator API. 15 | 16 | ## Usage 17 | 18 | This repository should be imported as: 19 | 20 | `import loggregator "code.cloudfoundry.org/go-loggregator/v10"` 21 | 22 | ## Examples 23 | 24 | To build the examples, `cd` into the directory of the example and run `go build` 25 | 26 | ### V1 Ingress 27 | 28 | Emits envelopes to metron using dropsonde. 29 | 30 | ### V2 Ingress 31 | 32 | Emits envelopes to metron using the V2 loggregator-api. 33 | 34 | Required Environment Variables: 35 | 36 | * `CA_CERT_PATH` 37 | * `CERT_PATH` 38 | * `KEY_PATH` 39 | 40 | ### Runtime Stats 41 | 42 | Emits information about the running Go proccess using a V2 ingress client. 43 | 44 | Required Environment Variables: 45 | 46 | * `CA_CERT_PATH` 47 | * `CERT_PATH` 48 | * `KEY_PATH` 49 | 50 | ### Envelope Stream Connector 51 | 52 | Reads envelopes from the Loggregator API (e.g. Reverse Log Proxy). 53 | 54 | Required Environment Variables: 55 | 56 | * `CA_CERT_PATH` 57 | * `CERT_PATH` 58 | * `KEY_PATH` 59 | * `LOGS_API_ADDR` 60 | * `SHARD_ID` 61 | 62 | [loggregator]: https://github.com/cloudfoundry/loggregator-release 63 | [go-doc-badge]: https://godoc.org/code.cloudfoundry.org/go-loggregator?status.svg 64 | [go-doc]: https://godoc.org/code.cloudfoundry.org/go-loggregator 65 | -------------------------------------------------------------------------------- /rfc5424/message.go: -------------------------------------------------------------------------------- 1 | // Pacakge rfc5424 is a library for parsing and serializing RFC-5424 structured 2 | // syslog messages. 3 | // 4 | // Example usage: 5 | // 6 | // m := rfc5424.Message{ 7 | // Priority: rfc5424.Daemon | rfc5424.Info, 8 | // Timestamp: time.Now(), 9 | // Hostname: "myhostname", 10 | // AppName: "someapp", 11 | // Message: []byte("Hello, World!"), 12 | // } 13 | // m.AddDatum("foo@1234", "Revision", "1.2.3.4") 14 | // m.WriteTo(os.Stdout) 15 | // 16 | // Produces output like: 17 | // 18 | // 107 <7>1 2016-02-28T09:57:10.804642398-05:00 myhostname someapp - - [foo@1234 Revision="1.2.3.4"] Hello, World! 19 | // 20 | // You can also use the library to parse syslog messages: 21 | // 22 | // m := rfc5424.Message{} 23 | // _, err := m.ReadFrom(os.Stdin) 24 | // fmt.Printf("%s\n", m.Message) 25 | package rfc5424 26 | 27 | import "time" 28 | 29 | // Message represents a log message as defined by RFC-5424 30 | // (https://tools.ietf.org/html/rfc5424) 31 | type Message struct { 32 | Priority Priority 33 | Timestamp time.Time 34 | UseUTC bool 35 | Hostname string 36 | AppName string 37 | ProcessID string 38 | MessageID string 39 | StructuredData []StructuredData 40 | Message []byte 41 | } 42 | 43 | // SDParam represents parameters for structured data 44 | type SDParam struct { 45 | Name string 46 | Value string 47 | } 48 | 49 | // StructuredData represents structured data within a log message 50 | type StructuredData struct { 51 | ID string 52 | Parameters []SDParam 53 | } 54 | 55 | // AddDatum adds structured data to a log message 56 | func (m *Message) AddDatum(ID string, Name string, Value string) { 57 | if m.StructuredData == nil { 58 | m.StructuredData = []StructuredData{} 59 | } 60 | for i, sd := range m.StructuredData { 61 | if sd.ID == ID { 62 | sd.Parameters = append(sd.Parameters, SDParam{Name: Name, Value: Value}) 63 | m.StructuredData[i] = sd 64 | return 65 | } 66 | } 67 | 68 | m.StructuredData = append(m.StructuredData, StructuredData{ 69 | ID: ID, 70 | Parameters: []SDParam{ 71 | { 72 | Name: Name, 73 | Value: Value, 74 | }, 75 | }, 76 | }) 77 | } 78 | -------------------------------------------------------------------------------- /pulseemitter/gauge_metric.go: -------------------------------------------------------------------------------- 1 | package pulseemitter 2 | 3 | import ( 4 | "math" 5 | "sync/atomic" 6 | 7 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 8 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 9 | "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | // GaugeMetric is used by the pulse emitter to emit gauge metrics to the 13 | // LogClient. 14 | type GaugeMetric interface { 15 | // Set sets the current value of the gauge metric. 16 | Set(n float64) 17 | 18 | // Emit sends the counter values to the LogClient. 19 | Emit(c LogClient) 20 | } 21 | 22 | // gaugeMetric is used by the pulse emitter to emit gauge metrics to the 23 | // LogClient. 24 | type gaugeMetric struct { 25 | name string 26 | unit string 27 | sourceID string 28 | value uint64 29 | tags map[string]string 30 | } 31 | 32 | // NewGaugeMetric returns a new gaugeMetric that has a value that can be set 33 | // and emitted via a LogClient. 34 | func NewGaugeMetric(name, unit, sourceID string, opts ...MetricOption) GaugeMetric { 35 | g := &gaugeMetric{ 36 | name: name, 37 | unit: unit, 38 | sourceID: sourceID, 39 | tags: make(map[string]string), 40 | } 41 | 42 | for _, opt := range opts { 43 | opt(g.tags) 44 | } 45 | 46 | return g 47 | } 48 | 49 | // Set will set the current value of the gauge metric to the given number. 50 | func (g *gaugeMetric) Set(n float64) { 51 | atomic.StoreUint64(&g.value, toUint64(n, 2)) 52 | } 53 | 54 | // Emit will send the current value and tagging options to the LogClient to 55 | // be emitted. 56 | func (g *gaugeMetric) Emit(c LogClient) { 57 | options := []loggregator.EmitGaugeOption{ 58 | loggregator.WithGaugeValue( 59 | g.name, 60 | toFloat64(atomic.LoadUint64(&g.value), 2), 61 | g.unit, 62 | ), 63 | g.sourceIDOption, 64 | } 65 | 66 | for k, v := range g.tags { 67 | options = append(options, loggregator.WithEnvelopeTag(k, v)) 68 | } 69 | 70 | c.EmitGauge(options...) 71 | } 72 | 73 | func (g *gaugeMetric) sourceIDOption(p proto.Message) { 74 | env, ok := p.(*loggregator_v2.Envelope) 75 | if ok { 76 | env.SourceId = g.sourceID 77 | } 78 | } 79 | 80 | func toFloat64(v uint64, precision int) float64 { 81 | return float64(v) / math.Pow(10.0, float64(precision)) 82 | } 83 | 84 | func toUint64(v float64, precision int) uint64 { 85 | return uint64(v * math.Pow(10.0, float64(precision))) 86 | } 87 | -------------------------------------------------------------------------------- /servers_test.go: -------------------------------------------------------------------------------- 1 | package loggregator_test 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "net" 7 | 8 | "google.golang.org/grpc" 9 | "google.golang.org/grpc/credentials" 10 | 11 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 12 | "code.cloudfoundry.org/tlsconfig" 13 | ) 14 | 15 | type testIngressServer struct { 16 | loggregator_v2.UnimplementedIngressServer 17 | 18 | receivers chan loggregator_v2.Ingress_BatchSenderServer 19 | sendReceiver chan *loggregator_v2.EnvelopeBatch 20 | addr string 21 | tlsConfig *tls.Config 22 | grpcServer *grpc.Server 23 | grpc.Stream 24 | } 25 | 26 | func newTestIngressServer(serverCert, serverKey, caCert string) (*testIngressServer, error) { 27 | tlsConfig, err := tlsconfig.Build( 28 | tlsconfig.WithInternalServiceDefaults(), 29 | tlsconfig.WithIdentityFromFile(serverCert, serverKey), 30 | ).Server( 31 | tlsconfig.WithClientAuthenticationFromFile(caCert), 32 | ) 33 | 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | return &testIngressServer{ 39 | tlsConfig: tlsConfig, 40 | receivers: make(chan loggregator_v2.Ingress_BatchSenderServer), 41 | sendReceiver: make(chan *loggregator_v2.EnvelopeBatch, 100), 42 | addr: "localhost:0", 43 | }, nil 44 | } 45 | 46 | func (*testIngressServer) Sender(srv loggregator_v2.Ingress_SenderServer) error { 47 | return nil 48 | } 49 | 50 | func (t *testIngressServer) BatchSender(srv loggregator_v2.Ingress_BatchSenderServer) error { 51 | t.receivers <- srv 52 | 53 | <-srv.Context().Done() 54 | 55 | return nil 56 | } 57 | 58 | func (t *testIngressServer) Send(_ context.Context, b *loggregator_v2.EnvelopeBatch) (*loggregator_v2.SendResponse, error) { 59 | t.sendReceiver <- b 60 | return &loggregator_v2.SendResponse{}, nil 61 | } 62 | 63 | func (t *testIngressServer) start() error { 64 | listener, err := net.Listen("tcp4", t.addr) 65 | if err != nil { 66 | return err 67 | } 68 | t.addr = listener.Addr().String() 69 | 70 | var opts []grpc.ServerOption 71 | if t.tlsConfig != nil { 72 | opts = append(opts, grpc.Creds(credentials.NewTLS(t.tlsConfig))) 73 | } 74 | t.grpcServer = grpc.NewServer(opts...) 75 | 76 | loggregator_v2.RegisterIngressServer(t.grpcServer, t) 77 | 78 | go func() { _ = t.grpcServer.Serve(listener) }() 79 | 80 | return nil 81 | } 82 | 83 | func (t *testIngressServer) stop() { 84 | t.grpcServer.Stop() 85 | } 86 | -------------------------------------------------------------------------------- /pulseemitter/pulse_emitter_test.go: -------------------------------------------------------------------------------- 1 | package pulseemitter_test 2 | 3 | import ( 4 | "time" 5 | 6 | "code.cloudfoundry.org/go-loggregator/v10/pulseemitter" 7 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 8 | 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var _ = Describe("Pulse EmitterClient", func() { 14 | It("emits a counter with a zero delta", func() { 15 | spyLogClient := newSpyLogClient() 16 | client := pulseemitter.New( 17 | spyLogClient, 18 | pulseemitter.WithPulseInterval(50*time.Millisecond), 19 | pulseemitter.WithSourceID("my-source-id"), 20 | ) 21 | 22 | client.NewCounterMetric("some-name") 23 | Eventually(spyLogClient.CounterName).Should(Equal("some-name")) 24 | 25 | e := &loggregator_v2.Envelope{ 26 | Message: &loggregator_v2.Envelope_Counter{ 27 | Counter: &loggregator_v2.Counter{}, 28 | }, 29 | } 30 | for _, o := range spyLogClient.CounterOpts() { 31 | o(e) 32 | } 33 | Expect(e.GetCounter().GetDelta()).To(Equal(uint64(0))) 34 | Expect(e.GetSourceId()).To(Equal("my-source-id")) 35 | }) 36 | 37 | It("emits a gauge with a zero value", func() { 38 | spyLogClient := newSpyLogClient() 39 | client := pulseemitter.New( 40 | spyLogClient, 41 | pulseemitter.WithPulseInterval(50*time.Millisecond), 42 | pulseemitter.WithSourceID("my-source-id"), 43 | ) 44 | 45 | client.NewGaugeMetric("some-name", "some-unit") 46 | Eventually(spyLogClient.GaugeOpts).Should(HaveLen(2)) 47 | 48 | e := &loggregator_v2.Envelope{ 49 | Message: &loggregator_v2.Envelope_Gauge{ 50 | Gauge: &loggregator_v2.Gauge{ 51 | Metrics: make(map[string]*loggregator_v2.GaugeValue), 52 | }, 53 | }, 54 | } 55 | for _, o := range spyLogClient.GaugeOpts() { 56 | o(e) 57 | } 58 | Expect(e.GetGauge().GetMetrics()).To(HaveLen(1)) 59 | Expect(e.GetGauge().GetMetrics()).To(HaveKey("some-name")) 60 | Expect(e.GetGauge().GetMetrics()["some-name"].GetUnit()).To(Equal("some-unit")) 61 | Expect(e.GetGauge().GetMetrics()["some-name"].GetValue()).To(Equal(0.0)) 62 | Expect(e.GetSourceId()).To(Equal("my-source-id")) 63 | }) 64 | 65 | It("pulses", func() { 66 | spyLogClient := newSpyLogClient() 67 | client := pulseemitter.New( 68 | spyLogClient, 69 | pulseemitter.WithPulseInterval(time.Millisecond), 70 | ) 71 | 72 | client.NewGaugeMetric("some-name", "some-unit") 73 | Eventually(spyLogClient.GaugeCallCount).Should(BeNumerically(">", 1)) 74 | }) 75 | }) 76 | -------------------------------------------------------------------------------- /conversion/error_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | 7 | "github.com/cloudfoundry/sonde-go/events" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | "google.golang.org/protobuf/proto" 11 | ) 12 | 13 | var _ = Describe("HTTP", func() { 14 | Context("given a v1 envelope", func() { 15 | It("converts to a v2 envelope", func() { 16 | v1Envelope := &events.Envelope{ 17 | EventType: events.Envelope_Error.Enum(), 18 | Origin: proto.String("fake-origin"), 19 | Deployment: proto.String("some-deployment"), 20 | Job: proto.String("some-job"), 21 | Index: proto.String("some-index"), 22 | Ip: proto.String("some-ip"), 23 | Error: &events.Error{ 24 | Source: proto.String("test-source"), 25 | Code: proto.Int32(12345), 26 | Message: proto.String("test-message"), 27 | }, 28 | } 29 | 30 | expectedV2Envelope := &loggregator_v2.Envelope{ 31 | DeprecatedTags: map[string]*loggregator_v2.Value{ 32 | "__v1_type": {Data: &loggregator_v2.Value_Text{Text: "Error"}}, 33 | "source": {Data: &loggregator_v2.Value_Text{Text: "test-source"}}, 34 | "code": {Data: &loggregator_v2.Value_Text{Text: "12345"}}, 35 | "origin": {Data: &loggregator_v2.Value_Text{Text: "fake-origin"}}, 36 | "deployment": {Data: &loggregator_v2.Value_Text{Text: "some-deployment"}}, 37 | "job": {Data: &loggregator_v2.Value_Text{Text: "some-job"}}, 38 | "index": {Data: &loggregator_v2.Value_Text{Text: "some-index"}}, 39 | "ip": {Data: &loggregator_v2.Value_Text{Text: "some-ip"}}, 40 | }, 41 | Message: &loggregator_v2.Envelope_Log{ 42 | Log: &loggregator_v2.Log{ 43 | Payload: []byte("test-message"), 44 | Type: loggregator_v2.Log_OUT, 45 | }, 46 | }, 47 | } 48 | 49 | converted := conversion.ToV2(v1Envelope, false) 50 | 51 | _, err := proto.Marshal(converted) 52 | Expect(err).ToNot(HaveOccurred()) 53 | 54 | for k, v := range expectedV2Envelope.DeprecatedTags { 55 | Expect(proto.Equal(converted.GetDeprecatedTags()[k], v)).To(BeTrue()) 56 | } 57 | 58 | // Expect(converted.GetError().GetSource()).To(Equal(expectedV2Envelope.GetError().GetSource())) 59 | // Expect(converted.GetError().GetCode()).To(Equal(expectedV2Envelope.GetError().GetCode())) 60 | Expect(string(converted.GetLog().GetPayload())).To(Equal(string(expectedV2Envelope.GetLog().GetPayload()))) 61 | }) 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /rfc5424/stream_test.go: -------------------------------------------------------------------------------- 1 | package rfc5424 2 | 3 | import ( 4 | "bytes" 5 | 6 | . "gopkg.in/check.v1" 7 | ) 8 | 9 | var _ = Suite(&StreamTest{}) 10 | 11 | type StreamTest struct { 12 | } 13 | 14 | func (s *StreamTest) TestCanReadAndWrite(c *C) { 15 | stream := bytes.Buffer{} 16 | for i := 0; i < 4; i++ { 17 | m := Message{Priority: Priority(i), Timestamp: T("2003-08-24T05:14:15.000003-07:00")} 18 | nbytes, err := m.WriteTo(&stream) 19 | c.Assert(err, IsNil) 20 | c.Assert(nbytes, Equals, int64(50)) 21 | } 22 | 23 | c.Assert(stream.String(), Equals, 24 | `47 <0>1 2003-08-24T05:14:15.000003-07:00 - - - - -`+ 25 | `47 <1>1 2003-08-24T05:14:15.000003-07:00 - - - - -`+ 26 | `47 <2>1 2003-08-24T05:14:15.000003-07:00 - - - - -`+ 27 | `47 <3>1 2003-08-24T05:14:15.000003-07:00 - - - - -`) 28 | 29 | for i := 0; i < 4; i++ { 30 | m := Message{Priority: Priority(i << 3)} 31 | nbytes, err := m.ReadFrom(&stream) 32 | c.Assert(err, IsNil) 33 | c.Assert(nbytes, Equals, int64(50)) 34 | c.Assert(m, DeepEquals, Message{Priority: Priority(i), 35 | Timestamp: T("2003-08-24T05:14:15.000003-07:00"), 36 | StructuredData: []StructuredData{}}) 37 | } 38 | } 39 | 40 | func (s *StreamTest) TestUtcTimestamps(c *C) { 41 | stream := bytes.Buffer{} 42 | for i := 0; i < 4; i++ { 43 | m := Message{Priority: Priority(i), Timestamp: T("2003-08-24T05:14:15.000003+00:00"), UseUTC: true} 44 | nbytes, err := m.WriteTo(&stream) 45 | c.Assert(err, IsNil) 46 | c.Assert(nbytes, Equals, int64(45)) 47 | } 48 | 49 | c.Assert(stream.String(), Equals, 50 | `42 <0>1 2003-08-24T05:14:15.000003Z - - - - -`+ 51 | `42 <1>1 2003-08-24T05:14:15.000003Z - - - - -`+ 52 | `42 <2>1 2003-08-24T05:14:15.000003Z - - - - -`+ 53 | `42 <3>1 2003-08-24T05:14:15.000003Z - - - - -`) 54 | 55 | for i := 0; i < 4; i++ { 56 | m := Message{Priority: Priority(i << 3)} 57 | nbytes, err := m.ReadFrom(&stream) 58 | c.Assert(err, IsNil) 59 | c.Assert(nbytes, Equals, int64(45)) 60 | c.Assert(m, DeepEquals, Message{Priority: Priority(i), 61 | Timestamp: UTC("2003-08-24T05:14:15.000003Z"), 62 | StructuredData: []StructuredData{}, 63 | UseUTC: true, 64 | }) 65 | } 66 | } 67 | 68 | func (s *StreamTest) TestRejectsInvalidStream(c *C) { 69 | stream := bytes.NewBufferString(`99 <0>1 2003-08-24T05:14:15.000003-07:00 - - - - -`) 70 | for i := 0; i < 4; i++ { 71 | m := Message{Priority: Priority(i << 3)} 72 | _, err := m.ReadFrom(stream) 73 | c.Assert(err, Not(IsNil)) 74 | } 75 | } 76 | 77 | func (s *StreamTest) TestRejectsInvalidStream2(c *C) { 78 | stream := bytes.NewBufferString(`0 <0>1 2003-08-24T05:14:15.000003-07:00 - - - - -`) 79 | for i := 0; i < 4; i++ { 80 | m := Message{Priority: Priority(i << 3)} 81 | _, err := m.ReadFrom(stream) 82 | c.Assert(err, Not(IsNil)) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /pulseemitter/counter_metric.go: -------------------------------------------------------------------------------- 1 | package pulseemitter 2 | 3 | import ( 4 | "fmt" 5 | "sync/atomic" 6 | 7 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 8 | "google.golang.org/protobuf/proto" 9 | 10 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 11 | ) 12 | 13 | // MetricOption defines a function type that can be used to configure tags for 14 | // many types of metrics. 15 | type MetricOption func(map[string]string) 16 | 17 | // WithVersion will apply a `metric_version` tag to all envelopes sent about 18 | // the metric. 19 | func WithVersion(major, minor uint) MetricOption { 20 | return WithTags(map[string]string{ 21 | "metric_version": fmt.Sprintf("%d.%d", major, minor), 22 | }) 23 | } 24 | 25 | // WithTags will set the tags to apply to every envelopes sent about the 26 | // metric.. 27 | func WithTags(tags map[string]string) MetricOption { 28 | return func(c map[string]string) { 29 | for k, v := range tags { 30 | c[k] = v 31 | } 32 | } 33 | } 34 | 35 | // counterMetric is used by the pulse emitter to emit counter metrics to the 36 | // LogClient. 37 | type counterMetric struct { 38 | name string 39 | sourceID string 40 | delta uint64 41 | tags map[string]string 42 | } 43 | 44 | // CounterMetric is used by the pulse emitter to emit counter metrics to the 45 | // LogClient. 46 | type CounterMetric interface { 47 | // Increment increases the counter's delta by the given value 48 | Increment(c uint64) 49 | 50 | // Emit sends the counter values to the LogClient. 51 | Emit(c LogClient) 52 | } 53 | 54 | // NewCounterMetric returns a new counterMetric that can be incremented and 55 | // emitted via a LogClient. 56 | func NewCounterMetric(name, sourceID string, opts ...MetricOption) CounterMetric { 57 | m := &counterMetric{ 58 | name: name, 59 | sourceID: sourceID, 60 | tags: make(map[string]string), 61 | } 62 | 63 | for _, opt := range opts { 64 | opt(m.tags) 65 | } 66 | 67 | return m 68 | } 69 | 70 | // Increment will add the given uint64 to the current delta. 71 | func (m *counterMetric) Increment(c uint64) { 72 | atomic.AddUint64(&m.delta, c) 73 | } 74 | 75 | // Emit will send the current delta and tagging options to the LogClient to 76 | // be emitted. The delta on the counterMetric will be reset to 0. 77 | func (m *counterMetric) Emit(c LogClient) { 78 | d := atomic.SwapUint64(&m.delta, 0) 79 | options := []loggregator.EmitCounterOption{ 80 | loggregator.WithDelta(d), 81 | m.sourceIDOption, 82 | } 83 | 84 | for k, v := range m.tags { 85 | options = append(options, loggregator.WithEnvelopeTag(k, v)) 86 | } 87 | 88 | c.EmitCounter(m.name, options...) 89 | } 90 | 91 | func (m *counterMetric) sourceIDOption(p proto.Message) { 92 | env, ok := p.(*loggregator_v2.Envelope) 93 | if ok { 94 | env.SourceId = m.sourceID 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /pulseemitter/pulse_emitter.go: -------------------------------------------------------------------------------- 1 | package pulseemitter 2 | 3 | import ( 4 | "time" 5 | 6 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 7 | ) 8 | 9 | // LogClient is the client used by PulseEmitter to emit metrics. This would 10 | // usually be the go-loggregator v2 client. 11 | type LogClient interface { 12 | EmitCounter(name string, opts ...loggregator.EmitCounterOption) 13 | EmitGauge(opts ...loggregator.EmitGaugeOption) 14 | } 15 | 16 | type emitter interface { 17 | Emit(c LogClient) 18 | } 19 | 20 | // PulseEmitterOption is a function type that is used to configure optional 21 | // settings for a PulseEmitter. 22 | type PulseEmitterOption func(*PulseEmitter) 23 | 24 | // WithPulseInterval is a PulseEmitterOption for setting the pulsing interval. 25 | func WithPulseInterval(d time.Duration) PulseEmitterOption { 26 | return func(c *PulseEmitter) { 27 | c.pulseInterval = d 28 | } 29 | } 30 | 31 | // WithSourceID is a PulseEmitterOption for setting the source ID that will be 32 | // set on all outgoing metrics. 33 | func WithSourceID(id string) PulseEmitterOption { 34 | return func(c *PulseEmitter) { 35 | c.sourceID = id 36 | } 37 | } 38 | 39 | // PulseEmitter will emit metrics on a given interval. 40 | type PulseEmitter struct { 41 | logClient LogClient 42 | 43 | pulseInterval time.Duration 44 | sourceID string 45 | } 46 | 47 | // New returns a PulseEmitter configured with the given LogClient and 48 | // PulseEmitterOptions. The default pulse interval is 60 seconds. 49 | func New(c LogClient, opts ...PulseEmitterOption) *PulseEmitter { 50 | pe := &PulseEmitter{ 51 | pulseInterval: 60 * time.Second, 52 | logClient: c, 53 | } 54 | 55 | for _, opt := range opts { 56 | opt(pe) 57 | } 58 | 59 | return pe 60 | } 61 | 62 | // NewCounterMetric returns a CounterMetric that can be incremented. After 63 | // calling NewCounterMetric the counter metric will begin to be emitted on the 64 | // interval configured on the PulseEmitter. If the counter metrics value has 65 | // not changed since last emitted a 0 value will be emitted. Every time the 66 | // counter metric is emitted, its delta is reset to 0. 67 | func (c *PulseEmitter) NewCounterMetric(name string, opts ...MetricOption) CounterMetric { 68 | m := NewCounterMetric(name, c.sourceID, opts...) 69 | go c.pulse(m) 70 | 71 | return m 72 | } 73 | 74 | // NewGaugeMetric returns a GaugeMetric that has a value that can be set. 75 | // After calling NewGaugeMetric the gauge metric will begin to be emitted on 76 | // the interval configured on the PulseEmitter. When emitting the gauge 77 | // metric, it will use the last value given when calling set on the gauge 78 | // metric. 79 | func (c *PulseEmitter) NewGaugeMetric(name, unit string, opts ...MetricOption) GaugeMetric { 80 | g := NewGaugeMetric(name, unit, c.sourceID, opts...) 81 | go c.pulse(g) 82 | 83 | return g 84 | } 85 | 86 | func (c *PulseEmitter) pulse(e emitter) { 87 | for range time.Tick(c.pulseInterval) { 88 | e.Emit(c.logClient) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /pulseemitter/counter_metric_test.go: -------------------------------------------------------------------------------- 1 | package pulseemitter_test 2 | 3 | import ( 4 | "sync" 5 | 6 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 7 | "code.cloudfoundry.org/go-loggregator/v10/pulseemitter" 8 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 9 | 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | var _ = Describe("CounterMetric", func() { 15 | Context("Emit", func() { 16 | It("prepares an envelope for delivery", func() { 17 | metric := pulseemitter.NewCounterMetric( 18 | "name", 19 | "my-source-id", 20 | pulseemitter.WithVersion(1, 2), 21 | ) 22 | 23 | metric.Increment(10) 24 | 25 | spy := newSpyLogClient() 26 | metric.Emit(spy) 27 | Expect(spy.CounterName()).To(Equal("name")) 28 | 29 | e := &loggregator_v2.Envelope{ 30 | Message: &loggregator_v2.Envelope_Counter{ 31 | Counter: &loggregator_v2.Counter{}, 32 | }, 33 | Tags: make(map[string]string), 34 | } 35 | for _, o := range spy.CounterOpts() { 36 | o(e) 37 | } 38 | 39 | Expect(e.GetCounter().GetDelta()).To(Equal(uint64(10))) 40 | Expect(e.Tags["metric_version"]).To(Equal("1.2")) 41 | }) 42 | 43 | It("decrements its value on success", func() { 44 | metric := pulseemitter.NewCounterMetric("name", "my-source-id") 45 | spy := newSpyLogClient() 46 | 47 | metric.Increment(10) 48 | metric.Emit(spy) 49 | 50 | metric.Emit(spy) 51 | e := &loggregator_v2.Envelope{ 52 | Message: &loggregator_v2.Envelope_Counter{ 53 | Counter: &loggregator_v2.Counter{}, 54 | }, 55 | } 56 | 57 | for _, o := range spy.counterOpts { 58 | o(e) 59 | } 60 | 61 | Expect(e.GetCounter().GetDelta()).To(Equal(uint64(0))) 62 | }) 63 | }) 64 | }) 65 | 66 | type spyLogClient struct { 67 | mu sync.Mutex 68 | name string 69 | counterOpts []loggregator.EmitCounterOption 70 | gaugeOpts []loggregator.EmitGaugeOption 71 | gaugeCallCount int 72 | } 73 | 74 | func newSpyLogClient() *spyLogClient { 75 | return &spyLogClient{} 76 | } 77 | 78 | func (s *spyLogClient) EmitCounter(name string, opts ...loggregator.EmitCounterOption) { 79 | s.mu.Lock() 80 | defer s.mu.Unlock() 81 | s.name = name 82 | s.counterOpts = opts 83 | } 84 | 85 | func (s *spyLogClient) EmitGauge(opts ...loggregator.EmitGaugeOption) { 86 | s.mu.Lock() 87 | defer s.mu.Unlock() 88 | s.gaugeCallCount++ 89 | s.gaugeOpts = opts 90 | } 91 | 92 | func (s *spyLogClient) CounterName() string { 93 | s.mu.Lock() 94 | defer s.mu.Unlock() 95 | 96 | return s.name 97 | } 98 | 99 | func (s *spyLogClient) CounterOpts() []loggregator.EmitCounterOption { 100 | s.mu.Lock() 101 | defer s.mu.Unlock() 102 | 103 | return s.counterOpts 104 | } 105 | 106 | func (s *spyLogClient) GaugeOpts() []loggregator.EmitGaugeOption { 107 | s.mu.Lock() 108 | defer s.mu.Unlock() 109 | 110 | return s.gaugeOpts 111 | } 112 | 113 | func (s *spyLogClient) GaugeCallCount() int { 114 | s.mu.Lock() 115 | defer s.mu.Unlock() 116 | 117 | return s.gaugeCallCount 118 | } 119 | -------------------------------------------------------------------------------- /runtimeemitter/runtime_emitter_test.go: -------------------------------------------------------------------------------- 1 | package runtimeemitter_test 2 | 3 | import ( 4 | "runtime" 5 | "time" 6 | 7 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 8 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 9 | "code.cloudfoundry.org/go-loggregator/v10/runtimeemitter" 10 | 11 | . "github.com/onsi/ginkgo/v2" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | var _ = Describe("RuntimeEmitter", func() { 16 | It("emits go runtime metrics on an interval", func() { 17 | v2Client := newSpyV2Client() 18 | emitter := runtimeemitter.New(v2Client, 19 | runtimeemitter.WithInterval(10*time.Millisecond), 20 | ) 21 | 22 | go emitter.Run() 23 | 24 | Eventually(v2Client.emitGaugeCalled).Should(BeNumerically(">", 1)) 25 | }) 26 | 27 | It("emits expected runtime metrics", func() { 28 | // Force garbage collection so GC stats aren't empty 29 | runtime.GC() 30 | 31 | v2Client := newSpyV2Client() 32 | emitter := runtimeemitter.New(v2Client, 33 | runtimeemitter.WithInterval(10*time.Millisecond), 34 | ) 35 | 36 | go emitter.Run() 37 | 38 | var env *loggregator_v2.Envelope 39 | Eventually(v2Client.envelopes).Should(Receive(&env)) 40 | Expect(env.GetGauge()).ToNot(BeNil()) 41 | 42 | metrics := env.GetGauge().Metrics 43 | Expect(metrics["memoryStats.numBytesAllocatedHeap"].Value).To(BeNumerically(">", 0.0)) 44 | Expect(metrics["memoryStats.numBytesAllocatedHeap"].Unit).To(Equal("Bytes")) 45 | 46 | Expect(metrics["memoryStats.numBytesAllocatedStack"].Value).To(BeNumerically(">", 0.0)) 47 | Expect(metrics["memoryStats.numBytesAllocatedStack"].Unit).To(Equal("Bytes")) 48 | 49 | Expect(metrics["numGoRoutines"].Value).To(BeNumerically(">", 0.0)) 50 | Expect(metrics["numGoRoutines"].Unit).To(Equal("Count")) 51 | 52 | Expect(metrics["memoryStats.lastGCPauseTimeNS"].Value).To(BeNumerically(">", 0.0)) 53 | Expect(metrics["memoryStats.lastGCPauseTimeNS"].Unit).To(Equal("ns")) 54 | }) 55 | 56 | Describe("V1 Emitter", func() { 57 | It("emits go runtime metrics on an interval", func() { 58 | v1Client := newSpyV1Client() 59 | emitter := runtimeemitter.NewV1(v1Client, 60 | runtimeemitter.WithInterval(10*time.Millisecond), 61 | ) 62 | 63 | go emitter.Run() 64 | 65 | Eventually(v1Client.sendCalled).Should(BeNumerically(">", 4)) 66 | }) 67 | }) 68 | }) 69 | 70 | type SpyV2Client struct { 71 | envelopes chan *loggregator_v2.Envelope 72 | } 73 | 74 | func newSpyV2Client() *SpyV2Client { 75 | return &SpyV2Client{ 76 | envelopes: make(chan *loggregator_v2.Envelope, 100), 77 | } 78 | } 79 | 80 | func (s *SpyV2Client) emitGaugeCalled() int64 { 81 | return int64(len(s.envelopes)) 82 | } 83 | 84 | func (s *SpyV2Client) EmitGauge(opts ...loggregator.EmitGaugeOption) { 85 | env := &loggregator_v2.Envelope{ 86 | Message: &loggregator_v2.Envelope_Gauge{ 87 | Gauge: &loggregator_v2.Gauge{ 88 | Metrics: make(map[string]*loggregator_v2.GaugeValue), 89 | }, 90 | }, 91 | } 92 | 93 | for _, o := range opts { 94 | o(env) 95 | } 96 | 97 | s.envelopes <- env 98 | } 99 | 100 | type SpyV1Client struct { 101 | called chan bool 102 | } 103 | 104 | func newSpyV1Client() *SpyV1Client { 105 | return &SpyV1Client{ 106 | called: make(chan bool, 100), 107 | } 108 | } 109 | 110 | func (c *SpyV1Client) sendCalled() int64 { 111 | return int64(len(c.called)) 112 | } 113 | 114 | func (c *SpyV1Client) SendComponentMetric(name string, value float64, unit string) error { 115 | c.called <- true 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /runtimeemitter/runtime_emitter.go: -------------------------------------------------------------------------------- 1 | package runtimeemitter 2 | 3 | import ( 4 | "runtime" 5 | "time" 6 | 7 | "code.cloudfoundry.org/go-loggregator/v10" 8 | ) 9 | 10 | // Emitter will emit a gauge with runtime stats via the sender on the given 11 | // interval. default interval is 15 seconds. 12 | type Emitter struct { 13 | interval time.Duration 14 | sender valueSender 15 | // sender Sender 16 | } 17 | 18 | type valueSender interface { 19 | send(heap, stack, gc, goroutines float64) 20 | } 21 | 22 | // Sender is the interface of the client that can be used to emit gauge 23 | // metrics. 24 | type Sender interface { 25 | EmitGauge(opts ...loggregator.EmitGaugeOption) 26 | } 27 | 28 | // RuntimeEmitterOption is the option provides configuration for an Emitter. 29 | type RuntimeEmitterOption func(e *Emitter) 30 | 31 | // WithInterval returns a RuntimeEmitterOption to configure the interval at 32 | // which the runtime emitter emits gauges. 33 | func WithInterval(d time.Duration) RuntimeEmitterOption { 34 | return func(e *Emitter) { 35 | e.interval = d 36 | } 37 | } 38 | 39 | // New returns an Emitter that is configured with the given sender and 40 | // RuntimeEmitterOptions. 41 | func New(sender Sender, opts ...RuntimeEmitterOption) *Emitter { 42 | e := &Emitter{ 43 | sender: v2Sender{sender: sender}, 44 | interval: 10 * time.Second, 45 | } 46 | 47 | for _, o := range opts { 48 | o(e) 49 | } 50 | 51 | return e 52 | } 53 | 54 | // V1Sender is the interface of the v1 client that can be used to emit value 55 | // metrics. 56 | type V1Sender interface { 57 | SendComponentMetric(name string, value float64, unit string) error 58 | } 59 | 60 | // NewV1 returns an Emitter that is configured with the given v1 sender and 61 | // RuntimeEmitterOptions. 62 | func NewV1(sender V1Sender, opts ...RuntimeEmitterOption) *Emitter { 63 | e := &Emitter{ 64 | sender: v1Sender{sender: sender}, 65 | interval: 10 * time.Second, 66 | } 67 | 68 | for _, o := range opts { 69 | o(e) 70 | } 71 | 72 | return e 73 | } 74 | 75 | // Run starts the ticker with the configured interval and emits a gauge on 76 | // that interval. This method will block but the user may run in a go routine. 77 | func (e *Emitter) Run() { 78 | for range time.Tick(e.interval) { 79 | memstats := &runtime.MemStats{} 80 | runtime.ReadMemStats(memstats) 81 | e.sender.send( 82 | float64(memstats.HeapAlloc), 83 | float64(memstats.StackInuse), 84 | float64(memstats.PauseNs[(memstats.NumGC+255)%256]), 85 | float64(runtime.NumGoroutine()), 86 | ) 87 | } 88 | } 89 | 90 | type v2Sender struct { 91 | sender Sender 92 | } 93 | 94 | func (s v2Sender) send(heap, stack, gc, goroutines float64) { 95 | s.sender.EmitGauge( 96 | loggregator.WithGaugeValue("memoryStats.numBytesAllocatedHeap", heap, "Bytes"), 97 | loggregator.WithGaugeValue("memoryStats.numBytesAllocatedStack", stack, "Bytes"), 98 | loggregator.WithGaugeValue("memoryStats.lastGCPauseTimeNS", gc, "ns"), 99 | loggregator.WithGaugeValue("numGoRoutines", goroutines, "Count"), 100 | ) 101 | } 102 | 103 | type v1Sender struct { 104 | sender V1Sender 105 | } 106 | 107 | func (s v1Sender) send(heap, stack, gc, goroutines float64) { 108 | _ = s.sender.SendComponentMetric("memoryStats.numBytesAllocatedHeap", heap, "Bytes") 109 | _ = s.sender.SendComponentMetric("memoryStats.numBytesAllocatedStack", stack, "Bytes") 110 | _ = s.sender.SendComponentMetric("memoryStats.lastGCPauseTimeNS", gc, "ns") 111 | _ = s.sender.SendComponentMetric("numGoRoutines", goroutines, "Count") 112 | } 113 | -------------------------------------------------------------------------------- /conversion/log_message_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | 7 | "github.com/cloudfoundry/sonde-go/events" 8 | "google.golang.org/protobuf/proto" 9 | 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | var _ = Describe("LogMessage", func() { 15 | Context("given a v2 envelope", func() { 16 | It("converts messages to a v1 envelope", func() { 17 | envelope := &loggregator_v2.Envelope{ 18 | Timestamp: 99, 19 | SourceId: "uuid", 20 | InstanceId: "test-source-instance", 21 | DeprecatedTags: map[string]*loggregator_v2.Value{ 22 | "source_type": {Data: &loggregator_v2.Value_Text{Text: "test-source-type"}}, 23 | }, 24 | Message: &loggregator_v2.Envelope_Log{ 25 | Log: &loggregator_v2.Log{ 26 | Payload: []byte("Hello World"), 27 | Type: loggregator_v2.Log_OUT, 28 | }, 29 | }, 30 | } 31 | 32 | envelopes := conversion.ToV1(envelope) 33 | Expect(len(envelopes)).To(Equal(1)) 34 | oldEnvelope := envelopes[0] 35 | 36 | Expect(oldEnvelope.GetEventType()).To(Equal(events.Envelope_LogMessage)) 37 | Expect(proto.Equal(oldEnvelope.LogMessage, &events.LogMessage{ 38 | Message: []byte("Hello World"), 39 | MessageType: events.LogMessage_OUT.Enum(), 40 | Timestamp: proto.Int64(99), 41 | AppId: proto.String("uuid"), 42 | SourceType: proto.String("test-source-type"), 43 | SourceInstance: proto.String("test-source-instance"), 44 | })).To(BeTrue()) 45 | }) 46 | }) 47 | 48 | Context("given a v1 envelop", func() { 49 | It("converts messages to v2 envelopes", func() { 50 | v1Envelope := &events.Envelope{ 51 | Origin: proto.String("some-origin"), 52 | EventType: events.Envelope_LogMessage.Enum(), 53 | Deployment: proto.String("some-deployment"), 54 | Job: proto.String("some-job"), 55 | Index: proto.String("some-index"), 56 | Ip: proto.String("some-ip"), 57 | LogMessage: &events.LogMessage{ 58 | Message: []byte("Hello World"), 59 | MessageType: events.LogMessage_OUT.Enum(), 60 | Timestamp: proto.Int64(99), 61 | AppId: proto.String("uuid"), 62 | SourceType: proto.String("test-source-type"), 63 | SourceInstance: proto.String("test-source-instance"), 64 | }, 65 | } 66 | 67 | expectedV2Envelope := &loggregator_v2.Envelope{ 68 | SourceId: "uuid", 69 | InstanceId: "test-source-instance", 70 | DeprecatedTags: map[string]*loggregator_v2.Value{ 71 | "__v1_type": {Data: &loggregator_v2.Value_Text{Text: "LogMessage"}}, 72 | "source_type": {Data: &loggregator_v2.Value_Text{Text: "test-source-type"}}, 73 | "origin": {Data: &loggregator_v2.Value_Text{Text: "some-origin"}}, 74 | "deployment": {Data: &loggregator_v2.Value_Text{Text: "some-deployment"}}, 75 | "job": {Data: &loggregator_v2.Value_Text{Text: "some-job"}}, 76 | "index": {Data: &loggregator_v2.Value_Text{Text: "some-index"}}, 77 | "ip": {Data: &loggregator_v2.Value_Text{Text: "some-ip"}}, 78 | }, 79 | Message: &loggregator_v2.Envelope_Log{ 80 | Log: &loggregator_v2.Log{ 81 | Payload: []byte("Hello World"), 82 | Type: loggregator_v2.Log_OUT, 83 | }, 84 | }, 85 | } 86 | 87 | v2Envelope := conversion.ToV2(v1Envelope, false) 88 | 89 | Expect(proto.Equal(v2Envelope, expectedV2Envelope)).To(BeTrue()) 90 | }) 91 | 92 | It("sets the source ID to deployment/job when App ID is missing", func() { 93 | v1Envelope := &events.Envelope{ 94 | Deployment: proto.String("some-deployment"), 95 | Job: proto.String("some-job"), 96 | EventType: events.Envelope_LogMessage.Enum(), 97 | LogMessage: &events.LogMessage{}, 98 | } 99 | 100 | expectedV2Envelope := &loggregator_v2.Envelope{ 101 | SourceId: "some-deployment/some-job", 102 | } 103 | 104 | converted := conversion.ToV2(v1Envelope, false) 105 | 106 | Expect(converted.SourceId).To(Equal(expectedV2Envelope.SourceId)) 107 | }) 108 | }) 109 | }) 110 | -------------------------------------------------------------------------------- /conversion/counter_event_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | "github.com/cloudfoundry/sonde-go/events" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | . "github.com/onsi/gomega/gstruct" 10 | "google.golang.org/protobuf/proto" 11 | ) 12 | 13 | var _ = Describe("CounterEvent", func() { 14 | Context("given a v2 envelope", func() { 15 | Context("with a total", func() { 16 | It("converts to a v1 envelope", func() { 17 | envelope := &loggregator_v2.Envelope{ 18 | Message: &loggregator_v2.Envelope_Counter{ 19 | Counter: &loggregator_v2.Counter{ 20 | Name: "name", 21 | Total: 99, 22 | }, 23 | }, 24 | } 25 | 26 | envelopes := conversion.ToV1(envelope) 27 | Expect(len(envelopes)).To(Equal(1)) 28 | Expect(envelopes[0].GetEventType()).To(Equal(events.Envelope_CounterEvent)) 29 | Expect(proto.Equal(envelopes[0].GetCounterEvent(), &events.CounterEvent{ 30 | Name: proto.String("name"), 31 | Total: proto.Uint64(99), 32 | Delta: proto.Uint64(0), 33 | })).To(BeTrue()) 34 | }) 35 | }) 36 | 37 | Context("with a delta", func() { 38 | It("converts to a v1 envelope", func() { 39 | envelope := &loggregator_v2.Envelope{ 40 | Message: &loggregator_v2.Envelope_Counter{ 41 | Counter: &loggregator_v2.Counter{ 42 | Name: "name", 43 | Delta: 99, 44 | }, 45 | }, 46 | } 47 | 48 | envelopes := conversion.ToV1(envelope) 49 | Expect(len(envelopes)).To(Equal(1)) 50 | Expect(envelopes[0].GetEventType()).To(Equal(events.Envelope_CounterEvent)) 51 | Expect(proto.Equal(envelopes[0].GetCounterEvent(), &events.CounterEvent{ 52 | Name: proto.String("name"), 53 | Total: proto.Uint64(0), 54 | Delta: proto.Uint64(99), 55 | })).To(BeTrue()) 56 | }) 57 | }) 58 | }) 59 | 60 | Context("given a v1 envelope", func() { 61 | var ( 62 | v1Envelope *events.Envelope 63 | expectedMessage *loggregator_v2.Envelope_Counter 64 | ) 65 | 66 | BeforeEach(func() { 67 | v1Envelope = &events.Envelope{ 68 | Origin: proto.String("an-origin"), 69 | Deployment: proto.String("a-deployment"), 70 | Job: proto.String("a-job"), 71 | Index: proto.String("an-index"), 72 | Ip: proto.String("an-ip"), 73 | Timestamp: proto.Int64(1234), 74 | EventType: events.Envelope_CounterEvent.Enum(), 75 | CounterEvent: &events.CounterEvent{ 76 | Name: proto.String("name"), 77 | Total: proto.Uint64(99), 78 | Delta: proto.Uint64(2), 79 | }, 80 | Tags: map[string]string{ 81 | "custom_tag": "custom-value", 82 | "source_id": "source-id", 83 | "instance_id": "instance-id", 84 | }, 85 | } 86 | expectedMessage = &loggregator_v2.Envelope_Counter{ 87 | Counter: &loggregator_v2.Counter{ 88 | Name: "name", 89 | Delta: 2, 90 | Total: 99, 91 | }, 92 | } 93 | }) 94 | 95 | Context("using deprecated tags", func() { 96 | It("converts to a v2 envelope with DeprecatedTags", func() { 97 | Expect(*conversion.ToV2(v1Envelope, false)).To(MatchFields(IgnoreExtras, Fields{ 98 | "Timestamp": Equal(int64(1234)), 99 | "SourceId": Equal("source-id"), 100 | "InstanceId": Equal("instance-id"), 101 | "Message": Equal(expectedMessage), 102 | "DeprecatedTags": Equal(map[string]*loggregator_v2.Value{ 103 | "origin": {Data: &loggregator_v2.Value_Text{Text: "an-origin"}}, 104 | "deployment": {Data: &loggregator_v2.Value_Text{Text: "a-deployment"}}, 105 | "job": {Data: &loggregator_v2.Value_Text{Text: "a-job"}}, 106 | "index": {Data: &loggregator_v2.Value_Text{Text: "an-index"}}, 107 | "ip": {Data: &loggregator_v2.Value_Text{Text: "an-ip"}}, 108 | "__v1_type": {Data: &loggregator_v2.Value_Text{Text: "CounterEvent"}}, 109 | "custom_tag": {Data: &loggregator_v2.Value_Text{Text: "custom-value"}}, 110 | }), 111 | "Tags": BeNil(), 112 | })) 113 | }) 114 | }) 115 | 116 | Context("using preferred tags", func() { 117 | It("converts to a v2 envelope with Tags", func() { 118 | Expect(*conversion.ToV2(v1Envelope, true)).To(MatchFields(IgnoreExtras, Fields{ 119 | "Timestamp": Equal(int64(1234)), 120 | "SourceId": Equal("source-id"), 121 | "InstanceId": Equal("instance-id"), 122 | "Message": Equal(expectedMessage), 123 | "DeprecatedTags": BeNil(), 124 | "Tags": Equal(map[string]string{ 125 | "origin": "an-origin", 126 | "deployment": "a-deployment", 127 | "job": "a-job", 128 | "index": "an-index", 129 | "ip": "an-ip", 130 | "__v1_type": "CounterEvent", 131 | "custom_tag": "custom-value", 132 | }), 133 | })) 134 | }) 135 | }) 136 | }) 137 | }) 138 | -------------------------------------------------------------------------------- /conversion/value_metric_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | 7 | "github.com/cloudfoundry/sonde-go/events" 8 | "google.golang.org/protobuf/proto" 9 | 10 | . "github.com/onsi/ginkgo/v2" 11 | . "github.com/onsi/gomega" 12 | . "github.com/onsi/gomega/gstruct" 13 | ) 14 | 15 | var _ = Describe("ValueMetric", func() { 16 | Context("given a v2 envelope", func() { 17 | It("converts to a v1 envelope", func() { 18 | envelope := &loggregator_v2.Envelope{ 19 | Message: &loggregator_v2.Envelope_Gauge{ 20 | Gauge: &loggregator_v2.Gauge{ 21 | Metrics: map[string]*loggregator_v2.GaugeValue{ 22 | "name": { 23 | Unit: "meters", 24 | Value: 123, 25 | }, 26 | }, 27 | }, 28 | }, 29 | } 30 | 31 | envelopes := conversion.ToV1(envelope) 32 | Expect(len(envelopes)).To(Equal(1)) 33 | 34 | converted := envelopes[0] 35 | Expect(converted.GetEventType()).To(Equal(events.Envelope_ValueMetric)) 36 | Expect(proto.Equal(converted.ValueMetric, &events.ValueMetric{ 37 | Name: proto.String("name"), 38 | Unit: proto.String("meters"), 39 | Value: proto.Float64(123), 40 | })).To(BeTrue()) 41 | }) 42 | 43 | It("converts multiple Gauge values", func() { 44 | envelope := &loggregator_v2.Envelope{ 45 | Message: &loggregator_v2.Envelope_Gauge{ 46 | Gauge: &loggregator_v2.Gauge{ 47 | Metrics: map[string]*loggregator_v2.GaugeValue{ 48 | "name": { 49 | Unit: "meters", 50 | Value: 123, 51 | }, 52 | "other-name": { 53 | Unit: "feet", 54 | Value: 321, 55 | }, 56 | }, 57 | }, 58 | }, 59 | Tags: map[string]string{ 60 | "origin": "my-origin", 61 | "deployment": "my-deployment", 62 | "job": "my-job", 63 | "index": "my-index", 64 | "ip": "my-ip", 65 | }, 66 | } 67 | 68 | envelopes := conversion.ToV1(envelope) 69 | Expect(len(envelopes)).To(Equal(2)) 70 | 71 | for _, e := range envelopes { 72 | Expect(e.GetOrigin()).To(Equal("my-origin")) 73 | Expect(e.GetDeployment()).To(Equal("my-deployment")) 74 | Expect(e.GetJob()).To(Equal("my-job")) 75 | Expect(e.GetIndex()).To(Equal("my-index")) 76 | Expect(e.GetIp()).To(Equal("my-ip")) 77 | } 78 | 79 | Expect(envelopes[0].GetValueMetric().GetName()).To(Or( 80 | Equal("name"), 81 | Equal("other-name"), 82 | )) 83 | 84 | Expect(envelopes[1].GetValueMetric().GetName()).To(Or( 85 | Equal("name"), 86 | Equal("other-name"), 87 | )) 88 | 89 | Expect(envelopes[0].GetValueMetric().GetName()).ToNot(Equal( 90 | envelopes[1].GetValueMetric().GetName())) 91 | }) 92 | 93 | It("is resilient to partial envelopes", func() { 94 | envelope := &loggregator_v2.Envelope{ 95 | Message: &loggregator_v2.Envelope_Gauge{ 96 | Gauge: &loggregator_v2.Gauge{ 97 | Metrics: map[string]*loggregator_v2.GaugeValue{ 98 | "name": nil, 99 | }, 100 | }, 101 | }, 102 | } 103 | Expect(conversion.ToV1(envelope)).To(BeNil()) 104 | }) 105 | 106 | It("converts gauge with 0 value to value metric", func() { 107 | envelope := &loggregator_v2.Envelope{ 108 | Message: &loggregator_v2.Envelope_Gauge{ 109 | Gauge: &loggregator_v2.Gauge{ 110 | Metrics: map[string]*loggregator_v2.GaugeValue{ 111 | "name": { 112 | Unit: "", 113 | Value: 0.0, 114 | }, 115 | }, 116 | }, 117 | }, 118 | } 119 | 120 | envelopes := conversion.ToV1(envelope) 121 | Expect(len(envelopes)).To(Equal(1)) 122 | 123 | converted := envelopes[0] 124 | Expect(converted.GetEventType()).To(Equal(events.Envelope_ValueMetric)) 125 | Expect(proto.Equal(converted.ValueMetric, &events.ValueMetric{ 126 | Name: proto.String("name"), 127 | Unit: proto.String(""), 128 | Value: proto.Float64(0.0), 129 | })).To(BeTrue()) 130 | }) 131 | }) 132 | 133 | Context("given a v1 envelope", func() { 134 | It("converts to a v2 envelope", func() { 135 | v1Envelope := &events.Envelope{ 136 | EventType: events.Envelope_ValueMetric.Enum(), 137 | ValueMetric: &events.ValueMetric{ 138 | Name: proto.String("name"), 139 | Unit: proto.String("meters"), 140 | Value: proto.Float64(123), 141 | }, 142 | } 143 | expectedV2Envelope := &loggregator_v2.Envelope{ 144 | Message: &loggregator_v2.Envelope_Gauge{ 145 | Gauge: &loggregator_v2.Gauge{ 146 | Metrics: map[string]*loggregator_v2.GaugeValue{ 147 | "name": { 148 | Unit: "meters", 149 | Value: 123, 150 | }, 151 | }, 152 | }, 153 | }, 154 | } 155 | Expect(*conversion.ToV2(v1Envelope, false)).To(MatchFields(IgnoreExtras, Fields{ 156 | "Message": Equal(expectedV2Envelope.Message), 157 | })) 158 | }) 159 | }) 160 | }) 161 | -------------------------------------------------------------------------------- /rfc5424/marshal.go: -------------------------------------------------------------------------------- 1 | package rfc5424 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "unicode/utf8" 7 | ) 8 | 9 | // allowLongSdNames is true to allow names longer than the RFC-specified limit 10 | // of 32-characters. (When true, this violates RFC-5424). 11 | const allowLongSdNames = true 12 | 13 | // RFC5424TimeOffsetNum is the timestamp defined by RFC-5424 with the 14 | // NUMOFFSET instead of Z. 15 | const RFC5424TimeOffsetNum = "2006-01-02T15:04:05.999999-07:00" 16 | 17 | // RFC5424TimeOffsetUTC is the timestamp defined by RFC-5424 with the offset 18 | // set to 0 for UTC. 19 | const RFC5424TimeOffsetUTC = "2006-01-02T15:04:05.999999Z" 20 | 21 | // ErrInvalidValue is returned when a log message cannot be emitted because one 22 | // of the values is invalid. 23 | type ErrInvalidValue struct { 24 | Property string 25 | Value interface{} 26 | } 27 | 28 | func (e ErrInvalidValue) Error() string { 29 | return fmt.Sprintf("Message cannot be serialized because %s is invalid: %v", 30 | e.Property, e.Value) 31 | } 32 | 33 | // invalidValue returns an invalid value error with the given property 34 | func invalidValue(property string, value interface{}) error { 35 | return ErrInvalidValue{Property: property, Value: value} 36 | } 37 | 38 | func nilify(x string) string { 39 | if x == "" { 40 | return "-" 41 | } 42 | return x 43 | } 44 | 45 | func escapeSDParam(s string) string { 46 | escapeCount := 0 47 | for i := 0; i < len(s); i++ { 48 | switch s[i] { 49 | case '\\', '"', ']': 50 | escapeCount++ 51 | } 52 | } 53 | if escapeCount == 0 { 54 | return s 55 | } 56 | 57 | t := make([]byte, len(s)+escapeCount) 58 | j := 0 59 | for i := 0; i < len(s); i++ { 60 | switch c := s[i]; c { 61 | case '\\', '"', ']': 62 | t[j] = '\\' 63 | t[j+1] = c 64 | j += 2 65 | default: 66 | t[j] = s[i] 67 | j++ 68 | } 69 | } 70 | return string(t) 71 | } 72 | 73 | func isPrintableUsASCII(s string) bool { 74 | for _, ch := range s { 75 | if ch < 33 || ch > 126 { 76 | return false 77 | } 78 | } 79 | return true 80 | } 81 | 82 | func isValidSdName(s string) bool { 83 | if !allowLongSdNames && len(s) > 32 { 84 | return false 85 | } 86 | for _, ch := range s { 87 | if ch < 33 || ch > 126 { 88 | return false 89 | } 90 | if ch == '=' || ch == ']' || ch == '"' { 91 | return false 92 | } 93 | } 94 | return true 95 | } 96 | 97 | func (m Message) assertValid() error { 98 | // HOSTNAME = NILVALUE / 1*255PRINTUSASCII 99 | if !isPrintableUsASCII(m.Hostname) { 100 | return invalidValue("Hostname", m.Hostname) 101 | } 102 | 103 | // APP-NAME = NILVALUE / 1*48PRINTUSASCII 104 | if !isPrintableUsASCII(m.AppName) { 105 | return invalidValue("AppName", m.AppName) 106 | } 107 | 108 | // PROCID = NILVALUE / 1*128PRINTUSASCII 109 | if !isPrintableUsASCII(m.ProcessID) { 110 | return invalidValue("ProcessID", m.ProcessID) 111 | } 112 | 113 | // MSGID = NILVALUE / 1*32PRINTUSASCII 114 | if !isPrintableUsASCII(m.MessageID) { 115 | return invalidValue("MessageID", m.MessageID) 116 | } 117 | 118 | for _, sdElement := range m.StructuredData { 119 | if !isValidSdName(sdElement.ID) { 120 | return invalidValue("StructuredData/ID", sdElement.ID) 121 | } 122 | for _, sdParam := range sdElement.Parameters { 123 | if !isValidSdName(sdParam.Name) { 124 | return invalidValue("StructuredData/Name", sdParam.Name) 125 | } 126 | if !utf8.ValidString(sdParam.Value) { 127 | return invalidValue("StructuredData/Value", sdParam.Value) 128 | } 129 | } 130 | } 131 | return nil 132 | } 133 | 134 | // MarshalBinary marshals the message to a byte slice, or returns an error 135 | func (m Message) MarshalBinary() ([]byte, error) { 136 | if len(m.Hostname) > 255 { 137 | m.Hostname = m.Hostname[:255] 138 | } 139 | 140 | if len(m.AppName) > 48 { 141 | m.AppName = m.AppName[:48] 142 | } 143 | 144 | if len(m.ProcessID) > 128 { 145 | m.ProcessID = m.ProcessID[:128] 146 | } 147 | 148 | if len(m.MessageID) > 32 { 149 | m.MessageID = m.MessageID[:32] 150 | } 151 | 152 | if err := m.assertValid(); err != nil { 153 | return nil, err 154 | } 155 | 156 | format := RFC5424TimeOffsetNum 157 | if m.UseUTC { 158 | format = RFC5424TimeOffsetUTC 159 | } 160 | b := bytes.NewBuffer(nil) 161 | fmt.Fprintf(b, "<%d>1 %s %s %s %s %s ", 162 | m.Priority, 163 | m.Timestamp.Format(format), 164 | nilify(m.Hostname), 165 | nilify(m.AppName), 166 | nilify(m.ProcessID), 167 | nilify(m.MessageID)) 168 | 169 | if len(m.StructuredData) == 0 { 170 | fmt.Fprint(b, "-") 171 | } 172 | for _, sdElement := range m.StructuredData { 173 | fmt.Fprintf(b, "[%s", sdElement.ID) 174 | for _, sdParam := range sdElement.Parameters { 175 | fmt.Fprintf(b, " %s=\"%s\"", sdParam.Name, 176 | escapeSDParam(sdParam.Value)) 177 | } 178 | fmt.Fprintf(b, "]") 179 | } 180 | 181 | if len(m.Message) > 0 { 182 | fmt.Fprint(b, " ") 183 | b.Write(m.Message) 184 | } 185 | return b.Bytes(), nil 186 | } 187 | -------------------------------------------------------------------------------- /envelope_stream_connector.go: -------------------------------------------------------------------------------- 1 | package loggregator 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "io" 7 | "log" 8 | "time" 9 | 10 | gendiodes "code.cloudfoundry.org/go-diodes" 11 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/credentials" 14 | ) 15 | 16 | // EnvelopeStreamConnector provides a way to connect to loggregator and 17 | // consume a stream of envelopes. It handles reconnecting and provides 18 | // a stream for the lifecycle of the given context. It should be created with 19 | // the NewEnvelopeStreamConnector constructor. 20 | type EnvelopeStreamConnector struct { 21 | addr string 22 | tlsConf *tls.Config 23 | 24 | // Buffering 25 | bufferSize int 26 | alerter func(int) 27 | 28 | log Logger 29 | dialOptions []grpc.DialOption 30 | } 31 | 32 | // NewEnvelopeStreamConnector creates a new EnvelopeStreamConnector. Its TLS 33 | // configuration must share a CA with the loggregator server. 34 | func NewEnvelopeStreamConnector( 35 | addr string, 36 | t *tls.Config, 37 | opts ...EnvelopeStreamOption, 38 | ) *EnvelopeStreamConnector { 39 | 40 | c := &EnvelopeStreamConnector{ 41 | addr: addr, 42 | tlsConf: t, 43 | 44 | log: log.New(io.Discard, "", 0), 45 | } 46 | 47 | for _, o := range opts { 48 | o(c) 49 | } 50 | 51 | return c 52 | } 53 | 54 | // EnvelopeStreamOption configures a EnvelopeStreamConnector. 55 | type EnvelopeStreamOption func(*EnvelopeStreamConnector) 56 | 57 | // WithEnvelopeStreamLogger allows for the configuration of a logger. 58 | // By default, the logger is disabled. 59 | func WithEnvelopeStreamLogger(l Logger) EnvelopeStreamOption { 60 | return func(c *EnvelopeStreamConnector) { 61 | c.log = l 62 | } 63 | } 64 | 65 | // WithEnvelopeStreamConnectorDialOptions allows for configuration of 66 | // grpc dial options. 67 | func WithEnvelopeStreamConnectorDialOptions(opts ...grpc.DialOption) EnvelopeStreamOption { 68 | return func(c *EnvelopeStreamConnector) { 69 | c.dialOptions = opts 70 | } 71 | } 72 | 73 | // WithEnvelopeStreamBuffer enables the EnvelopeStream to read more quickly 74 | // from the stream. It puts each envelope in a buffer that overwrites data if 75 | // it is not being drained quick enough. If the buffer drops data, the 76 | // 'alerter' function will be invoked with the number of envelopes dropped. 77 | func WithEnvelopeStreamBuffer(size int, alerter func(missed int)) EnvelopeStreamOption { 78 | return func(c *EnvelopeStreamConnector) { 79 | c.bufferSize = size 80 | c.alerter = alerter 81 | } 82 | } 83 | 84 | // EnvelopeStream returns batches of envelopes. It blocks until its context 85 | // is done or a batch of envelopes is available. 86 | type EnvelopeStream func() []*loggregator_v2.Envelope 87 | 88 | // Stream returns a new EnvelopeStream for the given context and request. The 89 | // lifecycle of the EnvelopeStream is managed by the given context. If the 90 | // underlying gRPC stream dies, it attempts to reconnect until the context 91 | // is done. 92 | func (c *EnvelopeStreamConnector) Stream(ctx context.Context, req *loggregator_v2.EgressBatchRequest) EnvelopeStream { 93 | s := newStream(ctx, c.addr, req, c.tlsConf, c.dialOptions, c.log) 94 | if c.alerter != nil || c.bufferSize > 0 { 95 | d := NewOneToOneEnvelopeBatch( 96 | c.bufferSize, 97 | gendiodes.AlertFunc(c.alerter), 98 | gendiodes.WithPollingContext(ctx), 99 | ) 100 | 101 | go func() { 102 | for { 103 | select { 104 | case <-ctx.Done(): 105 | return 106 | default: 107 | } 108 | 109 | d.Set(s.recv()) 110 | } 111 | }() 112 | return d.Next 113 | } 114 | 115 | return s.recv 116 | } 117 | 118 | type stream struct { 119 | log Logger 120 | ctx context.Context 121 | req *loggregator_v2.EgressBatchRequest 122 | client loggregator_v2.EgressClient 123 | rx loggregator_v2.Egress_BatchedReceiverClient 124 | } 125 | 126 | func newStream( 127 | ctx context.Context, 128 | addr string, 129 | req *loggregator_v2.EgressBatchRequest, 130 | c *tls.Config, 131 | opts []grpc.DialOption, 132 | log Logger, 133 | ) *stream { 134 | opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(c))) 135 | conn, err := grpc.NewClient( 136 | addr, 137 | opts..., 138 | ) 139 | if err != nil { 140 | // This error occurs on invalid configuration. And more notably, 141 | // it does NOT occur if the server is not up. 142 | log.Panicf("invalid gRPC dial configuration: %s", err) 143 | } 144 | 145 | // Protect against a go-routine leak. gRPC will keep a go-routine active 146 | // within the connection to keep the connectin alive. We have to close 147 | // this or the go-routine leaks. This is untested. We had trouble exposing 148 | // the underlying connectin was still active. 149 | go func() { 150 | <-ctx.Done() 151 | conn.Close() 152 | }() 153 | 154 | client := loggregator_v2.NewEgressClient(conn) 155 | 156 | return &stream{ 157 | ctx: ctx, 158 | req: req, 159 | client: client, 160 | log: log, 161 | } 162 | } 163 | 164 | func (s *stream) recv() []*loggregator_v2.Envelope { 165 | for { 166 | ok := s.connect(s.ctx) 167 | if !ok { 168 | return nil 169 | } 170 | batch, err := s.rx.Recv() 171 | if err != nil { 172 | s.rx = nil 173 | continue 174 | } 175 | 176 | return batch.Batch 177 | } 178 | } 179 | 180 | func (s *stream) connect(ctx context.Context) bool { 181 | for { 182 | select { 183 | case <-ctx.Done(): 184 | return false 185 | default: 186 | if s.rx != nil { 187 | return true 188 | } 189 | 190 | var err error 191 | s.rx, err = s.client.BatchedReceiver( 192 | ctx, 193 | s.req, 194 | ) 195 | 196 | if err != nil { 197 | s.log.Printf("Error connecting to Logs Provider: %s", err) 198 | time.Sleep(50 * time.Millisecond) 199 | continue 200 | } 201 | 202 | return true 203 | } 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /conversion/envelope_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "fmt" 5 | 6 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 7 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 8 | "github.com/cloudfoundry/sonde-go/events" 9 | . "github.com/onsi/ginkgo/v2" 10 | . "github.com/onsi/gomega" 11 | "google.golang.org/protobuf/proto" 12 | ) 13 | 14 | var _ = Describe("Envelope", func() { 15 | Context("given a v2 envelope", func() { 16 | It("sets v1 specific properties", func() { 17 | envelope := &loggregator_v2.Envelope{ 18 | Timestamp: 99, 19 | DeprecatedTags: map[string]*loggregator_v2.Value{ 20 | "origin": {Data: &loggregator_v2.Value_Text{Text: "origin"}}, 21 | "deployment": {Data: &loggregator_v2.Value_Text{Text: "deployment"}}, 22 | "job": {Data: &loggregator_v2.Value_Text{Text: "job"}}, 23 | "index": {Data: &loggregator_v2.Value_Text{Text: "index"}}, 24 | "ip": {Data: &loggregator_v2.Value_Text{Text: "ip"}}, 25 | "random_text": {Data: &loggregator_v2.Value_Text{Text: "random_text"}}, 26 | "random_int": {Data: &loggregator_v2.Value_Integer{Integer: 123}}, 27 | "random_decimal": {Data: &loggregator_v2.Value_Decimal{Decimal: 123}}, 28 | }, 29 | Message: &loggregator_v2.Envelope_Log{Log: &loggregator_v2.Log{}}, 30 | } 31 | 32 | envelopes := conversion.ToV1(envelope) 33 | Expect(len(envelopes)).To(Equal(1)) 34 | oldEnvelope := envelopes[0] 35 | Expect(oldEnvelope.GetOrigin()).To(Equal("origin")) 36 | Expect(oldEnvelope.GetEventType()).To(Equal(events.Envelope_LogMessage)) 37 | Expect(oldEnvelope.GetTimestamp()).To(Equal(int64(99))) 38 | Expect(oldEnvelope.GetDeployment()).To(Equal("deployment")) 39 | Expect(oldEnvelope.GetJob()).To(Equal("job")) 40 | Expect(oldEnvelope.GetIndex()).To(Equal("index")) 41 | Expect(oldEnvelope.GetIp()).To(Equal("ip")) 42 | Expect(oldEnvelope.Tags).To(HaveKeyWithValue("random_text", "random_text")) 43 | Expect(oldEnvelope.Tags).To(HaveKeyWithValue("random_int", "123")) 44 | Expect(oldEnvelope.Tags).To(HaveKeyWithValue("random_decimal", fmt.Sprintf("%f", 123.0))) 45 | }) 46 | 47 | It("rejects empty tags", func() { 48 | envelope := &loggregator_v2.Envelope{ 49 | DeprecatedTags: map[string]*loggregator_v2.Value{ 50 | "foo": {Data: &loggregator_v2.Value_Text{Text: "bar"}}, 51 | "baz": nil, 52 | }, 53 | Message: &loggregator_v2.Envelope_Log{Log: &loggregator_v2.Log{}}, 54 | } 55 | 56 | envelopes := conversion.ToV1(envelope) 57 | Expect(len(envelopes)).To(Equal(1)) 58 | oldEnvelope := envelopes[0] 59 | Expect(oldEnvelope.Tags).To(Equal(map[string]string{ 60 | "foo": "bar", 61 | })) 62 | }) 63 | 64 | It("reads non-text v2 tags", func() { 65 | envelope := &loggregator_v2.Envelope{ 66 | DeprecatedTags: map[string]*loggregator_v2.Value{ 67 | "foo": {Data: &loggregator_v2.Value_Integer{Integer: 99}}, 68 | }, 69 | Message: &loggregator_v2.Envelope_Log{Log: &loggregator_v2.Log{}}, 70 | } 71 | 72 | envelopes := conversion.ToV1(envelope) 73 | Expect(len(envelopes)).To(Equal(1)) 74 | Expect(envelopes[0].GetTags()).To(HaveKeyWithValue("foo", "99")) 75 | }) 76 | 77 | It("uses non-deprecated v2 tags", func() { 78 | envelope := &loggregator_v2.Envelope{ 79 | Tags: map[string]string{ 80 | "foo": "bar", 81 | }, 82 | Message: &loggregator_v2.Envelope_Log{Log: &loggregator_v2.Log{}}, 83 | } 84 | 85 | envelopes := conversion.ToV1(envelope) 86 | Expect(len(envelopes)).To(Equal(1)) 87 | Expect(envelopes[0].GetTags()).To(HaveKeyWithValue("foo", "bar")) 88 | }) 89 | }) 90 | 91 | Context("given a v1 envelope", func() { 92 | It("sets v2 specific properties", func() { 93 | v1Envelope := &events.Envelope{ 94 | Timestamp: proto.Int64(99), 95 | Origin: proto.String("origin-value"), 96 | Deployment: proto.String("some-deployment"), 97 | Job: proto.String("some-job"), 98 | Index: proto.String("some-index"), 99 | Ip: proto.String("some-ip"), 100 | Tags: map[string]string{ 101 | "random-tag": "random-value", 102 | }, 103 | } 104 | 105 | expectedV2Envelope := &loggregator_v2.Envelope{ 106 | Timestamp: 99, 107 | SourceId: "some-deployment/some-job", 108 | DeprecatedTags: map[string]*loggregator_v2.Value{ 109 | "random-tag": ValueText("random-value"), 110 | "origin": ValueText("origin-value"), 111 | "deployment": ValueText("some-deployment"), 112 | "job": ValueText("some-job"), 113 | "index": ValueText("some-index"), 114 | "ip": ValueText("some-ip"), 115 | }, 116 | } 117 | 118 | converted := conversion.ToV2(v1Envelope, false) 119 | 120 | Expect(converted.SourceId).To(Equal(expectedV2Envelope.SourceId)) 121 | Expect(converted.Timestamp).To(Equal(expectedV2Envelope.Timestamp)) 122 | Expect(converted.DeprecatedTags["random-tag"]).To(Equal(expectedV2Envelope.DeprecatedTags["random-tag"])) 123 | Expect(converted.DeprecatedTags["origin"]).To(Equal(expectedV2Envelope.DeprecatedTags["origin"])) 124 | Expect(converted.DeprecatedTags["deployment"]).To(Equal(expectedV2Envelope.DeprecatedTags["deployment"])) 125 | Expect(converted.DeprecatedTags["job"]).To(Equal(expectedV2Envelope.DeprecatedTags["job"])) 126 | Expect(converted.DeprecatedTags["index"]).To(Equal(expectedV2Envelope.DeprecatedTags["index"])) 127 | Expect(converted.DeprecatedTags["ip"]).To(Equal(expectedV2Envelope.DeprecatedTags["ip"])) 128 | }) 129 | 130 | It("sets non-deprecated tags", func() { 131 | v1 := &events.Envelope{ 132 | Timestamp: proto.Int64(99), 133 | Origin: proto.String("origin-value"), 134 | Deployment: proto.String("some-deployment"), 135 | Job: proto.String("some-job"), 136 | Index: proto.String("some-index"), 137 | Ip: proto.String("some-ip"), 138 | Tags: map[string]string{ 139 | "random-tag": "random-value", 140 | "origin": "origin-value", 141 | "deployment": "some-deployment", 142 | "job": "some-job", 143 | "index": "some-index", 144 | "ip": "some-ip", 145 | }, 146 | } 147 | expected := proto.Clone(v1).(*events.Envelope) 148 | 149 | converted := conversion.ToV2(v1, true) 150 | 151 | Expect(converted.Tags["random-tag"]).To(Equal(expected.Tags["random-tag"])) 152 | Expect(converted.Tags["origin"]).To(Equal(expected.Tags["origin"])) 153 | Expect(converted.Tags["deployment"]).To(Equal(expected.Tags["deployment"])) 154 | Expect(converted.Tags["job"]).To(Equal(expected.Tags["job"])) 155 | Expect(converted.Tags["index"]).To(Equal(expected.Tags["index"])) 156 | Expect(converted.Tags["ip"]).To(Equal(expected.Tags["ip"])) 157 | }) 158 | }) 159 | }) 160 | -------------------------------------------------------------------------------- /rpc/loggregator_v2/egress_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.3.0 4 | // - protoc v5.26.1 5 | // source: loggregator-api/v2/egress.proto 6 | 7 | package loggregator_v2 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file 17 | // is compatible with the grpc package it is being compiled against. 18 | // Requires gRPC-Go v1.32.0 or later. 19 | const _ = grpc.SupportPackageIsVersion7 20 | 21 | const ( 22 | Egress_Receiver_FullMethodName = "/loggregator.v2.Egress/Receiver" 23 | Egress_BatchedReceiver_FullMethodName = "/loggregator.v2.Egress/BatchedReceiver" 24 | ) 25 | 26 | // EgressClient is the client API for Egress service. 27 | // 28 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 29 | type EgressClient interface { 30 | Receiver(ctx context.Context, in *EgressRequest, opts ...grpc.CallOption) (Egress_ReceiverClient, error) 31 | BatchedReceiver(ctx context.Context, in *EgressBatchRequest, opts ...grpc.CallOption) (Egress_BatchedReceiverClient, error) 32 | } 33 | 34 | type egressClient struct { 35 | cc grpc.ClientConnInterface 36 | } 37 | 38 | func NewEgressClient(cc grpc.ClientConnInterface) EgressClient { 39 | return &egressClient{cc} 40 | } 41 | 42 | func (c *egressClient) Receiver(ctx context.Context, in *EgressRequest, opts ...grpc.CallOption) (Egress_ReceiverClient, error) { 43 | stream, err := c.cc.NewStream(ctx, &Egress_ServiceDesc.Streams[0], Egress_Receiver_FullMethodName, opts...) 44 | if err != nil { 45 | return nil, err 46 | } 47 | x := &egressReceiverClient{stream} 48 | if err := x.ClientStream.SendMsg(in); err != nil { 49 | return nil, err 50 | } 51 | if err := x.ClientStream.CloseSend(); err != nil { 52 | return nil, err 53 | } 54 | return x, nil 55 | } 56 | 57 | type Egress_ReceiverClient interface { 58 | Recv() (*Envelope, error) 59 | grpc.ClientStream 60 | } 61 | 62 | type egressReceiverClient struct { 63 | grpc.ClientStream 64 | } 65 | 66 | func (x *egressReceiverClient) Recv() (*Envelope, error) { 67 | m := new(Envelope) 68 | if err := x.ClientStream.RecvMsg(m); err != nil { 69 | return nil, err 70 | } 71 | return m, nil 72 | } 73 | 74 | func (c *egressClient) BatchedReceiver(ctx context.Context, in *EgressBatchRequest, opts ...grpc.CallOption) (Egress_BatchedReceiverClient, error) { 75 | stream, err := c.cc.NewStream(ctx, &Egress_ServiceDesc.Streams[1], Egress_BatchedReceiver_FullMethodName, opts...) 76 | if err != nil { 77 | return nil, err 78 | } 79 | x := &egressBatchedReceiverClient{stream} 80 | if err := x.ClientStream.SendMsg(in); err != nil { 81 | return nil, err 82 | } 83 | if err := x.ClientStream.CloseSend(); err != nil { 84 | return nil, err 85 | } 86 | return x, nil 87 | } 88 | 89 | type Egress_BatchedReceiverClient interface { 90 | Recv() (*EnvelopeBatch, error) 91 | grpc.ClientStream 92 | } 93 | 94 | type egressBatchedReceiverClient struct { 95 | grpc.ClientStream 96 | } 97 | 98 | func (x *egressBatchedReceiverClient) Recv() (*EnvelopeBatch, error) { 99 | m := new(EnvelopeBatch) 100 | if err := x.ClientStream.RecvMsg(m); err != nil { 101 | return nil, err 102 | } 103 | return m, nil 104 | } 105 | 106 | // EgressServer is the server API for Egress service. 107 | // All implementations must embed UnimplementedEgressServer 108 | // for forward compatibility 109 | type EgressServer interface { 110 | Receiver(*EgressRequest, Egress_ReceiverServer) error 111 | BatchedReceiver(*EgressBatchRequest, Egress_BatchedReceiverServer) error 112 | mustEmbedUnimplementedEgressServer() 113 | } 114 | 115 | // UnimplementedEgressServer must be embedded to have forward compatible implementations. 116 | type UnimplementedEgressServer struct { 117 | } 118 | 119 | func (UnimplementedEgressServer) Receiver(*EgressRequest, Egress_ReceiverServer) error { 120 | return status.Errorf(codes.Unimplemented, "method Receiver not implemented") 121 | } 122 | func (UnimplementedEgressServer) BatchedReceiver(*EgressBatchRequest, Egress_BatchedReceiverServer) error { 123 | return status.Errorf(codes.Unimplemented, "method BatchedReceiver not implemented") 124 | } 125 | func (UnimplementedEgressServer) mustEmbedUnimplementedEgressServer() {} 126 | 127 | // UnsafeEgressServer may be embedded to opt out of forward compatibility for this service. 128 | // Use of this interface is not recommended, as added methods to EgressServer will 129 | // result in compilation errors. 130 | type UnsafeEgressServer interface { 131 | mustEmbedUnimplementedEgressServer() 132 | } 133 | 134 | func RegisterEgressServer(s grpc.ServiceRegistrar, srv EgressServer) { 135 | s.RegisterService(&Egress_ServiceDesc, srv) 136 | } 137 | 138 | func _Egress_Receiver_Handler(srv interface{}, stream grpc.ServerStream) error { 139 | m := new(EgressRequest) 140 | if err := stream.RecvMsg(m); err != nil { 141 | return err 142 | } 143 | return srv.(EgressServer).Receiver(m, &egressReceiverServer{stream}) 144 | } 145 | 146 | type Egress_ReceiverServer interface { 147 | Send(*Envelope) error 148 | grpc.ServerStream 149 | } 150 | 151 | type egressReceiverServer struct { 152 | grpc.ServerStream 153 | } 154 | 155 | func (x *egressReceiverServer) Send(m *Envelope) error { 156 | return x.ServerStream.SendMsg(m) 157 | } 158 | 159 | func _Egress_BatchedReceiver_Handler(srv interface{}, stream grpc.ServerStream) error { 160 | m := new(EgressBatchRequest) 161 | if err := stream.RecvMsg(m); err != nil { 162 | return err 163 | } 164 | return srv.(EgressServer).BatchedReceiver(m, &egressBatchedReceiverServer{stream}) 165 | } 166 | 167 | type Egress_BatchedReceiverServer interface { 168 | Send(*EnvelopeBatch) error 169 | grpc.ServerStream 170 | } 171 | 172 | type egressBatchedReceiverServer struct { 173 | grpc.ServerStream 174 | } 175 | 176 | func (x *egressBatchedReceiverServer) Send(m *EnvelopeBatch) error { 177 | return x.ServerStream.SendMsg(m) 178 | } 179 | 180 | // Egress_ServiceDesc is the grpc.ServiceDesc for Egress service. 181 | // It's only intended for direct use with grpc.RegisterService, 182 | // and not to be introspected or modified (even as a copy) 183 | var Egress_ServiceDesc = grpc.ServiceDesc{ 184 | ServiceName: "loggregator.v2.Egress", 185 | HandlerType: (*EgressServer)(nil), 186 | Methods: []grpc.MethodDesc{}, 187 | Streams: []grpc.StreamDesc{ 188 | { 189 | StreamName: "Receiver", 190 | Handler: _Egress_Receiver_Handler, 191 | ServerStreams: true, 192 | }, 193 | { 194 | StreamName: "BatchedReceiver", 195 | Handler: _Egress_BatchedReceiver_Handler, 196 | ServerStreams: true, 197 | }, 198 | }, 199 | Metadata: "loggregator-api/v2/egress.proto", 200 | } 201 | -------------------------------------------------------------------------------- /conversion/instance_id_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | "github.com/cloudfoundry/sonde-go/events" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | var _ = Describe("Converting Instance IDs", func() { 13 | Describe("LogMessage", func() { 14 | It("reads the v1 source_instance field when converting to v2", func() { 15 | v1Envelope := &events.Envelope{ 16 | Origin: proto.String(""), 17 | EventType: events.Envelope_LogMessage.Enum(), 18 | LogMessage: &events.LogMessage{ 19 | Message: []byte(""), 20 | MessageType: events.LogMessage_OUT.Enum(), 21 | Timestamp: proto.Int64(0), 22 | SourceInstance: proto.String("test-source-instance"), 23 | }, 24 | } 25 | actualV2Envelope := conversion.ToV2(v1Envelope, false) 26 | Expect(actualV2Envelope.InstanceId).To(Equal("test-source-instance")) 27 | }) 28 | 29 | It("writes into the v1 source_instance field when converting to v1", func() { 30 | v2Envelope := &loggregator_v2.Envelope{ 31 | InstanceId: "test-source-instance", 32 | Message: &loggregator_v2.Envelope_Log{ 33 | Log: &loggregator_v2.Log{ 34 | Payload: []byte("Hello World"), 35 | Type: loggregator_v2.Log_OUT, 36 | }, 37 | }, 38 | } 39 | envelopes := conversion.ToV1(v2Envelope) 40 | Expect(len(envelopes)).To(Equal(1)) 41 | Expect(*envelopes[0].LogMessage.SourceInstance).To(Equal("test-source-instance")) 42 | }) 43 | }) 44 | 45 | Describe("HttpStartStop", func() { 46 | It("reads the v1 instance_index field when converting to v2", func() { 47 | v1Envelope := &events.Envelope{ 48 | Origin: proto.String(""), 49 | EventType: events.Envelope_HttpStartStop.Enum(), 50 | HttpStartStop: &events.HttpStartStop{ 51 | StartTimestamp: proto.Int64(0), 52 | StopTimestamp: proto.Int64(0), 53 | RequestId: &events.UUID{}, 54 | PeerType: events.PeerType_Client.Enum(), 55 | Method: events.Method_GET.Enum(), 56 | Uri: proto.String(""), 57 | RemoteAddress: proto.String(""), 58 | UserAgent: proto.String(""), 59 | StatusCode: proto.Int32(0), 60 | ContentLength: proto.Int64(0), 61 | InstanceIndex: proto.Int32(1234), 62 | }, 63 | } 64 | actualV2Envelope := conversion.ToV2(v1Envelope, false) 65 | Expect(actualV2Envelope.InstanceId).To(Equal("1234")) 66 | }) 67 | 68 | It("writes into the v1 instance_index field when converting to v1", func() { 69 | v2Envelope := &loggregator_v2.Envelope{ 70 | InstanceId: "1234", 71 | Message: &loggregator_v2.Envelope_Timer{ 72 | Timer: &loggregator_v2.Timer{}, 73 | }, 74 | } 75 | envelopes := conversion.ToV1(v2Envelope) 76 | Expect(len(envelopes)).To(Equal(1)) 77 | Expect(*envelopes[0].HttpStartStop.InstanceIndex).To(Equal(int32(1234))) 78 | }) 79 | 80 | It("writes 0 into the v1 instance_index field if instance_id is not an int", func() { 81 | v2Envelope := &loggregator_v2.Envelope{ 82 | InstanceId: "garbage", 83 | Message: &loggregator_v2.Envelope_Timer{ 84 | Timer: &loggregator_v2.Timer{}, 85 | }, 86 | } 87 | envelopes := conversion.ToV1(v2Envelope) 88 | Expect(len(envelopes)).To(Equal(1)) 89 | Expect(*envelopes[0].HttpStartStop.InstanceIndex).To(Equal(int32(0))) 90 | }) 91 | }) 92 | 93 | Describe("ContainerMetric", func() { 94 | It("reads the v1 instance_index field when converting to v2", func() { 95 | v1Envelope := &events.Envelope{ 96 | Origin: proto.String(""), 97 | EventType: events.Envelope_ContainerMetric.Enum(), 98 | ContainerMetric: &events.ContainerMetric{ 99 | ApplicationId: proto.String(""), 100 | InstanceIndex: proto.Int32(4321), 101 | CpuPercentage: proto.Float64(0), 102 | MemoryBytes: proto.Uint64(0), 103 | DiskBytes: proto.Uint64(0), 104 | }, 105 | } 106 | actualV2Envelope := conversion.ToV2(v1Envelope, false) 107 | Expect(actualV2Envelope.InstanceId).To(Equal("4321")) 108 | }) 109 | 110 | It("writes into the v1 instance_index field when converting to v1", func() { 111 | v2Envelope := &loggregator_v2.Envelope{ 112 | InstanceId: "4321", 113 | Message: &loggregator_v2.Envelope_Gauge{ 114 | Gauge: &loggregator_v2.Gauge{ 115 | Metrics: map[string]*loggregator_v2.GaugeValue{ 116 | "cpu": {Unit: "percent"}, 117 | "memory": {Unit: "percent"}, 118 | "disk": {Unit: "percent"}, 119 | "memory_quota": {Unit: "bytes"}, 120 | "disk_quota": {Unit: "bytes"}, 121 | }, 122 | }, 123 | }, 124 | } 125 | envelopes := conversion.ToV1(v2Envelope) 126 | Expect(len(envelopes)).To(Equal(1)) 127 | Expect(*envelopes[0].ContainerMetric.InstanceIndex).To(Equal(int32(4321))) 128 | }) 129 | 130 | It("writes 0 into the v1 instance_index field if instance_id is not an int", func() { 131 | v2Envelope := &loggregator_v2.Envelope{ 132 | InstanceId: "garbage", 133 | Message: &loggregator_v2.Envelope_Gauge{ 134 | Gauge: &loggregator_v2.Gauge{ 135 | Metrics: map[string]*loggregator_v2.GaugeValue{ 136 | "cpu": {Unit: "percent"}, 137 | "memory": {Unit: "percent"}, 138 | "disk": {Unit: "percent"}, 139 | "memory_quota": {Unit: "bytes"}, 140 | "disk_quota": {Unit: "bytes"}, 141 | }, 142 | }, 143 | }, 144 | } 145 | envelopes := conversion.ToV1(v2Envelope) 146 | Expect(len(envelopes)).To(Equal(1)) 147 | Expect(*envelopes[0].ContainerMetric.InstanceIndex).To(Equal(int32(0))) 148 | }) 149 | }) 150 | 151 | Describe("CounterEvent and ValueMetric", func() { 152 | DescribeTable("reads the v1 instance_id tag when converting to v2", func(v1Envelope *events.Envelope) { 153 | actualV2Envelope := conversion.ToV2(v1Envelope, false) 154 | Expect(actualV2Envelope.InstanceId).To(Equal("test-source-instance")) 155 | }, 156 | Entry("CounterEvent", &events.Envelope{ 157 | Origin: proto.String(""), 158 | EventType: events.Envelope_CounterEvent.Enum(), 159 | CounterEvent: &events.CounterEvent{ 160 | Name: proto.String(""), 161 | Delta: proto.Uint64(0), 162 | }, 163 | Tags: map[string]string{ 164 | "instance_id": "test-source-instance", 165 | }, 166 | }), 167 | Entry("ValueMetric", &events.Envelope{ 168 | Origin: proto.String(""), 169 | EventType: events.Envelope_ValueMetric.Enum(), 170 | ValueMetric: &events.ValueMetric{ 171 | Name: proto.String(""), 172 | Value: proto.Float64(0), 173 | Unit: proto.String(""), 174 | }, 175 | Tags: map[string]string{ 176 | "instance_id": "test-source-instance", 177 | }, 178 | }), 179 | ) 180 | 181 | DescribeTable("writes into the v1 instance_id tag when converting to v1", func(v2Envelope *loggregator_v2.Envelope) { 182 | envelopes := conversion.ToV1(v2Envelope) 183 | Expect(len(envelopes)).To(Equal(1)) 184 | Expect(envelopes[0].Tags["instance_id"]).To(Equal("test-source-instance")) 185 | }, 186 | Entry("CounterEvent", &loggregator_v2.Envelope{ 187 | InstanceId: "test-source-instance", 188 | Message: &loggregator_v2.Envelope_Counter{ 189 | Counter: &loggregator_v2.Counter{}, 190 | }, 191 | }), 192 | Entry("ValueMetric", &loggregator_v2.Envelope{ 193 | InstanceId: "test-source-instance", 194 | Message: &loggregator_v2.Envelope_Gauge{ 195 | Gauge: &loggregator_v2.Gauge{ 196 | Metrics: map[string]*loggregator_v2.GaugeValue{ 197 | "some-metric": { 198 | Unit: "test", 199 | Value: 123.4, 200 | }, 201 | }, 202 | }, 203 | }, 204 | }), 205 | ) 206 | }) 207 | }) 208 | -------------------------------------------------------------------------------- /conversion/tov2.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | 9 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 10 | "github.com/cloudfoundry/sonde-go/events" 11 | ) 12 | 13 | // ToV2 converts v1 envelopes up to v2 envelopes. e may be mutated during the 14 | // conversion and share pointers with the resulting v2 envelope for efficiency 15 | // in creating the v2 envelope. As a result the envelope you pass in should no 16 | // longer be used. 17 | func ToV2(e *events.Envelope, usePreferredTags bool) *loggregator_v2.Envelope { 18 | v2e := &loggregator_v2.Envelope{ 19 | Timestamp: e.GetTimestamp(), 20 | } 21 | 22 | initTags(e, v2e, usePreferredTags) 23 | 24 | setV2Tag(v2e, "origin", e.GetOrigin(), usePreferredTags) 25 | setV2Tag(v2e, "deployment", e.GetDeployment(), usePreferredTags) 26 | setV2Tag(v2e, "job", e.GetJob(), usePreferredTags) 27 | setV2Tag(v2e, "index", e.GetIndex(), usePreferredTags) 28 | setV2Tag(v2e, "ip", e.GetIp(), usePreferredTags) 29 | setV2Tag(v2e, "__v1_type", e.GetEventType().String(), usePreferredTags) 30 | 31 | sourceId, ok := e.GetTags()["source_id"] 32 | v2e.SourceId = sourceId 33 | if !ok { 34 | v2e.SourceId = e.GetDeployment() + "/" + e.GetJob() 35 | } 36 | unsetV2Tag(v2e, "source_id") 37 | 38 | switch e.GetEventType() { 39 | case events.Envelope_LogMessage: 40 | convertLogMessage(v2e, e, usePreferredTags) 41 | case events.Envelope_HttpStartStop: 42 | convertHTTPStartStop(v2e, e, usePreferredTags) 43 | case events.Envelope_ValueMetric: 44 | convertValueMetric(v2e, e) 45 | case events.Envelope_CounterEvent: 46 | convertCounterEvent(v2e, e) 47 | case events.Envelope_Error: 48 | convertError(v2e, e, usePreferredTags) 49 | case events.Envelope_ContainerMetric: 50 | convertContainerMetric(v2e, e) 51 | } 52 | 53 | return v2e 54 | } 55 | 56 | // TODO: Do we still need to do an interface? 57 | func setV2Tag(e *loggregator_v2.Envelope, key string, value interface{}, usePreferredTags bool) { 58 | if usePreferredTags { 59 | if s, ok := value.(string); ok { 60 | e.GetTags()[key] = s 61 | return 62 | } 63 | 64 | e.GetTags()[key] = fmt.Sprintf("%v", value) 65 | return 66 | } 67 | 68 | e.GetDeprecatedTags()[key] = valueText(fmt.Sprintf("%v", value)) 69 | } 70 | 71 | func unsetV2Tag(e *loggregator_v2.Envelope, key string) { 72 | delete(e.GetDeprecatedTags(), key) 73 | delete(e.GetTags(), key) 74 | } 75 | 76 | func initTags(v1e *events.Envelope, v2e *loggregator_v2.Envelope, usePreferredTags bool) { 77 | if usePreferredTags { 78 | v2e.Tags = make(map[string]string) 79 | for k, v := range v1e.Tags { 80 | v2e.Tags[k] = v 81 | } 82 | 83 | return 84 | } 85 | 86 | v2e.DeprecatedTags = make(map[string]*loggregator_v2.Value) 87 | 88 | for k, v := range v1e.GetTags() { 89 | setV2Tag(v2e, k, v, usePreferredTags) 90 | } 91 | } 92 | 93 | func convertError(v2e *loggregator_v2.Envelope, v1e *events.Envelope, usePreferredTags bool) { 94 | t := v1e.GetError() 95 | setV2Tag(v2e, "source", t.GetSource(), usePreferredTags) 96 | setV2Tag(v2e, "code", t.GetCode(), usePreferredTags) 97 | 98 | v2e.Message = &loggregator_v2.Envelope_Log{ 99 | Log: &loggregator_v2.Log{ 100 | Payload: []byte(t.GetMessage()), 101 | Type: loggregator_v2.Log_OUT, 102 | }, 103 | } 104 | } 105 | 106 | func convertAppUUID(appID *events.UUID, sourceID string) string { 107 | if appID.GetLow() == 0 && appID.GetHigh() == 0 { 108 | return sourceID 109 | } 110 | return uuidToString(appID) 111 | } 112 | 113 | func convertAppID(appID, sourceID string) string { 114 | if appID == "" { 115 | return sourceID 116 | } 117 | return appID 118 | } 119 | 120 | func convertHTTPStartStop(v2e *loggregator_v2.Envelope, v1e *events.Envelope, usePreferredTags bool) { 121 | t := v1e.GetHttpStartStop() 122 | v2e.SourceId = convertAppUUID(t.GetApplicationId(), v2e.SourceId) 123 | v2e.InstanceId = strconv.Itoa(int(t.GetInstanceIndex())) 124 | v2e.Message = &loggregator_v2.Envelope_Timer{ 125 | Timer: &loggregator_v2.Timer{ 126 | Name: "http", 127 | Start: t.GetStartTimestamp(), 128 | Stop: t.GetStopTimestamp(), 129 | }, 130 | } 131 | setV2Tag(v2e, "request_id", uuidToString(t.GetRequestId()), usePreferredTags) 132 | setV2Tag(v2e, "peer_type", t.GetPeerType().String(), usePreferredTags) 133 | setV2Tag(v2e, "method", t.GetMethod().String(), usePreferredTags) 134 | setV2Tag(v2e, "uri", t.GetUri(), usePreferredTags) 135 | setV2Tag(v2e, "remote_address", t.GetRemoteAddress(), usePreferredTags) 136 | setV2Tag(v2e, "user_agent", t.GetUserAgent(), usePreferredTags) 137 | setV2Tag(v2e, "status_code", t.GetStatusCode(), usePreferredTags) 138 | setV2Tag(v2e, "content_length", t.GetContentLength(), usePreferredTags) 139 | setV2Tag(v2e, "routing_instance_id", t.GetInstanceId(), usePreferredTags) 140 | setV2Tag(v2e, "forwarded", strings.Join(t.GetForwarded(), "\n"), usePreferredTags) 141 | } 142 | 143 | func convertLogMessageType(t events.LogMessage_MessageType) loggregator_v2.Log_Type { 144 | name := events.LogMessage_MessageType_name[int32(t)] 145 | return loggregator_v2.Log_Type(loggregator_v2.Log_Type_value[name]) 146 | } 147 | 148 | func convertLogMessage(v2e *loggregator_v2.Envelope, e *events.Envelope, usePreferredTags bool) { 149 | t := e.GetLogMessage() 150 | setV2Tag(v2e, "source_type", t.GetSourceType(), usePreferredTags) 151 | v2e.InstanceId = t.GetSourceInstance() 152 | v2e.SourceId = convertAppID(t.GetAppId(), v2e.SourceId) 153 | 154 | v2e.Message = &loggregator_v2.Envelope_Log{ 155 | Log: &loggregator_v2.Log{ 156 | Payload: t.GetMessage(), 157 | Type: convertLogMessageType(t.GetMessageType()), 158 | }, 159 | } 160 | } 161 | 162 | func convertValueMetric(v2e *loggregator_v2.Envelope, e *events.Envelope) { 163 | t := e.GetValueMetric() 164 | v2e.InstanceId = e.GetTags()["instance_id"] 165 | v2e.Message = &loggregator_v2.Envelope_Gauge{ 166 | Gauge: &loggregator_v2.Gauge{ 167 | Metrics: map[string]*loggregator_v2.GaugeValue{ 168 | t.GetName(): { 169 | Unit: t.GetUnit(), 170 | Value: t.GetValue(), 171 | }, 172 | }, 173 | }, 174 | } 175 | } 176 | 177 | func convertCounterEvent(v2e *loggregator_v2.Envelope, e *events.Envelope) { 178 | t := e.GetCounterEvent() 179 | v2e.InstanceId = e.GetTags()["instance_id"] 180 | unsetV2Tag(v2e, "instance_id") 181 | v2e.Message = &loggregator_v2.Envelope_Counter{ 182 | Counter: &loggregator_v2.Counter{ 183 | Name: t.GetName(), 184 | Delta: t.GetDelta(), 185 | Total: t.GetTotal(), 186 | }, 187 | } 188 | } 189 | 190 | func convertContainerMetric(v2e *loggregator_v2.Envelope, e *events.Envelope) { 191 | t := e.GetContainerMetric() 192 | v2e.SourceId = convertAppID(t.GetApplicationId(), v2e.SourceId) 193 | v2e.InstanceId = strconv.Itoa(int(t.GetInstanceIndex())) 194 | v2e.Message = &loggregator_v2.Envelope_Gauge{ 195 | Gauge: &loggregator_v2.Gauge{ 196 | Metrics: map[string]*loggregator_v2.GaugeValue{ 197 | "cpu": { 198 | Unit: "percentage", 199 | Value: t.GetCpuPercentage(), 200 | }, 201 | "memory": { 202 | Unit: "bytes", 203 | Value: float64(t.GetMemoryBytes()), 204 | }, 205 | "disk": { 206 | Unit: "bytes", 207 | Value: float64(t.GetDiskBytes()), 208 | }, 209 | "memory_quota": { 210 | Unit: "bytes", 211 | Value: float64(t.GetMemoryBytesQuota()), 212 | }, 213 | "disk_quota": { 214 | Unit: "bytes", 215 | Value: float64(t.GetDiskBytesQuota()), 216 | }, 217 | }, 218 | }, 219 | } 220 | } 221 | 222 | func valueText(s string) *loggregator_v2.Value { 223 | return &loggregator_v2.Value{Data: &loggregator_v2.Value_Text{Text: s}} 224 | } 225 | 226 | func uuidToString(uuid *events.UUID) string { 227 | low := make([]byte, 8) 228 | high := make([]byte, 8) 229 | binary.LittleEndian.PutUint64(low, uuid.GetLow()) 230 | binary.LittleEndian.PutUint64(high, uuid.GetHigh()) 231 | return fmt.Sprintf("%x-%x-%x-%x-%x", low[:4], low[4:6], low[6:], high[:2], high[2:]) 232 | } 233 | -------------------------------------------------------------------------------- /conversion/container_metric_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | "github.com/cloudfoundry/sonde-go/events" 7 | . "github.com/onsi/ginkgo/v2" 8 | . "github.com/onsi/gomega" 9 | . "github.com/onsi/gomega/gstruct" 10 | "google.golang.org/protobuf/proto" 11 | ) 12 | 13 | var _ = Describe("ContainerMetric", func() { 14 | Context("given a v2 envelope", func() { 15 | It("converts to a v1 envelope", func() { 16 | envelope := &loggregator_v2.Envelope{ 17 | SourceId: "some-id", 18 | InstanceId: "123", 19 | Message: &loggregator_v2.Envelope_Gauge{ 20 | Gauge: &loggregator_v2.Gauge{ 21 | Metrics: map[string]*loggregator_v2.GaugeValue{ 22 | "cpu": { 23 | Unit: "percentage", 24 | Value: 11, 25 | }, 26 | "memory": { 27 | Unit: "bytes", 28 | Value: 13, 29 | }, 30 | "disk": { 31 | Unit: "bytes", 32 | Value: 15, 33 | }, 34 | "memory_quota": { 35 | Unit: "bytes", 36 | Value: 17, 37 | }, 38 | "disk_quota": { 39 | Unit: "bytes", 40 | Value: 19, 41 | }, 42 | }, 43 | }, 44 | }, 45 | } 46 | 47 | envelopes := conversion.ToV1(envelope) 48 | Expect(len(envelopes)).To(Equal(1)) 49 | Expect(envelopes[0].GetEventType()).To(Equal(events.Envelope_ContainerMetric)) 50 | Expect(proto.Equal(envelopes[0].GetContainerMetric(), &events.ContainerMetric{ 51 | ApplicationId: proto.String("some-id"), 52 | InstanceIndex: proto.Int32(123), 53 | CpuPercentage: proto.Float64(11), 54 | MemoryBytes: proto.Uint64(13), 55 | DiskBytes: proto.Uint64(15), 56 | MemoryBytesQuota: proto.Uint64(17), 57 | DiskBytesQuota: proto.Uint64(19), 58 | })).To(BeTrue()) 59 | }) 60 | 61 | It("sets InstanceIndex from GaugeValue if present", func() { 62 | envelope := &loggregator_v2.Envelope{ 63 | InstanceId: "123", 64 | Message: &loggregator_v2.Envelope_Gauge{ 65 | Gauge: &loggregator_v2.Gauge{ 66 | Metrics: map[string]*loggregator_v2.GaugeValue{ 67 | "instance_index": { 68 | Unit: "", 69 | Value: 19, 70 | }, 71 | "cpu": {Unit: "percent"}, 72 | "memory": {Unit: "percent"}, 73 | "disk": {Unit: "percent"}, 74 | "memory_quota": {Unit: "bytes"}, 75 | "disk_quota": {Unit: "bytes"}, 76 | }, 77 | }, 78 | }, 79 | } 80 | 81 | envelopes := conversion.ToV1(envelope) 82 | Expect(len(envelopes)).To(Equal(1)) 83 | Expect(envelopes[0].GetEventType()).To(Equal(events.Envelope_ContainerMetric)) 84 | Expect(proto.Equal(envelopes[0].GetContainerMetric(), &events.ContainerMetric{ 85 | ApplicationId: proto.String(""), 86 | InstanceIndex: proto.Int32(19), 87 | CpuPercentage: proto.Float64(0), 88 | MemoryBytes: proto.Uint64(0), 89 | DiskBytes: proto.Uint64(0), 90 | MemoryBytesQuota: proto.Uint64(0), 91 | DiskBytesQuota: proto.Uint64(0), 92 | })).To(BeTrue()) 93 | }) 94 | 95 | DescribeTable("it is resilient to malformed envelopes", func(v2e *loggregator_v2.Envelope) { 96 | Expect(conversion.ToV1(v2e)).To(HaveLen(0)) 97 | }, 98 | Entry("bare envelope", &loggregator_v2.Envelope{}), 99 | Entry("with empty fields", &loggregator_v2.Envelope{ 100 | Message: &loggregator_v2.Envelope_Gauge{ 101 | Gauge: &loggregator_v2.Gauge{ 102 | Metrics: map[string]*loggregator_v2.GaugeValue{ 103 | "cpu": nil, 104 | "memory": nil, 105 | "disk": nil, 106 | "memory_quota": nil, 107 | "disk_quota": nil, 108 | }, 109 | }, 110 | }, 111 | }), 112 | ) 113 | }) 114 | 115 | Context("given a v1 envelope", func() { 116 | var ( 117 | v1Envelope *events.Envelope 118 | expectedMessage *loggregator_v2.Envelope_Gauge 119 | ) 120 | 121 | BeforeEach(func() { 122 | v1Envelope = &events.Envelope{ 123 | Origin: proto.String("an-origin"), 124 | Deployment: proto.String("a-deployment"), 125 | Job: proto.String("a-job"), 126 | Index: proto.String("an-index"), 127 | Ip: proto.String("an-ip"), 128 | Timestamp: proto.Int64(1234), 129 | EventType: events.Envelope_ContainerMetric.Enum(), 130 | ContainerMetric: &events.ContainerMetric{ 131 | ApplicationId: proto.String("some-id"), 132 | InstanceIndex: proto.Int32(123), 133 | CpuPercentage: proto.Float64(11), 134 | MemoryBytes: proto.Uint64(13), 135 | DiskBytes: proto.Uint64(15), 136 | MemoryBytesQuota: proto.Uint64(17), 137 | DiskBytesQuota: proto.Uint64(19), 138 | }, 139 | Tags: map[string]string{ 140 | "custom_tag": "custom-value", 141 | }, 142 | } 143 | expectedMessage = &loggregator_v2.Envelope_Gauge{ 144 | Gauge: &loggregator_v2.Gauge{ 145 | Metrics: map[string]*loggregator_v2.GaugeValue{ 146 | "cpu": { 147 | Unit: "percentage", 148 | Value: 11, 149 | }, 150 | "memory": { 151 | Unit: "bytes", 152 | Value: 13, 153 | }, 154 | "disk": { 155 | Unit: "bytes", 156 | Value: 15, 157 | }, 158 | "memory_quota": { 159 | Unit: "bytes", 160 | Value: 17, 161 | }, 162 | "disk_quota": { 163 | Unit: "bytes", 164 | Value: 19, 165 | }, 166 | }, 167 | }, 168 | } 169 | }) 170 | 171 | Context("using deprecated tags", func() { 172 | It("converts to a v2 envelope using DeprecatedTags", func() { 173 | Expect(*conversion.ToV2(v1Envelope, false)).To(MatchFields(IgnoreExtras, Fields{ 174 | "Timestamp": Equal(int64(1234)), 175 | "SourceId": Equal("some-id"), 176 | "Message": Equal(expectedMessage), 177 | "InstanceId": Equal("123"), 178 | "DeprecatedTags": Equal(map[string]*loggregator_v2.Value{ 179 | "origin": {Data: &loggregator_v2.Value_Text{Text: "an-origin"}}, 180 | "deployment": {Data: &loggregator_v2.Value_Text{Text: "a-deployment"}}, 181 | "job": {Data: &loggregator_v2.Value_Text{Text: "a-job"}}, 182 | "index": {Data: &loggregator_v2.Value_Text{Text: "an-index"}}, 183 | "ip": {Data: &loggregator_v2.Value_Text{Text: "an-ip"}}, 184 | "__v1_type": {Data: &loggregator_v2.Value_Text{Text: "ContainerMetric"}}, 185 | "custom_tag": {Data: &loggregator_v2.Value_Text{Text: "custom-value"}}, 186 | }), 187 | "Tags": BeNil(), 188 | })) 189 | }) 190 | 191 | It("sets the source ID to deployment/job when App ID is missing", func() { 192 | localV1Envelope := &events.Envelope{ 193 | Deployment: proto.String("some-deployment"), 194 | Job: proto.String("some-job"), 195 | EventType: events.Envelope_ContainerMetric.Enum(), 196 | ContainerMetric: &events.ContainerMetric{}, 197 | } 198 | 199 | expectedV2Envelope := &loggregator_v2.Envelope{ 200 | SourceId: "some-deployment/some-job", 201 | } 202 | 203 | converted := conversion.ToV2(localV1Envelope, false) 204 | Expect(converted.GetSourceId()).To(Equal(expectedV2Envelope.GetSourceId())) 205 | }) 206 | }) 207 | 208 | Context("using preferred tags", func() { 209 | It("converts to a v2 envelope using Tags", func() { 210 | Expect(*conversion.ToV2(v1Envelope, true)).To(MatchFields(IgnoreExtras, Fields{ 211 | "Timestamp": Equal(int64(1234)), 212 | "SourceId": Equal("some-id"), 213 | "Message": Equal(expectedMessage), 214 | "InstanceId": Equal("123"), 215 | "DeprecatedTags": BeNil(), 216 | "Tags": Equal(map[string]string{ 217 | "origin": "an-origin", 218 | "deployment": "a-deployment", 219 | "job": "a-job", 220 | "index": "an-index", 221 | "ip": "an-ip", 222 | "__v1_type": "ContainerMetric", 223 | "custom_tag": "custom-value", 224 | }), 225 | })) 226 | }) 227 | }) 228 | }) 229 | }) 230 | -------------------------------------------------------------------------------- /v1/client.go: -------------------------------------------------------------------------------- 1 | // Package v1 provides a client to connect with the loggregtor v1 API 2 | // 3 | // Loggregator's v1 client library is better known to the Cloud Foundry 4 | // community as Dropsonde (github.com/cloudfoundry/dropsonde). The code here 5 | // wraps that library in the interest of consolidating all client code into 6 | // a single library which includes both v1 and v2 clients. 7 | package v1 8 | 9 | import ( 10 | "io" 11 | "log" 12 | "strconv" 13 | "time" 14 | 15 | loggregator "code.cloudfoundry.org/go-loggregator/v10" 16 | 17 | "github.com/cloudfoundry/dropsonde" 18 | "github.com/cloudfoundry/sonde-go/events" 19 | "google.golang.org/protobuf/proto" 20 | ) 21 | 22 | type ClientOption func(*Client) 23 | 24 | // WithTag allows for the configuration of arbitrary string value 25 | // metadata which will be included in all data sent to Loggregator 26 | func WithTag(name, value string) ClientOption { 27 | return func(c *Client) { 28 | c.tags[name] = value 29 | } 30 | } 31 | 32 | // WithLogger allows for the configuration of a logger. 33 | // By default, the logger is disabled. 34 | func WithLogger(l loggregator.Logger) ClientOption { 35 | return func(c *Client) { 36 | c.logger = l 37 | } 38 | } 39 | 40 | // Client represents an emitter into loggregator. It should be created with 41 | // the NewClient constructor. 42 | type Client struct { 43 | tags map[string]string 44 | logger loggregator.Logger 45 | } 46 | 47 | // NewClient creates a v1 loggregator client. This is a wrapper around the 48 | // dropsonde package that will write envelopes to loggregator over UDP. Before 49 | // calling NewClient you should call dropsonde.Initialize. 50 | func NewClient(opts ...ClientOption) (*Client, error) { 51 | c := &Client{ 52 | tags: make(map[string]string), 53 | logger: log.New(io.Discard, "", 0), 54 | } 55 | 56 | for _, o := range opts { 57 | o(c) 58 | } 59 | 60 | return c, nil 61 | } 62 | 63 | // EmitLog sends a message to loggregator. 64 | func (c *Client) EmitLog(message string, opts ...loggregator.EmitLogOption) { 65 | w := envelopeWrapper{ 66 | Messages: []*events.Envelope{ 67 | { 68 | Timestamp: proto.Int64(time.Now().UnixNano()), 69 | EventType: events.Envelope_LogMessage.Enum(), 70 | LogMessage: &events.LogMessage{ 71 | MessageType: events.LogMessage_ERR.Enum(), 72 | Message: []byte(message), 73 | Timestamp: proto.Int64(time.Now().UnixNano()), 74 | }, 75 | }, 76 | }, 77 | Tags: make(map[string]string), 78 | } 79 | 80 | for _, o := range opts { 81 | o(&w) 82 | } 83 | w.Messages[0].Tags = w.Tags 84 | 85 | c.emitEnvelope(w) 86 | } 87 | 88 | // EmitGauge sends the configured gauge values to loggregator. 89 | // If no EmitGaugeOption values are present, no envelopes will be emitted. 90 | func (c *Client) EmitGauge(opts ...loggregator.EmitGaugeOption) { 91 | w := envelopeWrapper{ 92 | Tags: make(map[string]string), 93 | } 94 | 95 | for _, o := range opts { 96 | o(&w) 97 | } 98 | 99 | if c.promoteToContainerMetric(w) { 100 | return 101 | } 102 | 103 | for _, e := range w.Messages { 104 | e.Timestamp = proto.Int64(time.Now().UnixNano()) 105 | e.EventType = events.Envelope_ValueMetric.Enum() 106 | e.Tags = w.Tags 107 | } 108 | 109 | c.emitEnvelope(w) 110 | } 111 | 112 | // Check to see if the envelope should be promoted to a ContainerMetric. 113 | func (c *Client) promoteToContainerMetric(w envelopeWrapper) bool { 114 | if len(w.Messages) != 5 { 115 | return false 116 | } 117 | appID, ok := w.Tags["source_id"] 118 | if !ok { 119 | return false 120 | } 121 | instanceIndex, err := strconv.ParseInt(w.Tags["instance_id"], 10, 32) 122 | if err != nil { 123 | return false 124 | } 125 | 126 | // We need 5 bools to determine if the envelope has all the required 127 | // name/value pairs. Each name will store it's presence in increasing 128 | // signifigance (i.e., 'cpu' is stored to bit 0, and 'memory' is stored to 129 | // bit 1 ...). 130 | cMetric := events.ContainerMetric{ 131 | ApplicationId: proto.String(appID), 132 | InstanceIndex: proto.Int32(int32(instanceIndex)), 133 | } 134 | 135 | var allPresent uint16 136 | for _, m := range w.Messages { 137 | switch m.GetValueMetric().GetName() { 138 | case "cpu": 139 | allPresent |= 1 140 | cMetric.CpuPercentage = proto.Float64(m.GetValueMetric().GetValue()) 141 | case "memory": 142 | allPresent |= 2 143 | cMetric.MemoryBytes = proto.Uint64(uint64(m.GetValueMetric().GetValue())) 144 | case "disk": 145 | allPresent |= 4 146 | cMetric.DiskBytes = proto.Uint64(uint64(m.GetValueMetric().GetValue())) 147 | case "memory_quota": 148 | allPresent |= 8 149 | cMetric.MemoryBytesQuota = proto.Uint64(uint64(m.GetValueMetric().GetValue())) 150 | case "disk_quota": 151 | allPresent |= 16 152 | cMetric.DiskBytesQuota = proto.Uint64(uint64(m.GetValueMetric().GetValue())) 153 | default: 154 | break 155 | } 156 | } 157 | 158 | // 0x1f implies that each of the required five fields were populated. 159 | if allPresent != 0x1f { 160 | return false 161 | } 162 | 163 | // Promote to Container 164 | container := &events.Envelope{ 165 | Timestamp: proto.Int64(time.Now().UnixNano()), 166 | EventType: events.Envelope_ContainerMetric.Enum(), 167 | Tags: w.Tags, 168 | ContainerMetric: &cMetric, 169 | } 170 | 171 | w.Messages = []*events.Envelope{container} 172 | c.emitEnvelope(w) 173 | 174 | return true 175 | } 176 | 177 | // EmitCounter sends a counter envelope with a delta of 1. 178 | func (c *Client) EmitCounter(name string, opts ...loggregator.EmitCounterOption) { 179 | w := envelopeWrapper{ 180 | Messages: []*events.Envelope{ 181 | { 182 | Timestamp: proto.Int64(time.Now().UnixNano()), 183 | EventType: events.Envelope_CounterEvent.Enum(), 184 | CounterEvent: &events.CounterEvent{ 185 | Name: proto.String(name), 186 | Delta: proto.Uint64(1), 187 | }, 188 | }, 189 | }, 190 | Tags: make(map[string]string), 191 | } 192 | 193 | for _, o := range opts { 194 | o(&w) 195 | } 196 | 197 | w.Messages[0].Tags = w.Tags 198 | 199 | c.emitEnvelope(w) 200 | } 201 | 202 | func (c *Client) emitEnvelope(w envelopeWrapper) { 203 | for _, e := range w.Messages { 204 | e.Origin = proto.String(dropsonde.DefaultEmitter.Origin()) 205 | for k, v := range c.tags { 206 | e.Tags[k] = v 207 | } 208 | 209 | err := dropsonde.DefaultEmitter.EmitEnvelope(e) 210 | if err != nil { 211 | c.logger.Printf("Failed to emit envelope: %s", err) 212 | } 213 | } 214 | } 215 | 216 | // envelopeWrapper is used to setup v1 Envelopes. 217 | type envelopeWrapper struct { 218 | proto.Message 219 | 220 | Messages []*events.Envelope 221 | Tags map[string]string 222 | } 223 | 224 | func (e *envelopeWrapper) SetGaugeAppInfo(appID string, index int) { 225 | e.SetSourceInfo(appID, strconv.Itoa(index)) 226 | } 227 | 228 | func (e *envelopeWrapper) SetCounterAppInfo(appID string, index int) { 229 | e.SetSourceInfo(appID, strconv.Itoa(index)) 230 | } 231 | 232 | func (e *envelopeWrapper) SetSourceInfo(sourceID, instanceID string) { 233 | e.Tags["source_id"] = sourceID 234 | e.Tags["instance_id"] = instanceID 235 | } 236 | 237 | func (e *envelopeWrapper) SetLogAppInfo(appID string, sourceType string, sourceInstance string) { 238 | e.Messages[0].GetLogMessage().AppId = proto.String(appID) 239 | e.Messages[0].GetLogMessage().SourceType = proto.String(sourceType) 240 | e.Messages[0].GetLogMessage().SourceInstance = proto.String(sourceInstance) 241 | } 242 | 243 | func (e *envelopeWrapper) SetLogToStdout() { 244 | e.Messages[0].GetLogMessage().MessageType = events.LogMessage_OUT.Enum() 245 | } 246 | 247 | func (e *envelopeWrapper) SetGaugeValue(name string, value float64, unit string) { 248 | e.Messages = append(e.Messages, &events.Envelope{ 249 | ValueMetric: &events.ValueMetric{ 250 | Name: proto.String(name), 251 | Value: proto.Float64(value), 252 | Unit: proto.String(unit), 253 | }, 254 | }) 255 | } 256 | 257 | func (e *envelopeWrapper) SetDelta(d uint64) { 258 | e.Messages[0].GetCounterEvent().Delta = proto.Uint64(d) 259 | } 260 | 261 | func (e *envelopeWrapper) SetTotal(t uint64) { 262 | e.Messages[0].GetCounterEvent().Delta = proto.Uint64(0) 263 | e.Messages[0].GetCounterEvent().Total = proto.Uint64(t) 264 | } 265 | 266 | func (e *envelopeWrapper) SetTag(name string, value string) { 267 | e.Tags[name] = value 268 | } 269 | -------------------------------------------------------------------------------- /examples/go.sum: -------------------------------------------------------------------------------- 1 | code.cloudfoundry.org/go-diodes v0.0.0-20180905200951-72629b5276e3 h1:oHsfl5AaineZubAUOXg2Vxcdu/TzgN/Q+/65lN70LZk= 2 | code.cloudfoundry.org/go-diodes v0.0.0-20180905200951-72629b5276e3/go.mod h1:Jzi+ccHgo/V/PLQUaQ6hnZcC1c4BS790gx21LRRui4g= 3 | code.cloudfoundry.org/tlsconfig v0.35.0 h1:WFzRmIxJXqeG7ThOiN4f6CgHfildCarAnm6USiTvbrY= 4 | code.cloudfoundry.org/tlsconfig v0.35.0/go.mod h1:IzS9v40FcSVaIXmBeRvliUySB6ToEDa2Qi3K5DYo+UU= 5 | filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= 6 | filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= 7 | github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= 8 | github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= 9 | github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 h1:afT88tB6u9JCKQZVAAaa9ICz/uGn5Uw9ekn6P22mYKM= 10 | github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:bXvGk6IkT1Agy7qzJ+DjIw/SJ1AaB3AvAuMDVV+Vkoo= 11 | github.com/cloudfoundry/dropsonde v1.1.0 h1:nerhj8K0heOsv/U/Ddou5Esw56YlNeHHJH6MP9QlACQ= 12 | github.com/cloudfoundry/dropsonde v1.1.0/go.mod h1:OrkxsBrAvM8X0Ve9vaSNKLR+/Jeohu3+J0M4JEaTmnM= 13 | github.com/cloudfoundry/sonde-go v0.0.0-20220627221915-ff36de9c3435 h1:xDsJBppu1bIny3d5CqbOZuCzezFbKhnrmrNucWLpNOk= 14 | github.com/cloudfoundry/sonde-go v0.0.0-20220627221915-ff36de9c3435/go.mod h1:DX2nqnCgs66F/YWnEtB8EtKWewkiKDu2SRn4y6Ln5NI= 15 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 16 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 17 | github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= 18 | github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 19 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 20 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 21 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 22 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 23 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 24 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 25 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 26 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 27 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 28 | github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= 29 | github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= 30 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 31 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 32 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 33 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 34 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 35 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 36 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 37 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 38 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 39 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 40 | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= 41 | github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= 42 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 43 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 44 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 45 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 46 | github.com/onsi/ginkgo/v2 v2.25.3 h1:Ty8+Yi/ayDAGtk4XxmmfUy4GabvM+MegeB4cDLRi6nw= 47 | github.com/onsi/ginkgo/v2 v2.25.3/go.mod h1:43uiyQC4Ed2tkOzLsEYm7hnrb7UJTWHYNsuy3bG/snE= 48 | github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= 49 | github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= 50 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 51 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 52 | github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= 53 | github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= 54 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 55 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 56 | go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= 57 | go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= 58 | go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= 59 | go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= 60 | go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= 61 | go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= 62 | go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= 63 | go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= 64 | go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= 65 | go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= 66 | go.step.sm/crypto v0.70.0 h1:Q9Ft7N637mucyZcHZd1+0VVQJVwDCKqcb9CYcYi7cds= 67 | go.step.sm/crypto v0.70.0/go.mod h1:pzfUhS5/ue7ev64PLlEgXvhx1opwbhFCjkvlhsxVds0= 68 | go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= 69 | go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= 70 | go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= 71 | go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= 72 | golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= 73 | golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= 74 | golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= 75 | golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= 76 | golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= 77 | golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 78 | golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= 79 | golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= 80 | golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= 81 | golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= 82 | gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= 83 | gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= 84 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= 85 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 86 | google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= 87 | google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= 88 | google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= 89 | google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 90 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 91 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 92 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 93 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 94 | -------------------------------------------------------------------------------- /envelope_stream_connector_test.go: -------------------------------------------------------------------------------- 1 | package loggregator_test 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "log" 8 | "net" 9 | "sync" 10 | "time" 11 | 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/codes" 14 | "google.golang.org/grpc/credentials" 15 | "google.golang.org/grpc/status" 16 | "google.golang.org/protobuf/proto" 17 | 18 | "code.cloudfoundry.org/go-loggregator/v10" 19 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 20 | "code.cloudfoundry.org/tlsconfig" 21 | . "github.com/onsi/ginkgo/v2" 22 | . "github.com/onsi/gomega" 23 | ) 24 | 25 | var _ = Describe("Connector", func() { 26 | It("initiates a connection to receive envelopes", func() { 27 | producer, err := newFakeEventProducer() 28 | Expect(err).NotTo(HaveOccurred()) 29 | producer.start() 30 | defer producer.stop() 31 | tlsConf, err := loggregator.NewIngressTLSConfig( 32 | fixture("CA.crt"), 33 | fixture("server.crt"), 34 | fixture("server.key"), 35 | ) 36 | Expect(err).NotTo(HaveOccurred()) 37 | 38 | addr := producer.addr 39 | req := &loggregator_v2.EgressBatchRequest{ShardId: "some-id"} 40 | c := loggregator.NewEnvelopeStreamConnector( 41 | addr, 42 | tlsConf, 43 | ) 44 | 45 | rx := c.Stream(context.Background(), req) 46 | 47 | Expect(len(rx())).NotTo(BeZero()) 48 | Expect(proto.Equal(producer.actualReq(), req)).To(BeTrue()) 49 | }) 50 | 51 | It("reconnects if the stream fails", func() { 52 | producer, err := newFakeEventProducer() 53 | Expect(err).NotTo(HaveOccurred()) 54 | 55 | // Producer will grab a port on start. When the producer is restarted, 56 | // it will grab the same port. 57 | producer.start() 58 | 59 | tlsConf, err := loggregator.NewIngressTLSConfig( 60 | fixture("CA.crt"), 61 | fixture("server.crt"), 62 | fixture("server.key"), 63 | ) 64 | Expect(err).NotTo(HaveOccurred()) 65 | 66 | addr := producer.addr 67 | c := loggregator.NewEnvelopeStreamConnector( 68 | addr, 69 | tlsConf, 70 | ) 71 | 72 | ctx, cancel := context.WithCancel(context.Background()) 73 | defer cancel() 74 | go func(ctx context.Context) { 75 | rx := c.Stream(ctx, &loggregator_v2.EgressBatchRequest{}) 76 | for { 77 | if ctx.Err() != nil { 78 | return 79 | } 80 | rx() 81 | } 82 | }(ctx) 83 | 84 | Eventually(producer.connectionAttempts).Should(Equal(1)) 85 | producer.stop() 86 | producer.start() 87 | defer producer.stop() 88 | 89 | // Reconnect after killing the server. 90 | Eventually(producer.connectionAttempts, 5).Should(Equal(2)) 91 | 92 | // Ensure we don't create new connections when we don't need to. 93 | Consistently(producer.connectionAttempts).Should(Equal(2)) 94 | }) 95 | 96 | It("enables buffering", func() { 97 | producer, err := newFakeEventProducer() 98 | Expect(err).NotTo(HaveOccurred()) 99 | 100 | // Producer will grab a port on start. When the producer is restarted, 101 | // it will grab the same port. 102 | producer.start() 103 | defer producer.stop() 104 | 105 | tlsConf, err := loggregator.NewIngressTLSConfig( 106 | fixture("CA.crt"), 107 | fixture("server.crt"), 108 | fixture("server.key"), 109 | ) 110 | Expect(err).NotTo(HaveOccurred()) 111 | 112 | var ( 113 | mu sync.Mutex 114 | missed int 115 | ) 116 | addr := producer.addr 117 | c := loggregator.NewEnvelopeStreamConnector( 118 | addr, 119 | tlsConf, 120 | loggregator.WithEnvelopeStreamBuffer(5, func(m int) { 121 | mu.Lock() 122 | defer mu.Unlock() 123 | missed += m 124 | }), 125 | ) 126 | rx := c.Stream(context.Background(), &loggregator_v2.EgressBatchRequest{}) 127 | 128 | var count int 129 | ctx, cancel := context.WithCancel(context.Background()) 130 | defer cancel() 131 | // Read to allow the diode to notice it dropped data 132 | go func(ctx context.Context) { 133 | for { 134 | select { 135 | case <-time.Tick(500 * time.Millisecond): 136 | // Do not invoke rx while mu is locked 137 | l := len(rx()) 138 | mu.Lock() 139 | count += l 140 | mu.Unlock() 141 | case <-ctx.Done(): 142 | return 143 | } 144 | } 145 | }(ctx) 146 | 147 | Eventually(func() int { 148 | mu.Lock() 149 | defer mu.Unlock() 150 | return missed 151 | }).ShouldNot(BeZero()) 152 | 153 | mu.Lock() 154 | l := count 155 | mu.Unlock() 156 | Expect(l).ToNot(BeZero()) 157 | }) 158 | 159 | It("won't panic when context canceled", func() { 160 | producer, err := newFakeEventProducer() 161 | Expect(err).NotTo(HaveOccurred()) 162 | producer.start() 163 | defer producer.stop() 164 | 165 | tlsConf, err := loggregator.NewIngressTLSConfig( 166 | fixture("CA.crt"), 167 | fixture("server.crt"), 168 | fixture("server.key"), 169 | ) 170 | Expect(err).NotTo(HaveOccurred()) 171 | 172 | c := loggregator.NewEnvelopeStreamConnector( 173 | producer.addr, 174 | tlsConf, 175 | loggregator.WithEnvelopeStreamBuffer(5, func(m int) {}), 176 | ) 177 | 178 | ctx, cancel := context.WithCancel(context.Background()) 179 | rx := c.Stream(ctx, &loggregator_v2.EgressBatchRequest{}) 180 | 181 | cancel() 182 | msg := rx() 183 | Expect(msg).To(BeNil()) 184 | }) 185 | }) 186 | 187 | type fakeEventProducer struct { 188 | loggregator_v2.UnimplementedEgressServer 189 | 190 | server *grpc.Server 191 | addr string 192 | 193 | mu sync.Mutex 194 | connectionAttempts_ int 195 | actualReq_ *loggregator_v2.EgressBatchRequest 196 | } 197 | 198 | func newFakeEventProducer() (*fakeEventProducer, error) { 199 | f := &fakeEventProducer{} 200 | 201 | return f, nil 202 | } 203 | 204 | func (f *fakeEventProducer) Receiver( 205 | *loggregator_v2.EgressRequest, 206 | loggregator_v2.Egress_ReceiverServer, 207 | ) error { 208 | 209 | return status.Errorf(codes.Unimplemented, "use BatchedReceiver instead") 210 | } 211 | 212 | func (f *fakeEventProducer) BatchedReceiver( 213 | req *loggregator_v2.EgressBatchRequest, 214 | srv loggregator_v2.Egress_BatchedReceiverServer, 215 | ) error { 216 | f.mu.Lock() 217 | f.connectionAttempts_++ 218 | f.actualReq_ = req 219 | f.mu.Unlock() 220 | var i int 221 | for range time.Tick(10 * time.Millisecond) { 222 | err := srv.Send(&loggregator_v2.EnvelopeBatch{ 223 | Batch: []*loggregator_v2.Envelope{ 224 | { 225 | SourceId: fmt.Sprintf("envelope-%d", i), 226 | Message: &loggregator_v2.Envelope_Event{ 227 | Event: &loggregator_v2.Event{ 228 | Title: "event-name", 229 | Body: "event-body", 230 | }, 231 | }, 232 | }, 233 | }, 234 | }) 235 | if err != nil { 236 | return err 237 | } 238 | i++ 239 | } 240 | return nil 241 | } 242 | 243 | func (f *fakeEventProducer) start() { 244 | addr := f.addr 245 | if addr == "" { 246 | addr = "127.0.0.1:0" 247 | } 248 | var lis net.Listener 249 | for i := 0; ; i++ { 250 | var err error 251 | lis, err = net.Listen("tcp", addr) 252 | if err != nil { 253 | // This can happen if the port is already in use... 254 | if i < 50 { 255 | log.Printf("failed to bind for fake producer. Trying again (%d/50)...: %s", i+1, err) 256 | time.Sleep(100 * time.Millisecond) 257 | continue 258 | } 259 | panic(err) 260 | } 261 | break 262 | } 263 | f.addr = lis.Addr().String() 264 | c, err := newServerMutualTLSConfig() 265 | if err != nil { 266 | panic(err) 267 | } 268 | opt := grpc.Creds(credentials.NewTLS(c)) 269 | f.server = grpc.NewServer(opt) 270 | loggregator_v2.RegisterEgressServer(f.server, f) 271 | 272 | go f.listen(lis) 273 | } 274 | 275 | func (f *fakeEventProducer) listen(lis net.Listener) { 276 | _ = f.server.Serve(lis) 277 | } 278 | 279 | func (f *fakeEventProducer) stop() bool { 280 | if f.server == nil { 281 | return false 282 | } 283 | 284 | f.server.Stop() 285 | f.server = nil 286 | return true 287 | } 288 | 289 | func (f *fakeEventProducer) actualReq() *loggregator_v2.EgressBatchRequest { 290 | f.mu.Lock() 291 | defer f.mu.Unlock() 292 | return f.actualReq_ 293 | } 294 | 295 | func (f *fakeEventProducer) connectionAttempts() int { 296 | f.mu.Lock() 297 | defer f.mu.Unlock() 298 | return f.connectionAttempts_ 299 | } 300 | 301 | func newServerMutualTLSConfig() (*tls.Config, error) { 302 | certFile := fixture("server.crt") 303 | keyFile := fixture("server.key") 304 | caCertFile := fixture("CA.crt") 305 | 306 | return tlsconfig.Build( 307 | tlsconfig.WithInternalServiceDefaults(), 308 | tlsconfig.WithIdentityFromFile(certFile, keyFile), 309 | ).Server( 310 | tlsconfig.WithClientAuthenticationFromFile(caCertFile), 311 | ) 312 | } 313 | -------------------------------------------------------------------------------- /rpc/loggregator_v2/ingress_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.3.0 4 | // - protoc v5.26.1 5 | // source: loggregator-api/v2/ingress.proto 6 | 7 | package loggregator_v2 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file 17 | // is compatible with the grpc package it is being compiled against. 18 | // Requires gRPC-Go v1.32.0 or later. 19 | const _ = grpc.SupportPackageIsVersion7 20 | 21 | const ( 22 | Ingress_Sender_FullMethodName = "/loggregator.v2.Ingress/Sender" 23 | Ingress_BatchSender_FullMethodName = "/loggregator.v2.Ingress/BatchSender" 24 | Ingress_Send_FullMethodName = "/loggregator.v2.Ingress/Send" 25 | ) 26 | 27 | // IngressClient is the client API for Ingress service. 28 | // 29 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 30 | type IngressClient interface { 31 | Sender(ctx context.Context, opts ...grpc.CallOption) (Ingress_SenderClient, error) 32 | BatchSender(ctx context.Context, opts ...grpc.CallOption) (Ingress_BatchSenderClient, error) 33 | Send(ctx context.Context, in *EnvelopeBatch, opts ...grpc.CallOption) (*SendResponse, error) 34 | } 35 | 36 | type ingressClient struct { 37 | cc grpc.ClientConnInterface 38 | } 39 | 40 | func NewIngressClient(cc grpc.ClientConnInterface) IngressClient { 41 | return &ingressClient{cc} 42 | } 43 | 44 | func (c *ingressClient) Sender(ctx context.Context, opts ...grpc.CallOption) (Ingress_SenderClient, error) { 45 | stream, err := c.cc.NewStream(ctx, &Ingress_ServiceDesc.Streams[0], Ingress_Sender_FullMethodName, opts...) 46 | if err != nil { 47 | return nil, err 48 | } 49 | x := &ingressSenderClient{stream} 50 | return x, nil 51 | } 52 | 53 | type Ingress_SenderClient interface { 54 | Send(*Envelope) error 55 | CloseAndRecv() (*IngressResponse, error) 56 | grpc.ClientStream 57 | } 58 | 59 | type ingressSenderClient struct { 60 | grpc.ClientStream 61 | } 62 | 63 | func (x *ingressSenderClient) Send(m *Envelope) error { 64 | return x.ClientStream.SendMsg(m) 65 | } 66 | 67 | func (x *ingressSenderClient) CloseAndRecv() (*IngressResponse, error) { 68 | if err := x.ClientStream.CloseSend(); err != nil { 69 | return nil, err 70 | } 71 | m := new(IngressResponse) 72 | if err := x.ClientStream.RecvMsg(m); err != nil { 73 | return nil, err 74 | } 75 | return m, nil 76 | } 77 | 78 | func (c *ingressClient) BatchSender(ctx context.Context, opts ...grpc.CallOption) (Ingress_BatchSenderClient, error) { 79 | stream, err := c.cc.NewStream(ctx, &Ingress_ServiceDesc.Streams[1], Ingress_BatchSender_FullMethodName, opts...) 80 | if err != nil { 81 | return nil, err 82 | } 83 | x := &ingressBatchSenderClient{stream} 84 | return x, nil 85 | } 86 | 87 | type Ingress_BatchSenderClient interface { 88 | Send(*EnvelopeBatch) error 89 | CloseAndRecv() (*BatchSenderResponse, error) 90 | grpc.ClientStream 91 | } 92 | 93 | type ingressBatchSenderClient struct { 94 | grpc.ClientStream 95 | } 96 | 97 | func (x *ingressBatchSenderClient) Send(m *EnvelopeBatch) error { 98 | return x.ClientStream.SendMsg(m) 99 | } 100 | 101 | func (x *ingressBatchSenderClient) CloseAndRecv() (*BatchSenderResponse, error) { 102 | if err := x.ClientStream.CloseSend(); err != nil { 103 | return nil, err 104 | } 105 | m := new(BatchSenderResponse) 106 | if err := x.ClientStream.RecvMsg(m); err != nil { 107 | return nil, err 108 | } 109 | return m, nil 110 | } 111 | 112 | func (c *ingressClient) Send(ctx context.Context, in *EnvelopeBatch, opts ...grpc.CallOption) (*SendResponse, error) { 113 | out := new(SendResponse) 114 | err := c.cc.Invoke(ctx, Ingress_Send_FullMethodName, in, out, opts...) 115 | if err != nil { 116 | return nil, err 117 | } 118 | return out, nil 119 | } 120 | 121 | // IngressServer is the server API for Ingress service. 122 | // All implementations must embed UnimplementedIngressServer 123 | // for forward compatibility 124 | type IngressServer interface { 125 | Sender(Ingress_SenderServer) error 126 | BatchSender(Ingress_BatchSenderServer) error 127 | Send(context.Context, *EnvelopeBatch) (*SendResponse, error) 128 | mustEmbedUnimplementedIngressServer() 129 | } 130 | 131 | // UnimplementedIngressServer must be embedded to have forward compatible implementations. 132 | type UnimplementedIngressServer struct { 133 | } 134 | 135 | func (UnimplementedIngressServer) Sender(Ingress_SenderServer) error { 136 | return status.Errorf(codes.Unimplemented, "method Sender not implemented") 137 | } 138 | func (UnimplementedIngressServer) BatchSender(Ingress_BatchSenderServer) error { 139 | return status.Errorf(codes.Unimplemented, "method BatchSender not implemented") 140 | } 141 | func (UnimplementedIngressServer) Send(context.Context, *EnvelopeBatch) (*SendResponse, error) { 142 | return nil, status.Errorf(codes.Unimplemented, "method Send not implemented") 143 | } 144 | func (UnimplementedIngressServer) mustEmbedUnimplementedIngressServer() {} 145 | 146 | // UnsafeIngressServer may be embedded to opt out of forward compatibility for this service. 147 | // Use of this interface is not recommended, as added methods to IngressServer will 148 | // result in compilation errors. 149 | type UnsafeIngressServer interface { 150 | mustEmbedUnimplementedIngressServer() 151 | } 152 | 153 | func RegisterIngressServer(s grpc.ServiceRegistrar, srv IngressServer) { 154 | s.RegisterService(&Ingress_ServiceDesc, srv) 155 | } 156 | 157 | func _Ingress_Sender_Handler(srv interface{}, stream grpc.ServerStream) error { 158 | return srv.(IngressServer).Sender(&ingressSenderServer{stream}) 159 | } 160 | 161 | type Ingress_SenderServer interface { 162 | SendAndClose(*IngressResponse) error 163 | Recv() (*Envelope, error) 164 | grpc.ServerStream 165 | } 166 | 167 | type ingressSenderServer struct { 168 | grpc.ServerStream 169 | } 170 | 171 | func (x *ingressSenderServer) SendAndClose(m *IngressResponse) error { 172 | return x.ServerStream.SendMsg(m) 173 | } 174 | 175 | func (x *ingressSenderServer) Recv() (*Envelope, error) { 176 | m := new(Envelope) 177 | if err := x.ServerStream.RecvMsg(m); err != nil { 178 | return nil, err 179 | } 180 | return m, nil 181 | } 182 | 183 | func _Ingress_BatchSender_Handler(srv interface{}, stream grpc.ServerStream) error { 184 | return srv.(IngressServer).BatchSender(&ingressBatchSenderServer{stream}) 185 | } 186 | 187 | type Ingress_BatchSenderServer interface { 188 | SendAndClose(*BatchSenderResponse) error 189 | Recv() (*EnvelopeBatch, error) 190 | grpc.ServerStream 191 | } 192 | 193 | type ingressBatchSenderServer struct { 194 | grpc.ServerStream 195 | } 196 | 197 | func (x *ingressBatchSenderServer) SendAndClose(m *BatchSenderResponse) error { 198 | return x.ServerStream.SendMsg(m) 199 | } 200 | 201 | func (x *ingressBatchSenderServer) Recv() (*EnvelopeBatch, error) { 202 | m := new(EnvelopeBatch) 203 | if err := x.ServerStream.RecvMsg(m); err != nil { 204 | return nil, err 205 | } 206 | return m, nil 207 | } 208 | 209 | func _Ingress_Send_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 210 | in := new(EnvelopeBatch) 211 | if err := dec(in); err != nil { 212 | return nil, err 213 | } 214 | if interceptor == nil { 215 | return srv.(IngressServer).Send(ctx, in) 216 | } 217 | info := &grpc.UnaryServerInfo{ 218 | Server: srv, 219 | FullMethod: Ingress_Send_FullMethodName, 220 | } 221 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 222 | return srv.(IngressServer).Send(ctx, req.(*EnvelopeBatch)) 223 | } 224 | return interceptor(ctx, in, info, handler) 225 | } 226 | 227 | // Ingress_ServiceDesc is the grpc.ServiceDesc for Ingress service. 228 | // It's only intended for direct use with grpc.RegisterService, 229 | // and not to be introspected or modified (even as a copy) 230 | var Ingress_ServiceDesc = grpc.ServiceDesc{ 231 | ServiceName: "loggregator.v2.Ingress", 232 | HandlerType: (*IngressServer)(nil), 233 | Methods: []grpc.MethodDesc{ 234 | { 235 | MethodName: "Send", 236 | Handler: _Ingress_Send_Handler, 237 | }, 238 | }, 239 | Streams: []grpc.StreamDesc{ 240 | { 241 | StreamName: "Sender", 242 | Handler: _Ingress_Sender_Handler, 243 | ClientStreams: true, 244 | }, 245 | { 246 | StreamName: "BatchSender", 247 | Handler: _Ingress_BatchSender_Handler, 248 | ClientStreams: true, 249 | }, 250 | }, 251 | Metadata: "loggregator-api/v2/ingress.proto", 252 | } 253 | -------------------------------------------------------------------------------- /conversion/benchmark/benchmark_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 8 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 9 | 10 | "github.com/cloudfoundry/sonde-go/events" 11 | "google.golang.org/protobuf/proto" 12 | ) 13 | 14 | func BenchmarkToV2Log(b *testing.B) { 15 | for i := 0; i < b.N; i++ { 16 | outV2 = conversion.ToV2(&logMessageV1, true) 17 | } 18 | } 19 | 20 | func BenchmarkToV2HttpStartStop(b *testing.B) { 21 | for i := 0; i < b.N; i++ { 22 | outV2 = conversion.ToV2(&httpStartStopV1, true) 23 | } 24 | } 25 | 26 | func BenchmarkToV2ValueMetric(b *testing.B) { 27 | for i := 0; i < b.N; i++ { 28 | outV2 = conversion.ToV2(&valueMetricV1, true) 29 | } 30 | } 31 | 32 | func BenchmarkToV2CounterEvent(b *testing.B) { 33 | for i := 0; i < b.N; i++ { 34 | outV2 = conversion.ToV2(&counterEventV1, true) 35 | } 36 | } 37 | 38 | func BenchmarkToV1EnvelopeLog(b *testing.B) { 39 | for i := 0; i < b.N; i++ { 40 | outV1 = conversion.ToV1(envelopeLogV2) 41 | } 42 | } 43 | 44 | func BenchmarkToV1EnvelopeCounter(b *testing.B) { 45 | for i := 0; i < b.N; i++ { 46 | outV1 = conversion.ToV1(envelopeCounterV2) 47 | } 48 | } 49 | 50 | func BenchmarkToV1EnvelopeGauge(b *testing.B) { 51 | for i := 0; i < b.N; i++ { 52 | outV1 = conversion.ToV1(envelopeGaugeV2) 53 | } 54 | } 55 | 56 | func BenchmarkToV1EnvelopeTimer(b *testing.B) { 57 | for i := 0; i < b.N; i++ { 58 | outV1 = conversion.ToV1(envelopeTimerV2) 59 | } 60 | } 61 | 62 | var ( 63 | outV1 []*events.Envelope 64 | outV2 *loggregator_v2.Envelope 65 | httpStartStopV1 = events.Envelope{ 66 | Origin: proto.String("origin"), 67 | Timestamp: proto.Int64(time.Now().UnixNano()), 68 | EventType: events.Envelope_HttpStartStop.Enum(), 69 | Deployment: proto.String("deployment"), 70 | Job: proto.String("job-name"), 71 | Index: proto.String("just-an-average-sized-index-id"), 72 | Ip: proto.String("127.0.0.1"), 73 | HttpStartStop: &events.HttpStartStop{ 74 | StartTimestamp: proto.Int64(time.Now().UnixNano()), 75 | StopTimestamp: proto.Int64(time.Now().UnixNano()), 76 | RequestId: &events.UUID{ 77 | Low: proto.Uint64(12345), 78 | High: proto.Uint64(12345), 79 | }, 80 | PeerType: events.PeerType_Client.Enum(), 81 | Method: events.Method_GET.Enum(), 82 | Uri: proto.String("https://thing.thing.thing/thing"), 83 | RemoteAddress: proto.String("127.0.0.1"), 84 | UserAgent: proto.String("super-realistic-agent"), 85 | StatusCode: proto.Int32(123), 86 | ContentLength: proto.Int64(1234), 87 | ApplicationId: &events.UUID{ 88 | Low: proto.Uint64(12345), 89 | High: proto.Uint64(12345), 90 | }, 91 | InstanceIndex: proto.Int32(2), 92 | InstanceId: proto.String("a-really-medium-sized-guid-maybe"), 93 | Forwarded: []string{"forwarded for someone"}, 94 | }, 95 | Tags: map[string]string{ 96 | "adding": "tags", 97 | "for": "good", 98 | "measure": "mkay", 99 | }, 100 | } 101 | logMessageV1 = events.Envelope{ 102 | Origin: proto.String("origin"), 103 | Timestamp: proto.Int64(time.Now().UnixNano()), 104 | EventType: events.Envelope_LogMessage.Enum(), 105 | Deployment: proto.String("deployment"), 106 | Job: proto.String("job-name"), 107 | Index: proto.String("just-an-average-sized-index-id"), 108 | Ip: proto.String("127.0.0.1"), 109 | LogMessage: &events.LogMessage{ 110 | Message: []byte("a log message of a decent size that none would mine too much cause its not too big but not too small"), 111 | MessageType: events.LogMessage_OUT.Enum(), 112 | Timestamp: proto.Int64(time.Now().UnixNano()), 113 | AppId: proto.String("just-an-average-sized-app-id"), 114 | SourceInstance: proto.String("just-an-average-sized-source-instance"), 115 | SourceType: proto.String("something/medium"), 116 | }, 117 | Tags: map[string]string{ 118 | "adding": "tags", 119 | "for": "good", 120 | "measure": "mkay", 121 | }, 122 | } 123 | valueMetricV1 = events.Envelope{ 124 | Origin: proto.String("origin"), 125 | Timestamp: proto.Int64(time.Now().UnixNano()), 126 | EventType: events.Envelope_ValueMetric.Enum(), 127 | Deployment: proto.String("deployment"), 128 | Job: proto.String("job-name"), 129 | Index: proto.String("just-an-average-sized-index-id"), 130 | Ip: proto.String("127.0.0.1"), 131 | ValueMetric: &events.ValueMetric{ 132 | Name: proto.String("some-value-metric"), 133 | Value: proto.Float64(0.23232), 134 | Unit: proto.String("things"), 135 | }, 136 | Tags: map[string]string{ 137 | "adding": "tags", 138 | "for": "good", 139 | "measure": "mkay", 140 | }, 141 | } 142 | counterEventV1 = events.Envelope{ 143 | Origin: proto.String("origin"), 144 | Timestamp: proto.Int64(time.Now().UnixNano()), 145 | EventType: events.Envelope_CounterEvent.Enum(), 146 | Deployment: proto.String("deployment"), 147 | Job: proto.String("job-name"), 148 | Index: proto.String("just-an-average-sized-index-id"), 149 | Ip: proto.String("127.0.0.1"), 150 | CounterEvent: &events.CounterEvent{ 151 | Name: proto.String("some-value-metric"), 152 | Total: proto.Uint64(23232), 153 | }, 154 | Tags: map[string]string{ 155 | "adding": "tags", 156 | "for": "good", 157 | "measure": "mkay", 158 | }, 159 | } 160 | envelopeLogV2 = &loggregator_v2.Envelope{ 161 | Timestamp: time.Now().UnixNano(), 162 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 163 | InstanceId: "99", 164 | Message: &loggregator_v2.Envelope_Log{ 165 | Log: &loggregator_v2.Log{ 166 | Payload: []byte("some-payload"), 167 | Type: loggregator_v2.Log_OUT, 168 | }, 169 | }, 170 | Tags: map[string]string{ 171 | "source_type": "some-source-type", 172 | "deployment": "some-deployment", 173 | "ip": "some-ip", 174 | "job": "some-job", 175 | "origin": "some-origin", 176 | "index": "some-index", 177 | "__v1_type": "LogMessage", 178 | }, 179 | } 180 | envelopeCounterV2 = &loggregator_v2.Envelope{ 181 | Timestamp: time.Now().UnixNano(), 182 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 183 | InstanceId: "99", 184 | Message: &loggregator_v2.Envelope_Counter{ 185 | Counter: &loggregator_v2.Counter{ 186 | Name: "some-name", 187 | Total: 99, 188 | }, 189 | }, 190 | Tags: map[string]string{ 191 | "deployment": "some-deployment", 192 | "ip": "some-ip", 193 | "job": "some-job", 194 | "origin": "some-origin", 195 | "index": "some-index", 196 | "__v1_type": "CounterEvent", 197 | }, 198 | } 199 | envelopeGaugeV2 = &loggregator_v2.Envelope{ 200 | Timestamp: time.Now().UnixNano(), 201 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 202 | InstanceId: "99", 203 | Message: &loggregator_v2.Envelope_Gauge{ 204 | Gauge: &loggregator_v2.Gauge{ 205 | Metrics: map[string]*loggregator_v2.GaugeValue{ 206 | "cpu": { 207 | Unit: "percentage", Value: 0.18079146710267877, 208 | }, 209 | "disk": { 210 | Unit: "bytes", Value: 7.9466496e+07, 211 | }, 212 | "disk_quota": { 213 | Unit: "bytes", Value: 1.073741824e+09, 214 | }, 215 | "memory": { 216 | Unit: "bytes", Value: 2.5223168e+07, 217 | }, 218 | "memory_quota": { 219 | Unit: "bytes", Value: 2.68435456e+08, 220 | }, 221 | }, 222 | }, 223 | }, 224 | Tags: map[string]string{ 225 | "deployment": "some-deployment", 226 | "ip": "some-ip", 227 | "job": "some-job", 228 | "origin": "some-origin", 229 | "index": "some-index", 230 | "__v1_type": "ContainerMetric", 231 | }, 232 | } 233 | envelopeTimerV2 = &loggregator_v2.Envelope{ 234 | Timestamp: time.Now().UnixNano(), 235 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 236 | InstanceId: "99", 237 | Message: &loggregator_v2.Envelope_Timer{ 238 | Timer: &loggregator_v2.Timer{ 239 | Name: "http", 240 | Start: 99, 241 | Stop: 100, 242 | }, 243 | }, 244 | Tags: map[string]string{ 245 | "request_id": "954f61c4-ac84-44be-9217-cdfa3117fb41", 246 | "peer_type": "Client", 247 | "method": "GET", 248 | "uri": "/hello-world", 249 | "remote_address": "10.1.1.0", 250 | "user_agent": "Mozilla/5.0", 251 | "status_code": "200", 252 | "content_length": "1000000", 253 | "routing_instance_id": "application-id", 254 | "forwarded": "6.6.6.6\n8.8.8.8", 255 | "deployment": "some-deployment", 256 | "ip": "some-ip", 257 | "job": "some-job", 258 | "origin": "some-origin", 259 | "index": "some-index", 260 | "__v1_type": "HttpStartStop", 261 | }, 262 | } 263 | ) 264 | -------------------------------------------------------------------------------- /rlp_gateway_client.go: -------------------------------------------------------------------------------- 1 | package loggregator 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "log" 11 | "net/http" 12 | "strings" 13 | "time" 14 | 15 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 16 | "google.golang.org/protobuf/encoding/protojson" 17 | ) 18 | 19 | type RLPGatewayClient struct { 20 | addr string 21 | log Logger 22 | doer Doer 23 | maxRetries int 24 | errChan chan error 25 | } 26 | 27 | type GatewayLogger interface { 28 | Printf(format string, v ...interface{}) 29 | Panicf(format string, v ...interface{}) 30 | } 31 | 32 | func NewRLPGatewayClient(addr string, opts ...RLPGatewayClientOption) *RLPGatewayClient { 33 | c := &RLPGatewayClient{ 34 | addr: addr, 35 | log: log.New(io.Discard, "", 0), 36 | doer: http.DefaultClient, 37 | maxRetries: 10, 38 | } 39 | 40 | for _, o := range opts { 41 | o(c) 42 | } 43 | 44 | return c 45 | } 46 | 47 | // RLPGatewayClientOption is the type of a configurable client option. 48 | type RLPGatewayClientOption func(*RLPGatewayClient) 49 | 50 | // WithRLPGatewayClientLogger returns a RLPGatewayClientOption to configure 51 | // the logger of the RLPGatewayClient. It defaults to a silent logger. 52 | func WithRLPGatewayClientLogger(log GatewayLogger) RLPGatewayClientOption { 53 | return func(c *RLPGatewayClient) { 54 | c.log = log 55 | } 56 | } 57 | 58 | // WithRLPGatewayClientLogger returns a RLPGatewayClientOption to configure 59 | // the HTTP client. It defaults to the http.DefaultClient. 60 | func WithRLPGatewayHTTPClient(d Doer) RLPGatewayClientOption { 61 | return func(c *RLPGatewayClient) { 62 | c.doer = d 63 | } 64 | } 65 | 66 | // WithRLPGatewayMaxRetries returns a RLPGatewayClientOption to configure 67 | // how many times the client will attempt to connect to the RLP gateway 68 | // before giving up. 69 | func WithRLPGatewayMaxRetries(r int) RLPGatewayClientOption { 70 | return func(c *RLPGatewayClient) { 71 | c.maxRetries = r 72 | } 73 | } 74 | 75 | // WithRLPGatewayErrChan returns a RLPGatewayClientOption to configure 76 | // an error channel to communicate errors when the client exceeds max retries 77 | func WithRLPGatewayErrChan(errChan chan error) RLPGatewayClientOption { 78 | return func(c *RLPGatewayClient) { 79 | c.errChan = errChan 80 | } 81 | } 82 | 83 | // Doer is used to make HTTP requests to the RLP Gateway. 84 | type Doer interface { 85 | // Do is a implementation of the http.Client's Do method. 86 | Do(*http.Request) (*http.Response, error) 87 | } 88 | 89 | // Stream returns a new EnvelopeStream for the given context and request. The 90 | // lifecycle of the EnvelopeStream is managed by the given context. If the 91 | // underlying SSE stream dies, it attempts to reconnect until the context 92 | // is done. Any errors are logged via the client's logger. 93 | func (c *RLPGatewayClient) Stream(ctx context.Context, req *loggregator_v2.EgressBatchRequest) EnvelopeStream { 94 | es := make(chan []*loggregator_v2.Envelope, 100) 95 | go c.connectToStream(es, ctx, req)() 96 | return streamEnvelopes(ctx, es) 97 | } 98 | 99 | func (c *RLPGatewayClient) connectToStream(es chan []*loggregator_v2.Envelope, ctx context.Context, req *loggregator_v2.EgressBatchRequest) func() { 100 | var numRetries int 101 | return func() { 102 | defer close(es) 103 | for ctx.Err() == nil && numRetries <= c.maxRetries { 104 | connectionSucceeded := c.connect(ctx, es, req) 105 | if connectionSucceeded { 106 | numRetries = 0 107 | continue 108 | } 109 | numRetries++ 110 | } 111 | 112 | if numRetries > c.maxRetries { 113 | select { 114 | case c.errChan <- errors.New("client connection attempts exceeded max retries -- giving up"): 115 | default: 116 | log.Printf("unable to write error to err chan -- givin up") 117 | } 118 | } 119 | } 120 | } 121 | 122 | func streamEnvelopes(ctx context.Context, es chan []*loggregator_v2.Envelope) func() []*loggregator_v2.Envelope { 123 | return func() []*loggregator_v2.Envelope { 124 | for { 125 | select { 126 | case <-ctx.Done(): 127 | return nil 128 | case e, ok := <-es: 129 | if !ok { 130 | return nil 131 | } 132 | return e 133 | default: 134 | time.Sleep(50 * time.Millisecond) 135 | } 136 | } 137 | } 138 | } 139 | 140 | func (c *RLPGatewayClient) connect( 141 | ctx context.Context, 142 | es chan<- []*loggregator_v2.Envelope, 143 | logReq *loggregator_v2.EgressBatchRequest, 144 | ) bool { 145 | readAddr := fmt.Sprintf("%s/v2/read%s", c.addr, c.buildQuery(logReq)) 146 | 147 | req, err := http.NewRequest(http.MethodGet, readAddr, nil) 148 | if err != nil { 149 | c.log.Panicf("failed to build request %s", err) 150 | } 151 | req.Header.Set("Accept", "text/event-stream") 152 | req.Header.Set("Cache-Control", "no-cache") 153 | 154 | resp, err := c.doer.Do(req.WithContext(ctx)) 155 | if err != nil { 156 | c.log.Printf("error making request: %s", err) 157 | return false 158 | } 159 | 160 | defer func() { 161 | _, _ = io.Copy(io.Discard, resp.Body) 162 | _ = resp.Body.Close() 163 | }() 164 | 165 | if resp.StatusCode != http.StatusOK { 166 | body, err := io.ReadAll(resp.Body) 167 | if err != nil { 168 | c.log.Printf("failed to read body: %s", err) 169 | return false 170 | } 171 | c.log.Printf("unexpected status code %d: %s", resp.StatusCode, body) 172 | return false 173 | } 174 | 175 | rawBatches := make(chan string, 100) 176 | defer close(rawBatches) 177 | c.initWorkerPool(rawBatches, es) 178 | 179 | return c.readStream(resp.Body, rawBatches) 180 | } 181 | 182 | func (c *RLPGatewayClient) readStream(r io.Reader, rawBatches chan string) bool { 183 | buf := bytes.NewBuffer(nil) 184 | reader := bufio.NewReader(r) 185 | for { 186 | line, err := reader.ReadBytes('\n') 187 | if err != nil { 188 | c.log.Printf("failed while reading stream: %s", err) 189 | return true 190 | } 191 | 192 | switch { 193 | case bytes.HasPrefix(line, []byte("heartbeat: ")): 194 | // TODO: Remove this old case 195 | continue 196 | case bytes.HasPrefix(line, []byte("event: closing")): 197 | return true 198 | case bytes.HasPrefix(line, []byte("event: heartbeat")): 199 | // Throw away the data of the heartbeat event and the next 200 | // newline. 201 | _, _ = reader.ReadBytes('\n') 202 | _, _ = reader.ReadBytes('\n') 203 | continue 204 | case bytes.HasPrefix(line, []byte("data: ")): 205 | buf.Write(line[len("data: "):]) 206 | case bytes.Equal(line, []byte("\n")): 207 | if buf.Len() == 0 { 208 | continue 209 | } 210 | rawBatches <- buf.String() 211 | buf.Reset() 212 | } 213 | } 214 | } 215 | 216 | func (c *RLPGatewayClient) initWorkerPool(rawBatches chan string, batches chan<- []*loggregator_v2.Envelope) { 217 | workerCount := 1000 218 | for i := 0; i < workerCount; i++ { 219 | go func(rawBatches chan string, es chan<- []*loggregator_v2.Envelope) { 220 | for batch := range rawBatches { 221 | var eb loggregator_v2.EnvelopeBatch 222 | if err := protojson.Unmarshal([]byte(batch), &eb); err != nil { 223 | c.log.Printf("failed to unmarshal envelope: %s", err) 224 | return 225 | } 226 | es <- eb.Batch 227 | } 228 | }(rawBatches, batches) 229 | } 230 | } 231 | 232 | func (c *RLPGatewayClient) buildQuery(req *loggregator_v2.EgressBatchRequest) string { 233 | var query []string 234 | if req.GetShardId() != "" { 235 | query = append(query, "shard_id="+req.GetShardId()) 236 | } 237 | 238 | if req.GetDeterministicName() != "" { 239 | query = append(query, "deterministic_name="+req.GetDeterministicName()) 240 | } 241 | 242 | for _, selector := range req.GetSelectors() { 243 | if selector.GetSourceId() != "" { 244 | query = append(query, "source_id="+selector.GetSourceId()) 245 | } 246 | 247 | switch selector.Message.(type) { 248 | case *loggregator_v2.Selector_Log: 249 | query = append(query, "log") 250 | case *loggregator_v2.Selector_Counter: 251 | if selector.GetCounter().GetName() != "" { 252 | query = append(query, "counter.name="+selector.GetCounter().GetName()) 253 | continue 254 | } 255 | query = append(query, "counter") 256 | case *loggregator_v2.Selector_Gauge: 257 | if len(selector.GetGauge().GetNames()) > 1 { 258 | // TODO: This is a mistake in the gateway. 259 | panic("This is not yet supported") 260 | } 261 | 262 | if len(selector.GetGauge().GetNames()) != 0 { 263 | query = append(query, "gauge.name="+selector.GetGauge().GetNames()[0]) 264 | continue 265 | } 266 | query = append(query, "gauge") 267 | case *loggregator_v2.Selector_Timer: 268 | query = append(query, "timer") 269 | case *loggregator_v2.Selector_Event: 270 | query = append(query, "event") 271 | } 272 | } 273 | 274 | namedCounter := containsPrefix(query, "counter.name") 275 | namedGauge := containsPrefix(query, "gauge.name") 276 | 277 | if namedCounter { 278 | query = filter(query, "counter") 279 | } 280 | 281 | if namedGauge { 282 | query = filter(query, "gauge") 283 | } 284 | 285 | query = removeDuplicateSourceIDs(query) 286 | if len(query) == 0 { 287 | return "" 288 | } 289 | 290 | return "?" + strings.Join(query, "&") 291 | } 292 | 293 | func removeDuplicateSourceIDs(query []string) []string { 294 | sids := map[string]bool{} 295 | duplicates := 0 296 | for i, j := 0, 0; i < len(query); i++ { 297 | if strings.HasPrefix(query[i], "source_id=") && sids[query[i]] { 298 | // Duplicate source ID 299 | duplicates++ 300 | continue 301 | } 302 | sids[query[i]] = true 303 | query[j] = query[i] 304 | j++ 305 | } 306 | 307 | return query[:len(query)-duplicates] 308 | } 309 | 310 | func containsPrefix(arr []string, prefix string) bool { 311 | for _, i := range arr { 312 | if strings.HasPrefix(i, prefix) { 313 | return true 314 | } 315 | } 316 | return false 317 | } 318 | 319 | func filter(arr []string, target string) []string { 320 | var filtered []string 321 | for _, i := range arr { 322 | if i != target { 323 | filtered = append(filtered, i) 324 | } 325 | } 326 | return filtered 327 | } 328 | -------------------------------------------------------------------------------- /conversion/tov1.go: -------------------------------------------------------------------------------- 1 | package conversion 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/hex" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | 10 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 11 | 12 | "github.com/cloudfoundry/sonde-go/events" 13 | "google.golang.org/protobuf/proto" 14 | ) 15 | 16 | // ToV1 converts v2 envelopes down to v1 envelopes. The v2 Envelope may be 17 | // mutated during the conversion and share pointers with the new v1 envelope 18 | // for efficiency in creating the v1 envelope. As a result the envelope you 19 | // pass in should no longer be used. 20 | func ToV1(e *loggregator_v2.Envelope) []*events.Envelope { 21 | var envelopes []*events.Envelope 22 | switch (e.Message).(type) { 23 | case *loggregator_v2.Envelope_Log: 24 | envelopes = convertLog(e) 25 | case *loggregator_v2.Envelope_Counter: 26 | envelopes = convertCounter(e) 27 | case *loggregator_v2.Envelope_Gauge: 28 | envelopes = convertGauge(e) 29 | case *loggregator_v2.Envelope_Timer: 30 | envelopes = convertTimer(e) 31 | } 32 | 33 | for _, v1e := range envelopes { 34 | delete(v1e.Tags, "__v1_type") 35 | delete(v1e.Tags, "origin") 36 | delete(v1e.Tags, "deployment") 37 | delete(v1e.Tags, "job") 38 | delete(v1e.Tags, "index") 39 | delete(v1e.Tags, "ip") 40 | } 41 | 42 | return envelopes 43 | } 44 | 45 | func createBaseV1(e *loggregator_v2.Envelope) *events.Envelope { 46 | v1e := &events.Envelope{ 47 | Origin: proto.String(getV2Tag(e, "origin")), 48 | Deployment: proto.String(getV2Tag(e, "deployment")), 49 | Job: proto.String(getV2Tag(e, "job")), 50 | Index: proto.String(getV2Tag(e, "index")), 51 | Timestamp: proto.Int64(e.Timestamp), 52 | Ip: proto.String(getV2Tag(e, "ip")), 53 | Tags: convertTags(e), 54 | } 55 | 56 | if e.SourceId != "" { 57 | v1e.Tags["source_id"] = e.SourceId 58 | } 59 | 60 | return v1e 61 | } 62 | 63 | func getV2Tag(e *loggregator_v2.Envelope, key string) string { 64 | if value, ok := e.GetTags()[key]; ok { 65 | return value 66 | } 67 | 68 | d := e.GetDeprecatedTags()[key] 69 | if d == nil { 70 | return "" 71 | } 72 | 73 | switch v := d.Data.(type) { 74 | case *loggregator_v2.Value_Text: 75 | return v.Text 76 | case *loggregator_v2.Value_Integer: 77 | return fmt.Sprintf("%d", v.Integer) 78 | case *loggregator_v2.Value_Decimal: 79 | return fmt.Sprintf("%f", v.Decimal) 80 | default: 81 | return "" 82 | } 83 | } 84 | 85 | func convertTimer(v2e *loggregator_v2.Envelope) []*events.Envelope { 86 | v1e := createBaseV1(v2e) 87 | timer := v2e.GetTimer() 88 | v1e.EventType = events.Envelope_HttpStartStop.Enum() 89 | instanceIndex, err := strconv.ParseInt(v2e.InstanceId, 10, 32) 90 | if err != nil { 91 | instanceIndex = 0 92 | } 93 | 94 | method := events.Method(events.Method_value[getV2Tag(v2e, "method")]) 95 | peerType := events.PeerType(events.PeerType_value[getV2Tag(v2e, "peer_type")]) 96 | 97 | v1e.HttpStartStop = &events.HttpStartStop{ 98 | StartTimestamp: proto.Int64(timer.Start), 99 | StopTimestamp: proto.Int64(timer.Stop), 100 | RequestId: convertUUID(parseUUID(getV2Tag(v2e, "request_id"))), 101 | ApplicationId: convertUUID(parseUUID(v2e.SourceId)), 102 | PeerType: &peerType, 103 | Method: &method, 104 | Uri: proto.String(getV2Tag(v2e, "uri")), 105 | RemoteAddress: proto.String(getV2Tag(v2e, "remote_address")), 106 | UserAgent: proto.String(getV2Tag(v2e, "user_agent")), 107 | StatusCode: proto.Int32(int32(atoi(getV2Tag(v2e, "status_code")))), 108 | ContentLength: proto.Int64(atoi(getV2Tag(v2e, "content_length"))), 109 | InstanceIndex: proto.Int32(int32(instanceIndex)), 110 | InstanceId: proto.String(getV2Tag(v2e, "routing_instance_id")), 111 | Forwarded: strings.Split(getV2Tag(v2e, "forwarded"), "\n"), 112 | } 113 | 114 | delete(v1e.Tags, "peer_type") 115 | delete(v1e.Tags, "method") 116 | delete(v1e.Tags, "request_id") 117 | delete(v1e.Tags, "uri") 118 | delete(v1e.Tags, "remote_address") 119 | delete(v1e.Tags, "user_agent") 120 | delete(v1e.Tags, "status_code") 121 | delete(v1e.Tags, "content_length") 122 | delete(v1e.Tags, "routing_instance_id") 123 | delete(v1e.Tags, "forwarded") 124 | 125 | return []*events.Envelope{v1e} 126 | } 127 | 128 | func atoi(s string) int64 { 129 | i, err := strconv.ParseInt(s, 10, 64) 130 | if err != nil { 131 | return 0 132 | } 133 | 134 | return i 135 | } 136 | 137 | func convertLog(v2e *loggregator_v2.Envelope) []*events.Envelope { 138 | v1e := createBaseV1(v2e) 139 | if getV2Tag(v2e, "__v1_type") == "Error" { 140 | recoverError(v1e, v2e) 141 | return []*events.Envelope{v1e} 142 | } 143 | logMessage := v2e.GetLog() 144 | v1e.EventType = events.Envelope_LogMessage.Enum() 145 | v1e.LogMessage = &events.LogMessage{ 146 | Message: logMessage.Payload, 147 | MessageType: messageType(logMessage), 148 | Timestamp: proto.Int64(v2e.Timestamp), 149 | AppId: proto.String(v2e.SourceId), 150 | SourceType: proto.String(getV2Tag(v2e, "source_type")), 151 | SourceInstance: proto.String(v2e.InstanceId), 152 | } 153 | delete(v1e.Tags, "source_type") 154 | 155 | return []*events.Envelope{v1e} 156 | } 157 | 158 | func recoverError(v1e *events.Envelope, v2e *loggregator_v2.Envelope) { 159 | logMessage := v2e.GetLog() 160 | v1e.EventType = events.Envelope_Error.Enum() 161 | code := int32(atoi(getV2Tag(v2e, "code"))) 162 | v1e.Error = &events.Error{ 163 | Source: proto.String(getV2Tag(v2e, "source")), 164 | Code: proto.Int32(code), 165 | Message: proto.String(string(logMessage.Payload)), 166 | } 167 | delete(v1e.Tags, "source") 168 | delete(v1e.Tags, "code") 169 | } 170 | 171 | func convertCounter(v2e *loggregator_v2.Envelope) []*events.Envelope { 172 | v1e := createBaseV1(v2e) 173 | counterEvent := v2e.GetCounter() 174 | v1e.EventType = events.Envelope_CounterEvent.Enum() 175 | if v2e.InstanceId != "" { 176 | v1e.GetTags()["instance_id"] = v2e.InstanceId 177 | } 178 | v1e.CounterEvent = &events.CounterEvent{ 179 | Name: proto.String(counterEvent.Name), 180 | Delta: proto.Uint64(counterEvent.GetDelta()), 181 | Total: proto.Uint64(counterEvent.GetTotal()), 182 | } 183 | 184 | return []*events.Envelope{v1e} 185 | } 186 | 187 | func convertGauge(v2e *loggregator_v2.Envelope) []*events.Envelope { 188 | if v1e := tryConvertContainerMetric(v2e); v1e != nil { 189 | return []*events.Envelope{v1e} 190 | } 191 | 192 | var results []*events.Envelope 193 | gaugeEvent := v2e.GetGauge() 194 | 195 | for key, metric := range gaugeEvent.Metrics { 196 | v1e := createBaseV1(v2e) 197 | v1e.EventType = events.Envelope_ValueMetric.Enum() 198 | unit, value, ok := extractGaugeValues(metric) 199 | if !ok { 200 | return nil 201 | } 202 | 203 | if v2e.InstanceId != "" { 204 | v1e.GetTags()["instance_id"] = v2e.InstanceId 205 | } 206 | v1e.ValueMetric = &events.ValueMetric{ 207 | Name: proto.String(key), 208 | Unit: proto.String(unit), 209 | Value: proto.Float64(value), 210 | } 211 | results = append(results, v1e) 212 | } 213 | 214 | return results 215 | } 216 | 217 | func extractGaugeValues(metric *loggregator_v2.GaugeValue) (string, float64, bool) { 218 | if metric == nil { 219 | return "", 0, false 220 | } 221 | 222 | return metric.Unit, metric.Value, true 223 | } 224 | 225 | func instanceIndex(v2e *loggregator_v2.Envelope) int32 { 226 | defaultIndex, err := strconv.ParseInt(v2e.InstanceId, 10, 32) 227 | if err != nil { 228 | defaultIndex = 0 229 | } 230 | 231 | id := v2e.GetGauge().GetMetrics()["instance_index"] 232 | if id == nil { 233 | return int32(defaultIndex) 234 | } 235 | return int32(id.Value) 236 | } 237 | 238 | func tryConvertContainerMetric(v2e *loggregator_v2.Envelope) *events.Envelope { 239 | v1e := createBaseV1(v2e) 240 | gaugeEvent := v2e.GetGauge() 241 | if len(gaugeEvent.Metrics) == 1 { 242 | return nil 243 | } 244 | 245 | required := []string{ 246 | "cpu", 247 | "memory", 248 | "disk", 249 | "memory_quota", 250 | "disk_quota", 251 | } 252 | 253 | for _, req := range required { 254 | if v, ok := gaugeEvent.Metrics[req]; !ok || v == nil || (v.Unit == "" && v.Value == 0) { 255 | return nil 256 | } 257 | } 258 | 259 | v1e.EventType = events.Envelope_ContainerMetric.Enum() 260 | v1e.ContainerMetric = &events.ContainerMetric{ 261 | ApplicationId: proto.String(v2e.SourceId), 262 | InstanceIndex: proto.Int32(instanceIndex(v2e)), 263 | CpuPercentage: proto.Float64(gaugeEvent.Metrics["cpu"].Value), 264 | MemoryBytes: proto.Uint64(uint64(gaugeEvent.Metrics["memory"].Value)), 265 | DiskBytes: proto.Uint64(uint64(gaugeEvent.Metrics["disk"].Value)), 266 | MemoryBytesQuota: proto.Uint64(uint64(gaugeEvent.Metrics["memory_quota"].Value)), 267 | DiskBytesQuota: proto.Uint64(uint64(gaugeEvent.Metrics["disk_quota"].Value)), 268 | } 269 | 270 | return v1e 271 | } 272 | 273 | func convertTags(e *loggregator_v2.Envelope) map[string]string { 274 | oldTags := make(map[string]string) 275 | for k, v := range e.Tags { 276 | oldTags[k] = v 277 | } 278 | 279 | for key, value := range e.GetDeprecatedTags() { 280 | if value == nil { 281 | continue 282 | } 283 | switch value.Data.(type) { 284 | case *loggregator_v2.Value_Text: 285 | oldTags[key] = value.GetText() 286 | case *loggregator_v2.Value_Integer: 287 | oldTags[key] = fmt.Sprintf("%d", value.GetInteger()) 288 | case *loggregator_v2.Value_Decimal: 289 | oldTags[key] = fmt.Sprintf("%f", value.GetDecimal()) 290 | } 291 | } 292 | 293 | return oldTags 294 | } 295 | 296 | func messageType(log *loggregator_v2.Log) *events.LogMessage_MessageType { 297 | if log.Type == loggregator_v2.Log_OUT { 298 | return events.LogMessage_OUT.Enum() 299 | } 300 | return events.LogMessage_ERR.Enum() 301 | } 302 | 303 | func parseUUID(id string) []byte { 304 | // e.g. b3015d69-09cd-476d-aace-ad2d824d5ab7 305 | if len(id) != 36 { 306 | return nil 307 | } 308 | h := id[:8] + id[9:13] + id[14:18] + id[19:23] + id[24:] 309 | 310 | data, err := hex.DecodeString(h) 311 | if err != nil { 312 | return nil 313 | } 314 | 315 | return data 316 | } 317 | 318 | func convertUUID(id []byte) *events.UUID { 319 | if len(id) != 16 { 320 | return nil 321 | } 322 | 323 | return &events.UUID{ 324 | Low: proto.Uint64(binary.LittleEndian.Uint64(id[:8])), 325 | High: proto.Uint64(binary.LittleEndian.Uint64(id[8:])), 326 | } 327 | } 328 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /conversion/http_test.go: -------------------------------------------------------------------------------- 1 | package conversion_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/v10/conversion" 5 | "code.cloudfoundry.org/go-loggregator/v10/rpc/loggregator_v2" 6 | 7 | "github.com/cloudfoundry/sonde-go/events" 8 | . "github.com/onsi/ginkgo/v2" 9 | . "github.com/onsi/gomega" 10 | "google.golang.org/protobuf/proto" 11 | ) 12 | 13 | var _ = Describe("HTTP", func() { 14 | Context("given a v2 envelope", func() { 15 | var ( 16 | v2Envelope *loggregator_v2.Envelope 17 | expectedV1Envelope *events.Envelope 18 | ) 19 | 20 | BeforeEach(func() { 21 | v2Envelope = &loggregator_v2.Envelope{ 22 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 23 | InstanceId: "10", 24 | Message: &loggregator_v2.Envelope_Timer{ 25 | Timer: &loggregator_v2.Timer{ 26 | Name: "http", 27 | Start: 99, 28 | Stop: 100, 29 | }, 30 | }, 31 | DeprecatedTags: map[string]*loggregator_v2.Value{ 32 | "request_id": ValueText("954f61c4-ac84-44be-9217-cdfa3117fb41"), 33 | "peer_type": ValueText("Client"), 34 | "method": ValueText("GET"), 35 | "uri": ValueText("/hello-world"), 36 | "remote_address": ValueText("10.1.1.0"), 37 | "user_agent": ValueText("Mozilla/5.0"), 38 | "status_code": ValueText("200"), 39 | "content_length": ValueText("1000000"), 40 | "routing_instance_id": ValueText("application-id"), 41 | "forwarded": ValueText("6.6.6.6\n8.8.8.8"), 42 | }, 43 | } 44 | 45 | expectedV1Envelope = &events.Envelope{ 46 | EventType: events.Envelope_HttpStartStop.Enum(), 47 | HttpStartStop: &events.HttpStartStop{ 48 | StartTimestamp: proto.Int64(99), 49 | StopTimestamp: proto.Int64(100), 50 | RequestId: &events.UUID{ 51 | Low: proto.Uint64(0xbe4484acc4614f95), 52 | High: proto.Uint64(0x41fb1731facd1792), 53 | }, 54 | ApplicationId: &events.UUID{ 55 | Low: proto.Uint64(0x6d47cd09695d01b3), 56 | High: proto.Uint64(0xb75a4d822dadceaa), 57 | }, 58 | PeerType: events.PeerType_Client.Enum(), 59 | Method: events.Method_GET.Enum(), 60 | Uri: proto.String("/hello-world"), 61 | RemoteAddress: proto.String("10.1.1.0"), 62 | UserAgent: proto.String("Mozilla/5.0"), 63 | StatusCode: proto.Int32(200), 64 | ContentLength: proto.Int64(1000000), 65 | InstanceIndex: proto.Int32(10), 66 | InstanceId: proto.String("application-id"), 67 | Forwarded: []string{"6.6.6.6", "8.8.8.8"}, 68 | }, 69 | } 70 | }) 71 | 72 | It("converts to a v1 envelope", func() { 73 | envelopes := conversion.ToV1(v2Envelope) 74 | Expect(len(envelopes)).To(Equal(1)) 75 | converted := envelopes[0] 76 | 77 | _, err := proto.Marshal(converted) 78 | Expect(err).ToNot(HaveOccurred()) 79 | 80 | Expect(proto.Equal(converted.HttpStartStop, expectedV1Envelope.HttpStartStop)).To(BeTrue()) 81 | Expect(converted.GetEventType()).To(Equal(expectedV1Envelope.GetEventType())) 82 | }) 83 | 84 | It("converts integer tag types", func() { 85 | v2Envelope.GetDeprecatedTags()["status_code"] = ValueInteger(200) 86 | 87 | envelopes := conversion.ToV1(v2Envelope) 88 | Expect(len(envelopes)).To(Equal(1)) 89 | converted := envelopes[0] 90 | 91 | _, err := proto.Marshal(converted) 92 | Expect(err).ToNot(HaveOccurred()) 93 | 94 | Expect(proto.Equal(converted.HttpStartStop, expectedV1Envelope.HttpStartStop)).To(BeTrue()) 95 | Expect(converted.GetEventType()).To(Equal(expectedV1Envelope.GetEventType())) 96 | }) 97 | }) 98 | 99 | Context("given a v1 envelope", func() { 100 | Context("with deprecated tags", func() { 101 | It("converts to a v2 envelope", func() { 102 | v1Envelope := &events.Envelope{ 103 | Origin: proto.String("some-origin"), 104 | EventType: events.Envelope_HttpStartStop.Enum(), 105 | Deployment: proto.String("some-deployment"), 106 | Job: proto.String("some-job"), 107 | Index: proto.String("some-index"), 108 | Ip: proto.String("some-ip"), 109 | HttpStartStop: &events.HttpStartStop{ 110 | StartTimestamp: proto.Int64(99), 111 | StopTimestamp: proto.Int64(100), 112 | RequestId: &events.UUID{ 113 | Low: proto.Uint64(0xbe4484acc4614f95), 114 | High: proto.Uint64(0x41fb1731facd1792), 115 | }, 116 | ApplicationId: &events.UUID{ 117 | Low: proto.Uint64(0x6d47cd09695d01b3), 118 | High: proto.Uint64(0xb75a4d822dadceaa), 119 | }, 120 | PeerType: events.PeerType_Client.Enum(), 121 | Method: events.Method_GET.Enum(), 122 | Uri: proto.String("/hello-world"), 123 | RemoteAddress: proto.String("10.1.1.0"), 124 | UserAgent: proto.String("Mozilla/5.0"), 125 | StatusCode: proto.Int32(200), 126 | ContentLength: proto.Int64(1000000), 127 | InstanceIndex: proto.Int32(10), 128 | InstanceId: proto.String("application-id"), 129 | Forwarded: []string{"6.6.6.6", "8.8.8.8"}, 130 | }, 131 | } 132 | 133 | expectedV2Envelope := &loggregator_v2.Envelope{ 134 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 135 | Message: &loggregator_v2.Envelope_Timer{ 136 | Timer: &loggregator_v2.Timer{ 137 | Name: "http", 138 | Start: 99, 139 | Stop: 100, 140 | }, 141 | }, 142 | DeprecatedTags: map[string]*loggregator_v2.Value{ 143 | "__v1_type": ValueText("HttpStartStop"), 144 | "origin": ValueText("some-origin"), 145 | "request_id": ValueText("954f61c4-ac84-44be-9217-cdfa3117fb41"), 146 | "peer_type": ValueText("Client"), 147 | "method": ValueText("GET"), 148 | "uri": ValueText("/hello-world"), 149 | "remote_address": ValueText("10.1.1.0"), 150 | "user_agent": ValueText("Mozilla/5.0"), 151 | "status_code": ValueText("200"), 152 | "content_length": ValueText("1000000"), 153 | "routing_instance_id": ValueText("application-id"), 154 | "forwarded": ValueText("6.6.6.6\n8.8.8.8"), 155 | "deployment": ValueText("some-deployment"), 156 | "job": ValueText("some-job"), 157 | "index": ValueText("some-index"), 158 | "ip": ValueText("some-ip"), 159 | }, 160 | } 161 | 162 | converted := conversion.ToV2(v1Envelope, false) 163 | 164 | _, err := proto.Marshal(converted) 165 | Expect(err).ToNot(HaveOccurred()) 166 | 167 | for k, v := range expectedV2Envelope.DeprecatedTags { 168 | Expect(proto.Equal(converted.DeprecatedTags[k], v)).To(BeTrue()) 169 | } 170 | 171 | Expect(converted.GetSourceId()).To(Equal(expectedV2Envelope.SourceId)) 172 | Expect(converted.GetTimer().GetName()).To(Equal(expectedV2Envelope.GetTimer().GetName())) 173 | Expect(converted.GetTimer().GetStart()).To(Equal(expectedV2Envelope.GetTimer().GetStart())) 174 | Expect(converted.GetTimer().GetStop()).To(Equal(expectedV2Envelope.GetTimer().GetStop())) 175 | }) 176 | 177 | It("sets the source ID to deployment/job when App ID is missing", func() { 178 | v1Envelope := &events.Envelope{ 179 | Deployment: proto.String("some-deployment"), 180 | Job: proto.String("some-job"), 181 | } 182 | 183 | expectedV2Envelope := &loggregator_v2.Envelope{ 184 | SourceId: "some-deployment/some-job", 185 | } 186 | 187 | converted := conversion.ToV2(v1Envelope, false) 188 | 189 | Expect(converted.SourceId).To(Equal(expectedV2Envelope.SourceId)) 190 | }) 191 | }) 192 | 193 | Context("using preferred tags", func() { 194 | It("returns a v2 envelope", func() { 195 | v1Envelope := &events.Envelope{ 196 | Origin: proto.String("some-origin"), 197 | EventType: events.Envelope_HttpStartStop.Enum(), 198 | Deployment: proto.String("some-deployment"), 199 | Job: proto.String("some-job"), 200 | Index: proto.String("some-index"), 201 | Ip: proto.String("some-ip"), 202 | HttpStartStop: &events.HttpStartStop{ 203 | StartTimestamp: proto.Int64(99), 204 | StopTimestamp: proto.Int64(100), 205 | RequestId: &events.UUID{ 206 | Low: proto.Uint64(0xbe4484acc4614f95), 207 | High: proto.Uint64(0x41fb1731facd1792), 208 | }, 209 | ApplicationId: &events.UUID{ 210 | Low: proto.Uint64(0x6d47cd09695d01b3), 211 | High: proto.Uint64(0xb75a4d822dadceaa), 212 | }, 213 | PeerType: events.PeerType_Client.Enum(), 214 | Method: events.Method_GET.Enum(), 215 | Uri: proto.String("/hello-world"), 216 | RemoteAddress: proto.String("10.1.1.0"), 217 | UserAgent: proto.String("Mozilla/5.0"), 218 | StatusCode: proto.Int32(200), 219 | ContentLength: proto.Int64(1000000), 220 | InstanceIndex: proto.Int32(10), 221 | InstanceId: proto.String("application-id"), 222 | Forwarded: []string{"6.6.6.6", "8.8.8.8"}, 223 | }, 224 | } 225 | 226 | expectedV2Envelope := &loggregator_v2.Envelope{ 227 | SourceId: "b3015d69-09cd-476d-aace-ad2d824d5ab7", 228 | Message: &loggregator_v2.Envelope_Timer{ 229 | Timer: &loggregator_v2.Timer{ 230 | Name: "http", 231 | Start: 99, 232 | Stop: 100, 233 | }, 234 | }, 235 | Tags: map[string]string{ 236 | "__v1_type": "HttpStartStop", 237 | "origin": "some-origin", 238 | "request_id": "954f61c4-ac84-44be-9217-cdfa3117fb41", 239 | "peer_type": "Client", 240 | "method": "GET", 241 | "uri": "/hello-world", 242 | "remote_address": "10.1.1.0", 243 | "user_agent": "Mozilla/5.0", 244 | "status_code": "200", 245 | "content_length": "1000000", 246 | "instance_index": "10", 247 | "routing_instance_id": "application-id", 248 | "forwarded": "6.6.6.6\n8.8.8.8", 249 | "deployment": "some-deployment", 250 | "job": "some-job", 251 | "index": "some-index", 252 | "ip": "some-ip", 253 | }, 254 | } 255 | 256 | converted := conversion.ToV2(v1Envelope, true) 257 | 258 | _, err := proto.Marshal(converted) 259 | Expect(err).ToNot(HaveOccurred()) 260 | 261 | for k, v := range expectedV2Envelope.DeprecatedTags { 262 | Expect(converted.DeprecatedTags).To(HaveKeyWithValue(k, v)) 263 | } 264 | 265 | Expect(converted.GetSourceId()).To(Equal(expectedV2Envelope.SourceId)) 266 | Expect(converted.GetTimer().GetName()).To(Equal(expectedV2Envelope.GetTimer().GetName())) 267 | Expect(converted.GetTimer().GetStart()).To(Equal(expectedV2Envelope.GetTimer().GetStart())) 268 | Expect(converted.GetTimer().GetStop()).To(Equal(expectedV2Envelope.GetTimer().GetStop())) 269 | }) 270 | }) 271 | }) 272 | }) 273 | -------------------------------------------------------------------------------- /rpc/loggregator_v2/ingress.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.34.1 4 | // protoc v5.26.1 5 | // source: loggregator-api/v2/ingress.proto 6 | 7 | package loggregator_v2 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type IngressResponse struct { 24 | state protoimpl.MessageState 25 | sizeCache protoimpl.SizeCache 26 | unknownFields protoimpl.UnknownFields 27 | } 28 | 29 | func (x *IngressResponse) Reset() { 30 | *x = IngressResponse{} 31 | if protoimpl.UnsafeEnabled { 32 | mi := &file_loggregator_api_v2_ingress_proto_msgTypes[0] 33 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 34 | ms.StoreMessageInfo(mi) 35 | } 36 | } 37 | 38 | func (x *IngressResponse) String() string { 39 | return protoimpl.X.MessageStringOf(x) 40 | } 41 | 42 | func (*IngressResponse) ProtoMessage() {} 43 | 44 | func (x *IngressResponse) ProtoReflect() protoreflect.Message { 45 | mi := &file_loggregator_api_v2_ingress_proto_msgTypes[0] 46 | if protoimpl.UnsafeEnabled && x != nil { 47 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 48 | if ms.LoadMessageInfo() == nil { 49 | ms.StoreMessageInfo(mi) 50 | } 51 | return ms 52 | } 53 | return mi.MessageOf(x) 54 | } 55 | 56 | // Deprecated: Use IngressResponse.ProtoReflect.Descriptor instead. 57 | func (*IngressResponse) Descriptor() ([]byte, []int) { 58 | return file_loggregator_api_v2_ingress_proto_rawDescGZIP(), []int{0} 59 | } 60 | 61 | type BatchSenderResponse struct { 62 | state protoimpl.MessageState 63 | sizeCache protoimpl.SizeCache 64 | unknownFields protoimpl.UnknownFields 65 | } 66 | 67 | func (x *BatchSenderResponse) Reset() { 68 | *x = BatchSenderResponse{} 69 | if protoimpl.UnsafeEnabled { 70 | mi := &file_loggregator_api_v2_ingress_proto_msgTypes[1] 71 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 72 | ms.StoreMessageInfo(mi) 73 | } 74 | } 75 | 76 | func (x *BatchSenderResponse) String() string { 77 | return protoimpl.X.MessageStringOf(x) 78 | } 79 | 80 | func (*BatchSenderResponse) ProtoMessage() {} 81 | 82 | func (x *BatchSenderResponse) ProtoReflect() protoreflect.Message { 83 | mi := &file_loggregator_api_v2_ingress_proto_msgTypes[1] 84 | if protoimpl.UnsafeEnabled && x != nil { 85 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 86 | if ms.LoadMessageInfo() == nil { 87 | ms.StoreMessageInfo(mi) 88 | } 89 | return ms 90 | } 91 | return mi.MessageOf(x) 92 | } 93 | 94 | // Deprecated: Use BatchSenderResponse.ProtoReflect.Descriptor instead. 95 | func (*BatchSenderResponse) Descriptor() ([]byte, []int) { 96 | return file_loggregator_api_v2_ingress_proto_rawDescGZIP(), []int{1} 97 | } 98 | 99 | type SendResponse struct { 100 | state protoimpl.MessageState 101 | sizeCache protoimpl.SizeCache 102 | unknownFields protoimpl.UnknownFields 103 | } 104 | 105 | func (x *SendResponse) Reset() { 106 | *x = SendResponse{} 107 | if protoimpl.UnsafeEnabled { 108 | mi := &file_loggregator_api_v2_ingress_proto_msgTypes[2] 109 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 110 | ms.StoreMessageInfo(mi) 111 | } 112 | } 113 | 114 | func (x *SendResponse) String() string { 115 | return protoimpl.X.MessageStringOf(x) 116 | } 117 | 118 | func (*SendResponse) ProtoMessage() {} 119 | 120 | func (x *SendResponse) ProtoReflect() protoreflect.Message { 121 | mi := &file_loggregator_api_v2_ingress_proto_msgTypes[2] 122 | if protoimpl.UnsafeEnabled && x != nil { 123 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 124 | if ms.LoadMessageInfo() == nil { 125 | ms.StoreMessageInfo(mi) 126 | } 127 | return ms 128 | } 129 | return mi.MessageOf(x) 130 | } 131 | 132 | // Deprecated: Use SendResponse.ProtoReflect.Descriptor instead. 133 | func (*SendResponse) Descriptor() ([]byte, []int) { 134 | return file_loggregator_api_v2_ingress_proto_rawDescGZIP(), []int{2} 135 | } 136 | 137 | var File_loggregator_api_v2_ingress_proto protoreflect.FileDescriptor 138 | 139 | var file_loggregator_api_v2_ingress_proto_rawDesc = []byte{ 140 | 0x0a, 0x20, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2d, 0x61, 0x70, 141 | 0x69, 0x2f, 0x76, 0x32, 0x2f, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x70, 0x72, 0x6f, 142 | 0x74, 0x6f, 0x12, 0x0e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2e, 143 | 0x76, 0x32, 0x1a, 0x21, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2d, 144 | 0x61, 0x70, 0x69, 0x2f, 0x76, 0x32, 0x2f, 0x65, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x2e, 145 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x11, 0x0a, 0x0f, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 146 | 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x15, 0x0a, 0x13, 0x42, 0x61, 0x74, 0x63, 147 | 0x68, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 148 | 0x0e, 0x0a, 0x0c, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 149 | 0xf0, 0x01, 0x0a, 0x07, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x47, 0x0a, 0x06, 0x53, 150 | 0x65, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 151 | 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x1a, 152 | 0x1f, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x76, 0x32, 153 | 0x2e, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 154 | 0x22, 0x00, 0x28, 0x01, 0x12, 0x55, 0x0a, 0x0b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x53, 0x65, 0x6e, 155 | 0x64, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 156 | 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x42, 0x61, 0x74, 157 | 0x63, 0x68, 0x1a, 0x23, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 158 | 0x2e, 0x76, 0x32, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x53, 0x65, 0x6e, 0x64, 0x65, 0x72, 0x52, 159 | 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x45, 0x0a, 0x04, 0x53, 160 | 0x65, 0x6e, 0x64, 0x12, 0x1d, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 161 | 0x72, 0x2e, 0x76, 0x32, 0x2e, 0x45, 0x6e, 0x76, 0x65, 0x6c, 0x6f, 0x70, 0x65, 0x42, 0x61, 0x74, 162 | 0x63, 0x68, 0x1a, 0x1c, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 163 | 0x2e, 0x76, 0x32, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 164 | 0x22, 0x00, 0x42, 0x71, 0x0a, 0x1f, 0x6f, 0x72, 0x67, 0x2e, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x66, 165 | 0x6f, 0x75, 0x6e, 0x64, 0x72, 0x79, 0x2e, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 166 | 0x6f, 0x72, 0x2e, 0x76, 0x32, 0x42, 0x12, 0x4c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 167 | 0x6f, 0x72, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5a, 0x3a, 0x63, 0x6f, 0x64, 0x65, 0x2e, 168 | 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x72, 0x79, 0x2e, 0x6f, 0x72, 0x67, 169 | 0x2f, 0x67, 0x6f, 0x2d, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2f, 170 | 0x76, 0x39, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 171 | 0x6f, 0x72, 0x5f, 0x76, 0x32, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 172 | } 173 | 174 | var ( 175 | file_loggregator_api_v2_ingress_proto_rawDescOnce sync.Once 176 | file_loggregator_api_v2_ingress_proto_rawDescData = file_loggregator_api_v2_ingress_proto_rawDesc 177 | ) 178 | 179 | func file_loggregator_api_v2_ingress_proto_rawDescGZIP() []byte { 180 | file_loggregator_api_v2_ingress_proto_rawDescOnce.Do(func() { 181 | file_loggregator_api_v2_ingress_proto_rawDescData = protoimpl.X.CompressGZIP(file_loggregator_api_v2_ingress_proto_rawDescData) 182 | }) 183 | return file_loggregator_api_v2_ingress_proto_rawDescData 184 | } 185 | 186 | var file_loggregator_api_v2_ingress_proto_msgTypes = make([]protoimpl.MessageInfo, 3) 187 | var file_loggregator_api_v2_ingress_proto_goTypes = []interface{}{ 188 | (*IngressResponse)(nil), // 0: loggregator.v2.IngressResponse 189 | (*BatchSenderResponse)(nil), // 1: loggregator.v2.BatchSenderResponse 190 | (*SendResponse)(nil), // 2: loggregator.v2.SendResponse 191 | (*Envelope)(nil), // 3: loggregator.v2.Envelope 192 | (*EnvelopeBatch)(nil), // 4: loggregator.v2.EnvelopeBatch 193 | } 194 | var file_loggregator_api_v2_ingress_proto_depIdxs = []int32{ 195 | 3, // 0: loggregator.v2.Ingress.Sender:input_type -> loggregator.v2.Envelope 196 | 4, // 1: loggregator.v2.Ingress.BatchSender:input_type -> loggregator.v2.EnvelopeBatch 197 | 4, // 2: loggregator.v2.Ingress.Send:input_type -> loggregator.v2.EnvelopeBatch 198 | 0, // 3: loggregator.v2.Ingress.Sender:output_type -> loggregator.v2.IngressResponse 199 | 1, // 4: loggregator.v2.Ingress.BatchSender:output_type -> loggregator.v2.BatchSenderResponse 200 | 2, // 5: loggregator.v2.Ingress.Send:output_type -> loggregator.v2.SendResponse 201 | 3, // [3:6] is the sub-list for method output_type 202 | 0, // [0:3] is the sub-list for method input_type 203 | 0, // [0:0] is the sub-list for extension type_name 204 | 0, // [0:0] is the sub-list for extension extendee 205 | 0, // [0:0] is the sub-list for field type_name 206 | } 207 | 208 | func init() { file_loggregator_api_v2_ingress_proto_init() } 209 | func file_loggregator_api_v2_ingress_proto_init() { 210 | if File_loggregator_api_v2_ingress_proto != nil { 211 | return 212 | } 213 | file_loggregator_api_v2_envelope_proto_init() 214 | if !protoimpl.UnsafeEnabled { 215 | file_loggregator_api_v2_ingress_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 216 | switch v := v.(*IngressResponse); i { 217 | case 0: 218 | return &v.state 219 | case 1: 220 | return &v.sizeCache 221 | case 2: 222 | return &v.unknownFields 223 | default: 224 | return nil 225 | } 226 | } 227 | file_loggregator_api_v2_ingress_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 228 | switch v := v.(*BatchSenderResponse); i { 229 | case 0: 230 | return &v.state 231 | case 1: 232 | return &v.sizeCache 233 | case 2: 234 | return &v.unknownFields 235 | default: 236 | return nil 237 | } 238 | } 239 | file_loggregator_api_v2_ingress_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 240 | switch v := v.(*SendResponse); i { 241 | case 0: 242 | return &v.state 243 | case 1: 244 | return &v.sizeCache 245 | case 2: 246 | return &v.unknownFields 247 | default: 248 | return nil 249 | } 250 | } 251 | } 252 | type x struct{} 253 | out := protoimpl.TypeBuilder{ 254 | File: protoimpl.DescBuilder{ 255 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 256 | RawDescriptor: file_loggregator_api_v2_ingress_proto_rawDesc, 257 | NumEnums: 0, 258 | NumMessages: 3, 259 | NumExtensions: 0, 260 | NumServices: 1, 261 | }, 262 | GoTypes: file_loggregator_api_v2_ingress_proto_goTypes, 263 | DependencyIndexes: file_loggregator_api_v2_ingress_proto_depIdxs, 264 | MessageInfos: file_loggregator_api_v2_ingress_proto_msgTypes, 265 | }.Build() 266 | File_loggregator_api_v2_ingress_proto = out.File 267 | file_loggregator_api_v2_ingress_proto_rawDesc = nil 268 | file_loggregator_api_v2_ingress_proto_goTypes = nil 269 | file_loggregator_api_v2_ingress_proto_depIdxs = nil 270 | } 271 | --------------------------------------------------------------------------------