├── .gitattributes ├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── api └── v1 │ ├── egress.proto │ ├── ingress.proto │ ├── orchestration.proto │ └── promql.proto ├── cmd ├── blackbox │ ├── .gitignore │ ├── config.go │ └── main.go ├── cf-auth-proxy │ ├── .gitignore │ ├── config.go │ └── main.go ├── gateway │ ├── .gitignore │ ├── config.go │ └── main.go ├── log-cache │ ├── .gitignore │ ├── config.go │ └── main.go ├── nozzle │ ├── .gitignore │ ├── config.go │ └── main.go ├── scheduler │ ├── .gitignore │ ├── config.go │ └── main.go └── syslog-server │ ├── .gitignore │ ├── config.go │ └── main.go ├── internal ├── auth │ ├── access_log.go │ ├── access_log_test.go │ ├── access_logger.go │ ├── access_logger_test.go │ ├── access_middleware.go │ ├── access_middleware_test.go │ ├── auth_suite_test.go │ ├── capi_client.go │ ├── capi_client_test.go │ ├── cf_auth_middleware.go │ ├── cf_auth_middleware_test.go │ ├── uaa_client.go │ └── uaa_client_test.go ├── blackbox │ ├── blackbox.go │ ├── blackbox_suite_test.go │ └── blackbox_test.go ├── cache │ ├── log_cache.go │ ├── log_cache_suite_test.go │ ├── log_cache_test.go │ ├── memory_analyzer.go │ └── store │ │ ├── prune_consultant.go │ │ ├── prune_consultant_test.go │ │ ├── store.go │ │ ├── store_benchmark_test.go │ │ ├── store_load_test.go │ │ ├── store_suite_test.go │ │ └── store_test.go ├── cfauthproxy │ ├── cf_auth_proxy.go │ ├── cf_auth_proxy_test.go │ └── proxy_suite_test.go ├── end2end │ ├── end2end_suite_test.go │ └── log_cache_test.go ├── gateway │ ├── gateway.go │ ├── gateway_suite_test.go │ └── gateway_test.go ├── matchers │ └── contain_metric.go ├── nozzle │ ├── nozzle.go │ ├── nozzle_suite_test.go │ └── nozzle_test.go ├── promql │ ├── data_reader │ │ ├── data_reader_suite_test.go │ │ ├── walking_data_reader.go │ │ └── walking_data_reader_test.go │ ├── parsing.go │ ├── parsing_test.go │ ├── promql.go │ ├── promql_suite_test.go │ └── promql_test.go ├── routing │ ├── batched_ingress_client.go │ ├── batched_ingress_client_test.go │ ├── egress_reverse_proxy.go │ ├── egress_reverse_proxy_test.go │ ├── ingress_reverse_proxy.go │ ├── ingress_reverse_proxy_test.go │ ├── local_store_reader.go │ ├── local_store_reader_test.go │ ├── orchestrator.go │ ├── orchestrator_test.go │ ├── routing_suite_test.go │ ├── routing_table.go │ ├── routing_table_test.go │ ├── static_lookup.go │ └── static_lookup_test.go ├── scheduler │ ├── scheduler.go │ ├── scheduler_suite_test.go │ └── scheduler_test.go ├── syslog │ ├── server.go │ ├── server_test.go │ └── syslog_suite_test.go ├── testing │ ├── acceptance.go │ ├── auth.go │ ├── bindata.go │ ├── certificates.go │ ├── spy_log_cache.go │ ├── spy_metrics.go │ ├── time.go │ └── tls_config.go └── tls │ └── tls.go └── pkg ├── client ├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── client.go ├── client_suite_test.go ├── client_test.go ├── examples │ ├── cf_user_auth │ │ └── main.go │ ├── promql │ │ ├── .gitignore │ │ └── main.go │ ├── sliding_window │ │ ├── .gitignore │ │ └── main.go │ └── walker │ │ ├── .gitignore │ │ └── main.go ├── oauth2_http_client.go ├── oauth2_http_client_test.go ├── walk.go ├── walk_test.go ├── window.go └── window_test.go ├── marshaler ├── marshaler.go ├── marshaler_test.go └── promql_suite_test.go └── rpc └── logcache_v1 ├── doc.go ├── egress.pb.go ├── egress.pb.gw.go ├── generate.sh ├── ingress.pb.go ├── orchestration.pb.go ├── promql.pb.go └── promql.pb.gw.go /.gitattributes: -------------------------------------------------------------------------------- 1 | *.go linguist-language=golang 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.test 2 | .idea/ 3 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 2 | 3 | This project is licensed to you under the Apache License, Version 2.0 (the "License"). 4 | You may not use this project except in compliance with the License. 5 | 6 | This project may include a number of subcomponents with separate copyright notices 7 | and license terms. Your use of these subcomponents is subject to the terms and 8 | conditions of the subcomponent's license, as noted in the LICENSE file. 9 | -------------------------------------------------------------------------------- /api/v1/egress.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package logcache.v1; 4 | 5 | import "v2/envelope.proto"; 6 | import "google/api/annotations.proto"; 7 | 8 | // The egress service is used to read data from the LogCache system. 9 | service Egress { 10 | rpc Read(ReadRequest) returns (ReadResponse) { 11 | option (google.api.http) = { 12 | get: "/api/v1/read/{source_id=**}" 13 | }; 14 | } 15 | rpc Meta(MetaRequest) returns (MetaResponse){ 16 | option (google.api.http) = { 17 | get: "/api/v1/meta" 18 | }; 19 | } 20 | } 21 | 22 | message ReadRequest { 23 | string source_id = 1; 24 | int64 start_time = 2; 25 | int64 end_time = 3; 26 | int64 limit = 4; 27 | repeated EnvelopeType envelope_types = 5; 28 | bool descending = 6; 29 | string name_filter = 7; 30 | } 31 | 32 | enum EnvelopeType { 33 | ANY = 0; 34 | LOG = 1; 35 | COUNTER = 2; 36 | GAUGE = 3; 37 | TIMER = 4; 38 | EVENT = 5; 39 | } 40 | 41 | message ReadResponse { 42 | loggregator.v2.EnvelopeBatch envelopes = 1; 43 | } 44 | 45 | message MetaRequest { 46 | bool local_only = 1; 47 | } 48 | 49 | message MetaResponse { 50 | map meta = 1; 51 | } 52 | 53 | message MetaInfo { 54 | int64 count = 1; 55 | int64 expired = 2; 56 | int64 oldest_timestamp = 3; 57 | int64 newest_timestamp = 4; 58 | } 59 | -------------------------------------------------------------------------------- /api/v1/ingress.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package logcache.v1; 4 | 5 | import "v2/envelope.proto"; 6 | 7 | // The ingress service is used to write data into the LogCache system. 8 | service Ingress { 9 | // Send is used to emit Envelopes batches into LogCache. The RPC function 10 | // will not return until the data has been stored. 11 | rpc Send(SendRequest) returns (SendResponse) {} 12 | } 13 | 14 | message SendRequest { 15 | loggregator.v2.EnvelopeBatch envelopes = 1; 16 | bool local_only = 2; 17 | } 18 | 19 | message SendResponse {} 20 | -------------------------------------------------------------------------------- /api/v1/orchestration.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package logcache.v1; 4 | 5 | service Orchestration { 6 | rpc AddRange(AddRangeRequest) returns (AddRangeResponse) {} 7 | rpc RemoveRange(RemoveRangeRequest) returns (RemoveRangeResponse) {} 8 | rpc ListRanges(ListRangesRequest) returns (ListRangesResponse) {} 9 | rpc SetRanges(SetRangesRequest) returns (SetRangesResponse) {} 10 | } 11 | 12 | message Range { 13 | // start is the first hash within the given range. [start..end] 14 | uint64 start = 1; 15 | 16 | // end is the last hash within the given range. [start..end] 17 | uint64 end = 2; 18 | } 19 | 20 | message Ranges { 21 | repeated Range ranges = 1; 22 | } 23 | 24 | message AddRangeRequest { 25 | Range range = 1; 26 | } 27 | 28 | message AddRangeResponse { 29 | } 30 | 31 | message RemoveRangeRequest { 32 | Range range = 1; 33 | } 34 | 35 | message RemoveRangeResponse { 36 | } 37 | 38 | message ListRangesRequest { 39 | } 40 | 41 | message ListRangesResponse { 42 | repeated Range ranges = 1; 43 | } 44 | 45 | message SetRangesRequest { 46 | // The key is the address of the Log Cache node. 47 | map ranges = 1; 48 | } 49 | 50 | message SetRangesResponse { 51 | } 52 | -------------------------------------------------------------------------------- /api/v1/promql.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package logcache.v1; 4 | 5 | import "google/api/annotations.proto"; 6 | 7 | service PromQLQuerier { 8 | rpc InstantQuery(PromQL.InstantQueryRequest) returns (PromQL.InstantQueryResult){ 9 | option (google.api.http) = { 10 | get: "/api/v1/query" 11 | }; 12 | } 13 | 14 | rpc RangeQuery(PromQL.RangeQueryRequest) returns (PromQL.RangeQueryResult){ 15 | option (google.api.http) = { 16 | get: "/api/v1/query_range" 17 | }; 18 | } 19 | } 20 | 21 | message PromQL { 22 | message InstantQueryRequest { 23 | string query = 1; 24 | string time = 2; 25 | } 26 | 27 | message RangeQueryRequest { 28 | string query = 1; 29 | string start = 2; 30 | string end = 3; 31 | string step = 4; 32 | } 33 | 34 | message InstantQueryResult { 35 | oneof Result { 36 | Scalar scalar = 1; 37 | Vector vector = 2; 38 | Matrix matrix = 3; 39 | } 40 | } 41 | 42 | message RangeQueryResult { 43 | oneof Result { 44 | Matrix matrix = 1; 45 | } 46 | } 47 | 48 | message Scalar { 49 | string time = 1; 50 | double value = 2; 51 | } 52 | 53 | message Vector { 54 | repeated Sample samples = 1; 55 | } 56 | 57 | message Point { 58 | string time = 1; 59 | double value = 2; 60 | } 61 | 62 | message Sample { 63 | map metric = 1; 64 | Point point = 2; 65 | } 66 | 67 | message Matrix { 68 | repeated Series series = 1; 69 | } 70 | 71 | message Series { 72 | map metric = 1; 73 | repeated Point points = 2; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /cmd/blackbox/.gitignore: -------------------------------------------------------------------------------- 1 | blackbox-test 2 | -------------------------------------------------------------------------------- /cmd/blackbox/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | envstruct "code.cloudfoundry.org/go-envstruct" 7 | "code.cloudfoundry.org/log-cache/internal/tls" 8 | ) 9 | 10 | type Config struct { 11 | EmissionInterval time.Duration `env:"EMISSION_INTERVAL, required, report"` 12 | SampleInterval time.Duration `env:"SAMPLE_INTERVAL, required, report"` 13 | WindowInterval time.Duration `env:"WINDOW_INTERVAL, required, report"` 14 | WindowLag time.Duration `env:"WINDOW_LAG, required, report"` 15 | SourceId string `env:"SOURCE_ID, required, report"` 16 | 17 | CfBlackboxEnabled bool `env:"CF_BLACKBOX_ENABLED, report"` 18 | DataSourceHTTPAddr string `env:"DATA_SOURCE_HTTP_ADDR, report"` 19 | UaaAddr string `env:"UAA_ADDR, report"` 20 | ClientID string `env:"CLIENT_ID, report"` 21 | ClientSecret string `env:"CLIENT_SECRET"` 22 | SkipTLSVerify bool `env:"SKIP_TLS_VERIFY, report"` 23 | 24 | DataSourceGrpcAddr string `env:"DATA_SOURCE_GRPC_ADDR, report"` 25 | TLS tls.TLS 26 | } 27 | 28 | func LoadConfig() (*Config, error) { 29 | c := Config{ 30 | DataSourceGrpcAddr: "localhost:8080", 31 | } 32 | 33 | if err := envstruct.Load(&c); err != nil { 34 | return nil, err 35 | } 36 | 37 | envstruct.WriteReport(&c) 38 | 39 | return &c, nil 40 | } 41 | -------------------------------------------------------------------------------- /cmd/blackbox/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | 8 | "code.cloudfoundry.org/log-cache/internal/blackbox" 9 | "google.golang.org/grpc" 10 | "google.golang.org/grpc/naming" 11 | ) 12 | 13 | func main() { 14 | infoLogger := log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds) 15 | errorLogger := log.New(os.Stderr, "", log.LstdFlags|log.Lmicroseconds) 16 | 17 | cfg, err := LoadConfig() 18 | if err != nil { 19 | log.Fatalf("failed to load configuration: %s", err) 20 | } 21 | 22 | resolver, _ := naming.NewDNSResolverWithFreq(1 * time.Minute) 23 | 24 | ingressClient := blackbox.NewIngressClient( 25 | cfg.DataSourceGrpcAddr, 26 | grpc.WithTransportCredentials(cfg.TLS.Credentials("log-cache")), 27 | grpc.WithBalancer( 28 | grpc.RoundRobin(resolver), 29 | ), 30 | ) 31 | 32 | go blackbox.StartEmittingTestMetrics(cfg.SourceId, cfg.EmissionInterval, ingressClient) 33 | 34 | grpcEgressClient := blackbox.NewGrpcEgressClient( 35 | cfg.DataSourceGrpcAddr, 36 | grpc.WithTransportCredentials(cfg.TLS.Credentials("log-cache")), 37 | grpc.WithBalancer( 38 | grpc.RoundRobin(resolver), 39 | ), 40 | ) 41 | 42 | var httpEgressClient blackbox.QueryableClient 43 | 44 | if cfg.CfBlackboxEnabled { 45 | httpEgressClient = blackbox.NewHttpEgressClient( 46 | cfg.DataSourceHTTPAddr, 47 | cfg.UaaAddr, 48 | cfg.ClientID, 49 | cfg.ClientSecret, 50 | cfg.SkipTLSVerify, 51 | ) 52 | } 53 | 54 | t := time.NewTicker(cfg.SampleInterval) 55 | rc := blackbox.ReliabilityCalculator{ 56 | SampleInterval: cfg.SampleInterval, 57 | WindowInterval: cfg.WindowInterval, 58 | WindowLag: cfg.WindowLag, 59 | EmissionInterval: cfg.EmissionInterval, 60 | SourceId: cfg.SourceId, 61 | InfoLogger: infoLogger, 62 | ErrorLogger: errorLogger, 63 | } 64 | 65 | for range t.C { 66 | reliabilityMetrics := make(map[string]float64) 67 | 68 | infoLogger.Println("Querying for gRPC reliability metric...") 69 | grpcReliability, err := rc.Calculate(grpcEgressClient) 70 | if err == nil { 71 | reliabilityMetrics["blackbox.grpc_reliability"] = grpcReliability 72 | } 73 | 74 | if cfg.CfBlackboxEnabled { 75 | infoLogger.Println("Querying for HTTP reliability metric...") 76 | httpReliability, err := rc.Calculate(httpEgressClient) 77 | if err == nil { 78 | reliabilityMetrics["blackbox.http_reliability"] = httpReliability 79 | } 80 | } 81 | 82 | blackbox.EmitMeasuredMetrics(cfg.SourceId, ingressClient, httpEgressClient, reliabilityMetrics) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /cmd/cf-auth-proxy/.gitignore: -------------------------------------------------------------------------------- 1 | cf-auth-proxy 2 | -------------------------------------------------------------------------------- /cmd/cf-auth-proxy/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | envstruct "code.cloudfoundry.org/go-envstruct" 7 | ) 8 | 9 | type CAPI struct { 10 | Addr string `env:"CAPI_ADDR, required, report"` 11 | CAPath string `env:"CAPI_CA_PATH, required, report"` 12 | CommonName string `env:"CAPI_COMMON_NAME, required, report"` 13 | } 14 | 15 | type UAA struct { 16 | ClientID string `env:"UAA_CLIENT_ID, required"` 17 | ClientSecret string `env:"UAA_CLIENT_SECRET, required"` 18 | Addr string `env:"UAA_ADDR, required, report"` 19 | CAPath string `env:"UAA_CA_PATH, required, report"` 20 | } 21 | 22 | type Config struct { 23 | LogCacheGatewayAddr string `env:"LOG_CACHE_GATEWAY_ADDR, required, report"` 24 | Addr string `env:"ADDR, required, report"` 25 | InternalIP string `env:"INTERNAL_IP, report"` 26 | HealthPort int `env:"HEALTH_PORT, report"` 27 | CertPath string `env:"EXTERNAL_CERT, required, report"` 28 | KeyPath string `env:"EXTERNAL_KEY, required, report"` 29 | SkipCertVerify bool `env:"SKIP_CERT_VERIFY, report"` 30 | ProxyCAPath string `env:"PROXY_CA_PATH, required, report"` 31 | SecurityEventLog string `env:"SECURITY_EVENT_LOG, report"` 32 | TokenPruningInterval time.Duration `env:"TOKEN_PRUNING_INTERVAL, report"` 33 | CacheExpirationInterval time.Duration `env:"CACHE_EXPIRATION_INTERVAL, report"` 34 | 35 | CAPI CAPI 36 | UAA UAA 37 | } 38 | 39 | func LoadConfig() (*Config, error) { 40 | cfg := Config{ 41 | SkipCertVerify: false, 42 | Addr: ":8083", 43 | InternalIP: "0.0.0.0", 44 | HealthPort: 6065, 45 | LogCacheGatewayAddr: "localhost:8081", 46 | TokenPruningInterval: time.Minute, 47 | CacheExpirationInterval: time.Minute, 48 | } 49 | 50 | err := envstruct.Load(&cfg) 51 | if err != nil { 52 | return nil, err 53 | } 54 | 55 | return &cfg, nil 56 | } 57 | -------------------------------------------------------------------------------- /cmd/cf-auth-proxy/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "net" 9 | "net/http" 10 | _ "net/http/pprof" 11 | "net/url" 12 | "os" 13 | "time" 14 | 15 | "crypto/x509" 16 | 17 | envstruct "code.cloudfoundry.org/go-envstruct" 18 | "code.cloudfoundry.org/log-cache/internal/auth" 19 | . "code.cloudfoundry.org/log-cache/internal/cfauthproxy" 20 | "code.cloudfoundry.org/log-cache/internal/promql" 21 | sharedtls "code.cloudfoundry.org/log-cache/internal/tls" 22 | "code.cloudfoundry.org/log-cache/pkg/client" 23 | ) 24 | 25 | func main() { 26 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 27 | 28 | loggr := log.New(os.Stderr, "", log.LstdFlags) 29 | loggr.Print("Starting Log Cache CF Auth Reverse Proxy...") 30 | defer loggr.Print("Closing Log Cache CF Auth Reverse Proxy.") 31 | 32 | cfg, err := LoadConfig() 33 | if err != nil { 34 | loggr.Fatalf("failed to load config: %s", err) 35 | } 36 | envstruct.WriteReport(cfg) 37 | 38 | metrics := metrics.NewRegistry(loggr) 39 | 40 | uaaClient := auth.NewUAAClient( 41 | cfg.UAA.Addr, 42 | buildUAAClient(cfg, loggr), 43 | metrics, 44 | loggr, 45 | ) 46 | 47 | // try to get our first token key, but bail out if we can't talk to UAA 48 | err = uaaClient.RefreshTokenKeys() 49 | if err != nil { 50 | loggr.Fatalf("failed to fetch token from UAA: %s", err) 51 | } 52 | 53 | gatewayURL, err := url.Parse(cfg.LogCacheGatewayAddr) 54 | if err != nil { 55 | loggr.Fatalf("failed to parse gateway address: %s", err) 56 | } 57 | 58 | // Force communication with the gateway to happen via HTTPS, regardless of 59 | // the scheme provided in the config 60 | gatewayURL.Scheme = "https" 61 | 62 | capiClient := auth.NewCAPIClient( 63 | cfg.CAPI.Addr, 64 | buildCAPIClient(cfg, loggr), 65 | metrics, 66 | loggr, 67 | auth.WithTokenPruningInterval(cfg.TokenPruningInterval), 68 | auth.WithCacheExpirationInterval(cfg.CacheExpirationInterval), 69 | ) 70 | 71 | proxyCACertPool := loadCA(cfg.ProxyCAPath, loggr) 72 | 73 | // Calls to /api/v1/meta get sent to the gateway, but not through the 74 | // reverse proxy like everything else. As a result, we also need to set 75 | // the Transport here to ensure the correct root CA is available. 76 | metaHTTPClient := &http.Client{ 77 | Timeout: 5 * time.Second, 78 | Transport: NewTransportWithRootCA(proxyCACertPool), 79 | } 80 | 81 | metaFetcher := client.NewClient( 82 | gatewayURL.String(), 83 | client.WithHTTPClient(metaHTTPClient), 84 | ) 85 | 86 | middlewareProvider := auth.NewCFAuthMiddlewareProvider( 87 | uaaClient, 88 | capiClient, 89 | metaFetcher, 90 | promql.ExtractSourceIds, 91 | capiClient, 92 | ) 93 | 94 | proxy := NewCFAuthProxy( 95 | gatewayURL.String(), 96 | cfg.Addr, 97 | cfg.CertPath, 98 | cfg.KeyPath, 99 | proxyCACertPool, 100 | WithAuthMiddleware(middlewareProvider.Middleware), 101 | ) 102 | 103 | if cfg.SecurityEventLog != "" { 104 | accessLog, err := os.OpenFile(cfg.SecurityEventLog, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) 105 | if err != nil { 106 | loggr.Panicf("Unable to open access log: %s", err) 107 | } 108 | defer func() { 109 | accessLog.Sync() 110 | accessLog.Close() 111 | }() 112 | 113 | _, localPort, err := net.SplitHostPort(cfg.Addr) 114 | if err != nil { 115 | loggr.Panicf("Unable to determine local port: %s", err) 116 | } 117 | 118 | accessLogger := auth.NewAccessLogger(accessLog) 119 | accessMiddleware := auth.NewAccessMiddleware(accessLogger, cfg.InternalIP, localPort) 120 | WithAccessMiddleware(accessMiddleware)(proxy) 121 | } 122 | 123 | proxy.Start() 124 | 125 | // health endpoints (pprof and prometheus) 126 | loggr.Printf("Health: %s", http.ListenAndServe(fmt.Sprintf("localhost:%d", cfg.HealthPort), nil)) 127 | } 128 | 129 | func buildUAAClient(cfg *Config, loggr *log.Logger) *http.Client { 130 | tlsConfig := sharedtls.NewBaseTLSConfig() 131 | tlsConfig.InsecureSkipVerify = cfg.SkipCertVerify 132 | 133 | tlsConfig.RootCAs = loadCA(cfg.UAA.CAPath, loggr) 134 | 135 | transport := &http.Transport{ 136 | TLSHandshakeTimeout: 10 * time.Second, 137 | TLSClientConfig: tlsConfig, 138 | DisableKeepAlives: true, 139 | } 140 | 141 | return &http.Client{ 142 | Timeout: 20 * time.Second, 143 | Transport: transport, 144 | } 145 | } 146 | 147 | func buildCAPIClient(cfg *Config, loggr *log.Logger) *http.Client { 148 | tlsConfig := sharedtls.NewBaseTLSConfig() 149 | tlsConfig.ServerName = cfg.CAPI.CommonName 150 | 151 | tlsConfig.RootCAs = loadCA(cfg.CAPI.CAPath, loggr) 152 | 153 | tlsConfig.InsecureSkipVerify = cfg.SkipCertVerify 154 | transport := &http.Transport{ 155 | TLSHandshakeTimeout: 10 * time.Second, 156 | TLSClientConfig: tlsConfig, 157 | DisableKeepAlives: true, 158 | } 159 | 160 | return &http.Client{ 161 | Timeout: 20 * time.Second, 162 | Transport: transport, 163 | } 164 | } 165 | 166 | func loadCA(caCertPath string, loggr *log.Logger) *x509.CertPool { 167 | caCert, err := ioutil.ReadFile(caCertPath) 168 | if err != nil { 169 | loggr.Fatalf("failed to read CA certificate: %s", err) 170 | } 171 | 172 | certPool := x509.NewCertPool() 173 | ok := certPool.AppendCertsFromPEM(caCert) 174 | if !ok { 175 | loggr.Fatal("failed to parse CA certificate.") 176 | } 177 | 178 | return certPool 179 | } 180 | -------------------------------------------------------------------------------- /cmd/gateway/.gitignore: -------------------------------------------------------------------------------- 1 | gateway 2 | -------------------------------------------------------------------------------- /cmd/gateway/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | envstruct "code.cloudfoundry.org/go-envstruct" 5 | "code.cloudfoundry.org/log-cache/internal/tls" 6 | ) 7 | 8 | var buildVersion string 9 | 10 | // Config is the configuration for a LogCache Gateway. 11 | type Config struct { 12 | Addr string `env:"ADDR, required, report"` 13 | LogCacheAddr string `env:"LOG_CACHE_ADDR, required, report"` 14 | HealthPort int `env:"HEALTH_PORT, report"` 15 | ProxyCertPath string `env:"PROXY_CERT_PATH, required, report"` 16 | ProxyKeyPath string `env:"PROXY_KEY_PATH, required, report"` 17 | Version string `env:"-, report"` 18 | 19 | TLS tls.TLS 20 | } 21 | 22 | // LoadConfig creates Config object from environment variables 23 | func LoadConfig() (*Config, error) { 24 | c := Config{ 25 | Addr: ":8081", 26 | HealthPort: 6063, 27 | LogCacheAddr: "localhost:8080", 28 | } 29 | 30 | if err := envstruct.Load(&c); err != nil { 31 | return nil, err 32 | } 33 | 34 | c.Version = buildVersion 35 | 36 | envstruct.WriteReport(&c) 37 | 38 | return &c, nil 39 | } 40 | -------------------------------------------------------------------------------- /cmd/gateway/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | 8 | "net/http" 9 | _ "net/http/pprof" 10 | 11 | . "code.cloudfoundry.org/log-cache/internal/gateway" 12 | "google.golang.org/grpc" 13 | ) 14 | 15 | func main() { 16 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 17 | 18 | log.Print("Starting Log Cache Gateway...") 19 | defer log.Print("Closing Log Cache Gateway.") 20 | 21 | cfg, err := LoadConfig() 22 | if err != nil { 23 | log.Fatalf("invalid configuration: %s", err) 24 | } 25 | 26 | gateway := NewGateway(cfg.LogCacheAddr, cfg.Addr, cfg.ProxyCertPath, cfg.ProxyKeyPath, 27 | WithGatewayLogger(log.New(os.Stderr, "[GATEWAY] ", log.LstdFlags)), 28 | WithGatewayLogCacheDialOpts( 29 | grpc.WithTransportCredentials(cfg.TLS.Credentials("log-cache")), 30 | ), 31 | WithGatewayVersion(cfg.Version), 32 | ) 33 | 34 | gateway.Start() 35 | 36 | // health endpoints (pprof) 37 | log.Printf("Health: %s", http.ListenAndServe(fmt.Sprintf("localhost:%d", cfg.HealthPort), nil)) 38 | } 39 | -------------------------------------------------------------------------------- /cmd/log-cache/.gitignore: -------------------------------------------------------------------------------- 1 | log-cache 2 | -------------------------------------------------------------------------------- /cmd/log-cache/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | envstruct "code.cloudfoundry.org/go-envstruct" 7 | "code.cloudfoundry.org/log-cache/internal/tls" 8 | ) 9 | 10 | // Config is the configuration for a LogCache. 11 | type Config struct { 12 | Addr string `env:"ADDR, required, report"` 13 | HealthPort int `env:"HEALTH_PORT, report"` 14 | 15 | // QueryTimeout sets the maximum allowed runtime for a single PromQL query. 16 | // Smaller timeouts are recommended. 17 | QueryTimeout time.Duration `env:"QUERY_TIMEOUT, report"` 18 | 19 | // MemoryLimit sets the percentage of total system memory to use for the 20 | // cache. If exceeded, the cache will prune. Default is 50%. 21 | MemoryLimit uint `env:"MEMORY_LIMIT_PERCENT, report"` 22 | 23 | // MaxPerSource sets the maximum number of items stored per source. 24 | // Because autoscaler requires a minute of data, apps with more than 1000 25 | // requests per second will fill up the router logs/metrics in less than a 26 | // minute. Default is 100000. 27 | MaxPerSource int `env:"MAX_PER_SOURCE, report"` 28 | 29 | // NodeIndex determines what data the node stores. It splits up the range 30 | // of 0 - 18446744073709551615 evenly. If data falls out of range of the 31 | // given node, it will be routed to theh correct one. 32 | NodeIndex int `env:"NODE_INDEX, report"` 33 | 34 | // NodeAddrs are all the LogCache addresses (including the current 35 | // address). They are in order according to their NodeIndex. 36 | // 37 | // If NodeAddrs is emptpy or size 1, then data is not routed as it is 38 | // assumed that the current node is the only one. 39 | NodeAddrs []string `env:"NODE_ADDRS, report"` 40 | 41 | TLS tls.TLS 42 | } 43 | 44 | // LoadConfig creates Config object from environment variables 45 | func LoadConfig() (*Config, error) { 46 | c := Config{ 47 | Addr: ":8080", 48 | HealthPort: 6060, 49 | QueryTimeout: 10 * time.Second, 50 | MemoryLimit: 50, 51 | MaxPerSource: 100000, 52 | } 53 | 54 | if err := envstruct.Load(&c); err != nil { 55 | return nil, err 56 | } 57 | 58 | return &c, nil 59 | } 60 | -------------------------------------------------------------------------------- /cmd/log-cache/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | "time" 11 | 12 | envstruct "code.cloudfoundry.org/go-envstruct" 13 | . "code.cloudfoundry.org/log-cache/internal/cache" 14 | "google.golang.org/grpc" 15 | ) 16 | 17 | func main() { 18 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 19 | 20 | log.Print("Starting Log Cache...") 21 | defer log.Print("Closing Log Cache.") 22 | 23 | cfg, err := LoadConfig() 24 | if err != nil { 25 | log.Fatalf("invalid configuration: %s", err) 26 | } 27 | 28 | envstruct.WriteReport(cfg) 29 | 30 | logger := log.New(os.Stderr, "", log.LstdFlags) 31 | 32 | m := metrics.NewRegistry(logger) 33 | uptimeFn := m.NewGauge( 34 | "log_cache_uptime", 35 | metrics.WithMetricTags(map[string]string{ 36 | "unit": "seconds", 37 | }), 38 | ) 39 | 40 | t := time.NewTicker(time.Second) 41 | go func(start time.Time) { 42 | for range t.C { 43 | uptimeFn.Set(float64(time.Since(start) / time.Second)) 44 | } 45 | }(time.Now()) 46 | 47 | cache := New( 48 | m, 49 | logger, 50 | WithAddr(cfg.Addr), 51 | WithMemoryLimit(float64(cfg.MemoryLimit)), 52 | WithMaxPerSource(cfg.MaxPerSource), 53 | WithQueryTimeout(cfg.QueryTimeout), 54 | WithClustered( 55 | cfg.NodeIndex, 56 | cfg.NodeAddrs, 57 | grpc.WithTransportCredentials( 58 | cfg.TLS.Credentials("log-cache"), 59 | ), 60 | ), 61 | WithServerOpts(grpc.Creds(cfg.TLS.Credentials("log-cache"))), 62 | ) 63 | 64 | cache.Start() 65 | 66 | // health endpoints (pprof and prometheus) 67 | log.Printf("Health: %s", http.ListenAndServe(fmt.Sprintf("localhost:%d", cfg.HealthPort), nil)) 68 | } 69 | -------------------------------------------------------------------------------- /cmd/nozzle/.gitignore: -------------------------------------------------------------------------------- 1 | nozzle 2 | -------------------------------------------------------------------------------- /cmd/nozzle/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | envstruct "code.cloudfoundry.org/go-envstruct" 5 | "code.cloudfoundry.org/log-cache/internal/tls" 6 | ) 7 | 8 | // Config is the configuration for a LogCache. 9 | type Config struct { 10 | LogProviderAddr string `env:"LOGS_PROVIDER_ADDR, required, report"` 11 | LogsProviderTLS LogsProviderTLS 12 | 13 | LogCacheAddr string `env:"LOG_CACHE_ADDR, required, report"` 14 | HealthPort int `env:"HEALTH_PORT, report"` 15 | ShardId string `env:"SHARD_ID, required, report"` 16 | Selectors []string `env:"SELECTORS, required, report"` 17 | 18 | LogCacheTLS tls.TLS 19 | } 20 | 21 | // LogsProviderTLS is the LogsProviderTLS configuration for a LogCache. 22 | type LogsProviderTLS struct { 23 | LogProviderCA string `env:"LOGS_PROVIDER_CA_FILE_PATH, required, report"` 24 | LogProviderCert string `env:"LOGS_PROVIDER_CERT_FILE_PATH, required, report"` 25 | LogProviderKey string `env:"LOGS_PROVIDER_KEY_FILE_PATH, required, report"` 26 | } 27 | 28 | // LoadConfig creates Config object from environment variables 29 | func LoadConfig() (*Config, error) { 30 | c := Config{ 31 | LogCacheAddr: ":8080", 32 | HealthPort: 6061, 33 | ShardId: "log-cache", 34 | Selectors: []string{"log", "gauge", "counter", "timer", "event"}, 35 | } 36 | 37 | if err := envstruct.Load(&c); err != nil { 38 | return nil, err 39 | } 40 | 41 | return &c, nil 42 | } 43 | -------------------------------------------------------------------------------- /cmd/nozzle/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | 11 | envstruct "code.cloudfoundry.org/go-envstruct" 12 | . "code.cloudfoundry.org/log-cache/internal/nozzle" 13 | "google.golang.org/grpc" 14 | 15 | loggregator "code.cloudfoundry.org/go-loggregator" 16 | ) 17 | 18 | func main() { 19 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 20 | 21 | log.Print("Starting LogCache Nozzle...") 22 | defer log.Print("Closing LogCache Nozzle.") 23 | 24 | cfg, err := LoadConfig() 25 | if err != nil { 26 | log.Fatalf("invalid configuration: %s", err) 27 | } 28 | 29 | envstruct.WriteReport(cfg) 30 | 31 | tlsCfg, err := loggregator.NewEgressTLSConfig( 32 | cfg.LogsProviderTLS.LogProviderCA, 33 | cfg.LogsProviderTLS.LogProviderCert, 34 | cfg.LogsProviderTLS.LogProviderKey, 35 | ) 36 | if err != nil { 37 | log.Fatalf("invalid LogsProviderTLS configuration: %s", err) 38 | } 39 | 40 | loggr := log.New(os.Stderr, "[LOGGR] ", log.LstdFlags) 41 | m := metrics.NewRegistry(loggr) 42 | 43 | dropped := m.NewCounter("nozzle_dropped") 44 | 45 | streamConnector := loggregator.NewEnvelopeStreamConnector( 46 | cfg.LogProviderAddr, 47 | tlsCfg, 48 | loggregator.WithEnvelopeStreamLogger(loggr), 49 | loggregator.WithEnvelopeStreamBuffer(10000, func(missed int) { 50 | loggr.Printf("dropped %d envelope batches", missed) 51 | dropped.Add(float64(missed)) 52 | }), 53 | ) 54 | 55 | nozzle := NewNozzle( 56 | streamConnector, 57 | cfg.LogCacheAddr, 58 | cfg.ShardId, 59 | m, 60 | loggr, 61 | WithDialOpts( 62 | grpc.WithTransportCredentials( 63 | cfg.LogCacheTLS.Credentials("log-cache"), 64 | ), 65 | ), 66 | WithSelectors(cfg.Selectors...), 67 | ) 68 | 69 | go nozzle.Start() 70 | 71 | // health endpoints (pprof and prometheus) 72 | log.Printf("Health: %s", http.ListenAndServe(fmt.Sprintf("localhost:%d", cfg.HealthPort), nil)) 73 | } 74 | -------------------------------------------------------------------------------- /cmd/scheduler/.gitignore: -------------------------------------------------------------------------------- 1 | scheduler 2 | -------------------------------------------------------------------------------- /cmd/scheduler/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | envstruct "code.cloudfoundry.org/go-envstruct" 7 | "code.cloudfoundry.org/log-cache/internal/tls" 8 | ) 9 | 10 | // Config is the configuration for a Scheduler. 11 | type Config struct { 12 | HealthPort int `env:"HEALTH_PORT, report"` 13 | 14 | Interval time.Duration `env:"INTERVAL, report"` 15 | Count int `env:"COUNT, report"` 16 | ReplicationFactor int `env:"REPLICATION_FACTOR, report"` 17 | 18 | // NodeAddrs are all the LogCache addresses. They are in order according 19 | // to their NodeIndex. 20 | NodeAddrs []string `env:"NODE_ADDRS, report"` 21 | 22 | // If empty, then the scheduler assumes it is always the leader. 23 | LeaderElectionEndpoint string `env:"LEADER_ELECTION_ENDPOINT, report"` 24 | 25 | TLS tls.TLS 26 | } 27 | 28 | // LoadConfig creates Config object from environment variables 29 | func LoadConfig() (*Config, error) { 30 | c := Config{ 31 | HealthPort: 6064, 32 | Count: 100, 33 | ReplicationFactor: 1, 34 | Interval: time.Minute, 35 | } 36 | 37 | if err := envstruct.Load(&c); err != nil { 38 | return nil, err 39 | } 40 | 41 | return &c, nil 42 | } 43 | -------------------------------------------------------------------------------- /cmd/scheduler/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | _ "net/http/pprof" 8 | "os" 9 | 10 | envstruct "code.cloudfoundry.org/go-envstruct" 11 | . "code.cloudfoundry.org/log-cache/internal/scheduler" 12 | "google.golang.org/grpc" 13 | ) 14 | 15 | func main() { 16 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 17 | 18 | log.Print("Starting Log Cache Scheduler...") 19 | defer log.Print("Closing Log Cache Scheduler.") 20 | 21 | cfg, err := LoadConfig() 22 | if err != nil { 23 | log.Fatalf("invalid configuration: %s", err) 24 | } 25 | 26 | envstruct.WriteReport(cfg) 27 | 28 | opts := []SchedulerOption{ 29 | WithSchedulerLogger(log.New(os.Stderr, "", log.LstdFlags)), 30 | WithSchedulerInterval(cfg.Interval), 31 | WithSchedulerCount(cfg.Count), 32 | WithSchedulerReplicationFactor(cfg.ReplicationFactor), 33 | WithSchedulerDialOpts( 34 | grpc.WithTransportCredentials(cfg.TLS.Credentials("log-cache")), 35 | ), 36 | } 37 | 38 | if cfg.LeaderElectionEndpoint != "" { 39 | opts = append(opts, WithSchedulerLeadership(func() bool { 40 | resp, err := http.Get(cfg.LeaderElectionEndpoint) 41 | if err != nil { 42 | log.Printf("failed to read from leaderhip endpoint: %s", err) 43 | return false 44 | } 45 | 46 | return resp.StatusCode == http.StatusOK 47 | })) 48 | } 49 | 50 | sched := NewScheduler( 51 | cfg.NodeAddrs, 52 | opts..., 53 | ) 54 | 55 | sched.Start() 56 | 57 | // health endpoints (pprof) 58 | log.Printf("Health: %s", http.ListenAndServe(fmt.Sprintf("localhost:%d", cfg.HealthPort), nil)) 59 | } 60 | -------------------------------------------------------------------------------- /cmd/syslog-server/.gitignore: -------------------------------------------------------------------------------- 1 | syslog-server 2 | -------------------------------------------------------------------------------- /cmd/syslog-server/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | envstruct "code.cloudfoundry.org/go-envstruct" 5 | "code.cloudfoundry.org/log-cache/internal/tls" 6 | "time" 7 | ) 8 | 9 | // Config is the configuration for a Syslog Server 10 | type Config struct { 11 | LogCacheAddr string `env:"LOG_CACHE_ADDR, required, report"` 12 | SyslogPort int `env:"SYSLOG_PORT, required, report"` 13 | HealthPort int `env:"HEALTH_PORT, report"` 14 | 15 | LogCacheTLS tls.TLS 16 | SyslogTLSCertPath string `env:"SYSLOG_TLS_CERT_PATH, required, report"` 17 | SyslogTLSKeyPath string `env:"SYSLOG_TLS_KEY_PATH, required, report"` 18 | 19 | SyslogIdleTimeout time.Duration `env:"SYSLOG_IDLE_TIMEOUT, report"` 20 | } 21 | 22 | // LoadConfig creates Config object from environment variables 23 | func LoadConfig() (*Config, error) { 24 | c := Config{ 25 | LogCacheAddr: ":8080", 26 | SyslogPort: 8888, 27 | HealthPort: 6061, 28 | SyslogIdleTimeout: 2 * time.Minute, 29 | } 30 | 31 | if err := envstruct.Load(&c); err != nil { 32 | return nil, err 33 | } 34 | 35 | return &c, nil 36 | } 37 | -------------------------------------------------------------------------------- /cmd/syslog-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "code.cloudfoundry.org/log-cache/internal/routing" 6 | "code.cloudfoundry.org/log-cache/internal/syslog" 7 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 8 | "fmt" 9 | "log" 10 | "net/http" 11 | _ "net/http/pprof" 12 | "os" 13 | "time" 14 | 15 | "code.cloudfoundry.org/go-envstruct" 16 | "google.golang.org/grpc" 17 | ) 18 | 19 | const ( 20 | BATCH_FLUSH_INTERVAL = 500 * time.Millisecond 21 | BATCH_CHANNEL_SIZE = 512 22 | ) 23 | 24 | func main() { 25 | log.SetFlags(log.LstdFlags | log.Lmicroseconds) 26 | 27 | log.Print("Starting Syslog Server...") 28 | defer log.Print("Closing Syslog Server.") 29 | 30 | cfg, err := LoadConfig() 31 | if err != nil { 32 | log.Fatalf("invalid configuration: %s", err) 33 | } 34 | 35 | envstruct.WriteReport(cfg) 36 | 37 | loggr := log.New(os.Stderr, "[LOGGR] ", log.LstdFlags) 38 | m := metrics.NewRegistry(loggr) 39 | 40 | egressDropped := m.NewCounter("egress_dropped") 41 | conn, err := grpc.Dial( 42 | cfg.LogCacheAddr, 43 | grpc.WithTransportCredentials( 44 | cfg.LogCacheTLS.Credentials("log-cache"), 45 | ), 46 | ) 47 | 48 | client := logcache_v1.NewIngressClient(conn) 49 | 50 | server := syslog.NewServer( 51 | loggr, 52 | routing.NewBatchedIngressClient(BATCH_CHANNEL_SIZE, BATCH_FLUSH_INTERVAL, client, egressDropped, loggr), 53 | m, 54 | cfg.SyslogTLSCertPath, 55 | cfg.SyslogTLSKeyPath, 56 | syslog.WithServerPort(cfg.SyslogPort), 57 | syslog.WithIdleTimeout(cfg.SyslogIdleTimeout), 58 | ) 59 | 60 | go server.Start() 61 | 62 | // health endpoints (pprof and prometheus) 63 | log.Printf("Health: %s", http.ListenAndServe(fmt.Sprintf("localhost:%d", cfg.HealthPort), nil)) 64 | } 65 | -------------------------------------------------------------------------------- /internal/auth/access_log.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "html/template" 7 | "log" 8 | "net" 9 | "net/http" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | const requestIDHeader = "X-Vcap-Request-ID" 15 | 16 | var logTemplate *template.Template 17 | 18 | type templateContext struct { 19 | Timestamp int64 20 | RequestID string 21 | Path string 22 | Method string 23 | SourceHost string 24 | SourcePort string 25 | DestinationHost string 26 | DestinationPort string 27 | } 28 | 29 | type AccessLog struct { 30 | request *http.Request 31 | timestamp time.Time 32 | host string 33 | port string 34 | } 35 | 36 | func NewAccessLog(req *http.Request, ts time.Time, host, port string) *AccessLog { 37 | return &AccessLog{ 38 | request: req, 39 | timestamp: ts, 40 | host: host, 41 | port: port, 42 | } 43 | } 44 | 45 | func (al *AccessLog) String() string { 46 | vcapRequestId := al.request.Header.Get(requestIDHeader) 47 | path := al.request.URL.Path 48 | if al.request.URL.RawQuery != "" { 49 | path = fmt.Sprintf("%s?%s", al.request.URL.Path, al.request.URL.RawQuery) 50 | } 51 | remoteHost, remotePort := al.extractRemoteInfo() 52 | 53 | context := templateContext{ 54 | toMillis(al.timestamp), 55 | vcapRequestId, 56 | path, 57 | al.request.Method, 58 | remoteHost, 59 | remotePort, 60 | al.host, 61 | al.port, 62 | } 63 | var buf bytes.Buffer 64 | err := logTemplate.Execute(&buf, context) 65 | if err != nil { 66 | log.Printf("Error executing security access log template: %s\n", err) 67 | return "" 68 | } 69 | return buf.String() 70 | } 71 | 72 | func (al *AccessLog) extractRemoteInfo() (string, string) { 73 | remoteAddr := al.request.Header.Get("X-Forwarded-For") 74 | index := strings.Index(remoteAddr, ",") 75 | if index > -1 { 76 | remoteAddr = remoteAddr[:index] 77 | } 78 | 79 | if remoteAddr == "" { 80 | remoteAddr = al.request.RemoteAddr 81 | } 82 | 83 | if !strings.Contains(remoteAddr, ":") { 84 | remoteAddr += ":" 85 | } 86 | host, port, err := net.SplitHostPort(remoteAddr) 87 | if err != nil { 88 | log.Printf("Error splitting host and port for access log: %s\n", err) 89 | return "", "" 90 | } 91 | return host, port 92 | } 93 | 94 | func toMillis(timestamp time.Time) int64 { 95 | return timestamp.UnixNano() / int64(time.Millisecond) 96 | } 97 | 98 | func init() { 99 | extensions := []string{ 100 | "rt={{ .Timestamp }}", 101 | "cs1Label=userAuthenticationMechanism", 102 | "cs1=oauth-access-token", 103 | "cs2Label=vcapRequestId", 104 | "cs2={{ .RequestID }}", 105 | "request={{ .Path }}", 106 | "requestMethod={{ .Method }}", 107 | "src={{ .SourceHost }}", 108 | "spt={{ .SourcePort }}", 109 | "dst={{ .DestinationHost }}", 110 | "dpt={{ .DestinationPort }}", 111 | } 112 | fields := []string{ 113 | "0", 114 | "cloud_foundry", 115 | "log_cache", 116 | "1.0", 117 | "{{ .Method }} {{ .Path }}", 118 | "{{ .Method }} {{ .Path }}", 119 | "0", 120 | strings.Join(extensions, " "), 121 | } 122 | templateSource := "CEF:" + strings.Join(fields, "|") 123 | var err error 124 | logTemplate, err = template.New("access_log").Parse(templateSource) 125 | if err != nil { 126 | log.Panic(err) 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /internal/auth/access_log_test.go: -------------------------------------------------------------------------------- 1 | package auth_test 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "net/http" 7 | "strconv" 8 | "time" 9 | 10 | "code.cloudfoundry.org/log-cache/internal/auth" 11 | 12 | "code.cloudfoundry.org/log-cache/internal/testing" 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/gomega" 15 | ) 16 | 17 | var _ = Describe("AccessLog", func() { 18 | var ( 19 | req *http.Request 20 | timestamp time.Time 21 | al *auth.AccessLog 22 | 23 | // request data 24 | method string 25 | path string 26 | url string 27 | sourceHost string 28 | sourcePort string 29 | remoteAddr string 30 | dstHost string 31 | dstPort string 32 | 33 | forwardedFor string 34 | requestId string 35 | ) 36 | 37 | BeforeEach(func() { 38 | req = nil 39 | timestamp = time.Now() 40 | 41 | method = "GET" 42 | path = fmt.Sprintf("/some/path?with_query=params-%d", rand.Int()) 43 | url = "http://example.com" + path 44 | sourceHost = fmt.Sprintf("10.0.1.%d", rand.Int()%256) 45 | sourcePort = strconv.Itoa(rand.Int()%65535 + 1) 46 | remoteAddr = sourceHost + ":" + sourcePort 47 | dstHost = fmt.Sprintf("10.1.2.%d", rand.Int()%256) 48 | dstPort = strconv.Itoa(rand.Int()%65535 + 1) 49 | 50 | forwardedFor = fmt.Sprintf("10.0.0.%d", rand.Int()%256) 51 | requestId = fmt.Sprintf("test-vcap-request-id-%d", rand.Int()) 52 | }) 53 | 54 | JustBeforeEach(func() { 55 | req = testing.BuildRequest(method, url, remoteAddr, requestId, forwardedFor) 56 | al = auth.NewAccessLog(req, timestamp, dstHost, dstPort) 57 | }) 58 | 59 | Describe("String", func() { 60 | Context("with a GET request", func() { 61 | BeforeEach(func() { 62 | method = "GET" 63 | }) 64 | 65 | It("returns a log with GET as the method", func() { 66 | expected := testing.BuildExpectedLog( 67 | timestamp, 68 | requestId, 69 | method, 70 | path, 71 | forwardedFor, 72 | "", 73 | dstHost, 74 | dstPort, 75 | ) 76 | Expect(al.String()).To(Equal(expected)) 77 | }) 78 | }) 79 | 80 | Context("with a POST request", func() { 81 | BeforeEach(func() { 82 | method = "POST" 83 | }) 84 | 85 | It("returns a log with POST as the method", func() { 86 | expected := testing.BuildExpectedLog( 87 | timestamp, 88 | requestId, 89 | method, 90 | path, 91 | forwardedFor, 92 | "", 93 | dstHost, 94 | dstPort, 95 | ) 96 | Expect(al.String()).To(Equal(expected)) 97 | }) 98 | }) 99 | 100 | Context("with X-Forwarded-For not set", func() { 101 | BeforeEach(func() { 102 | forwardedFor = "" 103 | }) 104 | 105 | It("uses remoteAddr", func() { 106 | expected := testing.BuildExpectedLog( 107 | timestamp, 108 | requestId, 109 | method, 110 | path, 111 | sourceHost, 112 | sourcePort, 113 | dstHost, 114 | dstPort, 115 | ) 116 | Expect(al.String()).To(Equal(expected)) 117 | }) 118 | }) 119 | 120 | Context("with X-Forwarded-For containing multiple values", func() { 121 | BeforeEach(func() { 122 | forwardedFor = "123.22.11.1, 6.3.4.5, 1.2.3.4" 123 | }) 124 | 125 | It("uses remoteAddr", func() { 126 | expected := testing.BuildExpectedLog( 127 | timestamp, 128 | requestId, 129 | method, 130 | path, 131 | "123.22.11.1", 132 | "", 133 | dstHost, 134 | dstPort, 135 | ) 136 | Expect(al.String()).To(Equal(expected)) 137 | }) 138 | }) 139 | 140 | Context("with a request that has no query params", func() { 141 | BeforeEach(func() { 142 | path = "/some/path" 143 | url = "http://example.com" + path 144 | }) 145 | 146 | It("writes log without question mark delimiter", func() { 147 | prefix := "CEF:0|cloud_foundry|log_cache|1.0|GET /some/path|" 148 | Expect(al.String()).To(HavePrefix(prefix)) 149 | }) 150 | }) 151 | }) 152 | }) 153 | -------------------------------------------------------------------------------- /internal/auth/access_logger.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | type DefaultAccessLogger struct { 10 | writer io.Writer 11 | } 12 | 13 | func NewAccessLogger(writer io.Writer) *DefaultAccessLogger { 14 | return &DefaultAccessLogger{ 15 | writer: writer, 16 | } 17 | } 18 | 19 | func (a *DefaultAccessLogger) LogAccess(req *http.Request, host, port string) error { 20 | al := NewAccessLog(req, time.Now(), host, port) 21 | _, err := a.writer.Write([]byte(al.String() + "\n")) 22 | return err 23 | } 24 | 25 | type NullAccessLogger struct { 26 | } 27 | 28 | func NewNullAccessLogger() *NullAccessLogger { 29 | return &NullAccessLogger{} 30 | } 31 | 32 | func (a *NullAccessLogger) LogAccess(req *http.Request, host, port string) error { 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /internal/auth/access_logger_test.go: -------------------------------------------------------------------------------- 1 | package auth_test 2 | 3 | import ( 4 | "errors" 5 | 6 | "code.cloudfoundry.org/log-cache/internal/auth" 7 | 8 | "code.cloudfoundry.org/log-cache/internal/testing" 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var _ = Describe("DefaultAccessLogger", func() { 14 | var ( 15 | writer *spyWriter 16 | logger *auth.DefaultAccessLogger 17 | ) 18 | 19 | BeforeEach(func() { 20 | writer = &spyWriter{} 21 | logger = auth.NewAccessLogger(writer) 22 | }) 23 | 24 | It("logs Access", func() { 25 | req, err := testing.NewServerRequest("GET", "some.url.com/foo", nil) 26 | Expect(err).ToNot(HaveOccurred()) 27 | 28 | Expect(logger.LogAccess(req, "1.1.1.1", "1")).To(Succeed()) 29 | prefix := "CEF:0|cloud_foundry|log_cache|1.0|GET some.url.com/foo|GET some.url.com/foo|0|" 30 | Expect(writer.message).To(HavePrefix(prefix)) 31 | }) 32 | 33 | It("includes details about the access", func() { 34 | req, err := testing.NewServerRequest("GET", "http://some.url.com/foo", nil) 35 | Expect(err).ToNot(HaveOccurred()) 36 | req.RemoteAddr = "127.0.0.1:4567" 37 | 38 | Expect(logger.LogAccess(req, "1.1.1.1", "1")).To(Succeed()) 39 | Expect(writer.message).To(ContainSubstring("src=127.0.0.1 spt=4567")) 40 | }) 41 | 42 | It("uses X-Forwarded-For if it exists", func() { 43 | req, err := testing.NewServerRequest("GET", "http://some.url.com/foo", nil) 44 | Expect(err).ToNot(HaveOccurred()) 45 | req.RemoteAddr = "127.0.0.1:4567" 46 | req.Header.Set("X-Forwarded-For", "50.60.70.80:1234") 47 | 48 | Expect(logger.LogAccess(req, "1.1.1.1", "1")).To(Succeed()) 49 | Expect(writer.message).To(ContainSubstring("src=50.60.70.80 spt=1234")) 50 | }) 51 | 52 | It("writes multiple log lines", func() { 53 | req, err := testing.NewServerRequest("GET", "http://some.url.com/foo", nil) 54 | Expect(err).ToNot(HaveOccurred()) 55 | req.RemoteAddr = "127.0.0.1:4567" 56 | 57 | Expect(logger.LogAccess(req, "1.1.1.1", "1")).To(Succeed()) 58 | expected := "src=127.0.0.1 spt=4567" 59 | Expect(writer.message).To(ContainSubstring(expected)) 60 | Expect(writer.message).To(HaveSuffix("\n")) 61 | 62 | req.Header.Set("X-Forwarded-For", "50.60.70.80:1234") 63 | 64 | Expect(logger.LogAccess(req, "1.1.1.1", "1")).To(Succeed()) 65 | expected = "src=50.60.70.80 spt=1234" 66 | Expect(writer.message).To(ContainSubstring(expected)) 67 | Expect(writer.message).To(HaveSuffix("\n")) 68 | }) 69 | 70 | It("returns an error", func() { 71 | writer = &spyWriter{} 72 | writer.err = errors.New("boom") 73 | logger = auth.NewAccessLogger(writer) 74 | 75 | req, err := testing.NewServerRequest("GET", "http://some.url.com/foo", nil) 76 | Expect(err).ToNot(HaveOccurred()) 77 | req.RemoteAddr = "127.0.0.1:4567" 78 | Expect(logger.LogAccess(req, "1.1.1.1", "1")).ToNot(Succeed()) 79 | }) 80 | }) 81 | 82 | type spyWriter struct { 83 | message []byte 84 | err error 85 | } 86 | 87 | func (s *spyWriter) Write(message []byte) (sent int, err error) { 88 | s.message = message 89 | return 0, s.err 90 | } 91 | -------------------------------------------------------------------------------- /internal/auth/access_middleware.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | ) 7 | 8 | func NewAccessMiddleware(accessLogger AccessLogger, host, port string) func(http.Handler) *AccessHandler { 9 | return func(handler http.Handler) *AccessHandler { 10 | return NewAccessHandler(handler, accessLogger, host, port) 11 | } 12 | } 13 | 14 | func NewNullAccessMiddleware() func(http.Handler) *AccessHandler { 15 | return func(handler http.Handler) *AccessHandler { 16 | return &AccessHandler{ 17 | handler: handler, 18 | accessLogger: NewNullAccessLogger(), 19 | } 20 | } 21 | } 22 | 23 | type AccessLogger interface { 24 | LogAccess(req *http.Request, host, port string) error 25 | } 26 | 27 | type AccessHandler struct { 28 | handler http.Handler 29 | accessLogger AccessLogger 30 | host string 31 | port string 32 | } 33 | 34 | func NewAccessHandler(handler http.Handler, accessLogger AccessLogger, host, port string) *AccessHandler { 35 | return &AccessHandler{ 36 | handler: handler, 37 | accessLogger: accessLogger, 38 | host: host, 39 | port: port, 40 | } 41 | } 42 | 43 | func (h *AccessHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { 44 | if err := h.accessLogger.LogAccess(req, h.host, h.port); err != nil { 45 | log.Printf("access handler: %s", err) 46 | } 47 | 48 | h.handler.ServeHTTP(rw, req) 49 | } 50 | -------------------------------------------------------------------------------- /internal/auth/access_middleware_test.go: -------------------------------------------------------------------------------- 1 | package auth_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | 7 | "code.cloudfoundry.org/log-cache/internal/auth" 8 | 9 | "code.cloudfoundry.org/log-cache/internal/testing" 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | var _ = Describe("AccessHandler", func() { 15 | const ( 16 | host = "1.2.3.4" 17 | port = "1234" 18 | ) 19 | var ( 20 | accessHandler *auth.AccessHandler 21 | handler *spyHandler 22 | spyLogger *spyAccessLogger 23 | ) 24 | 25 | BeforeEach(func() { 26 | handler = &spyHandler{} 27 | spyLogger = &spyAccessLogger{} 28 | accessMiddleware := auth.NewAccessMiddleware(spyLogger, host, port) 29 | accessHandler = accessMiddleware(handler) 30 | 31 | var _ http.Handler = accessHandler 32 | }) 33 | 34 | Describe("ServeHTTP", func() { 35 | It("Logs the access", func() { 36 | req, err := testing.NewServerRequest("GET", "https://foo.bar/baz", nil) 37 | Expect(err).ToNot(HaveOccurred()) 38 | resp := httptest.NewRecorder() 39 | accessHandler.ServeHTTP(resp, req) 40 | 41 | Expect(handler.response).To(Equal(resp)) 42 | Expect(handler.request).To(Equal(req)) 43 | Expect(spyLogger.request).To(Equal(req)) 44 | Expect(spyLogger.host).To(Equal(host)) 45 | Expect(spyLogger.port).To(Equal(port)) 46 | }) 47 | }) 48 | }) 49 | 50 | type spyHandler struct { 51 | response http.ResponseWriter 52 | request *http.Request 53 | } 54 | 55 | func (s *spyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 56 | s.response = resp 57 | s.request = req 58 | } 59 | 60 | type spyAccessLogger struct { 61 | request *http.Request 62 | host string 63 | port string 64 | } 65 | 66 | func (s *spyAccessLogger) LogAccess(req *http.Request, host, port string) error { 67 | s.request = req 68 | s.host = host 69 | s.port = port 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /internal/auth/auth_suite_test.go: -------------------------------------------------------------------------------- 1 | package auth_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestAuth(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Auth Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/auth/cf_auth_middleware.go: -------------------------------------------------------------------------------- 1 | package auth 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "time" 7 | 8 | "log" 9 | 10 | "context" 11 | 12 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 13 | "github.com/golang/protobuf/jsonpb" 14 | "github.com/gorilla/mux" 15 | 16 | "code.cloudfoundry.org/log-cache/internal/promql" 17 | ) 18 | 19 | type CFAuthMiddlewareProvider struct { 20 | oauth2Reader Oauth2ClientReader 21 | logAuthorizer LogAuthorizer 22 | metaFetcher MetaFetcher 23 | marshaller jsonpb.Marshaler 24 | promQLSourceIdExtractor PromQLSourceIdExtractor 25 | appNameTranslator AppNameTranslator 26 | } 27 | 28 | type Oauth2ClientContext struct { 29 | IsAdmin bool 30 | Token string 31 | ExpiresAt time.Time 32 | } 33 | 34 | type Oauth2ClientReader interface { 35 | Read(token string) (Oauth2ClientContext, error) 36 | } 37 | 38 | type LogAuthorizer interface { 39 | IsAuthorized(sourceID string, clientToken string) bool 40 | AvailableSourceIDs(token string) []string 41 | } 42 | 43 | type MetaFetcher interface { 44 | Meta(context.Context) (map[string]*rpc.MetaInfo, error) 45 | } 46 | 47 | type AppNameTranslator interface { 48 | GetRelatedSourceIds(appNames []string, token string) map[string][]string 49 | } 50 | 51 | type PromQLSourceIdExtractor func(query string) ([]string, error) 52 | 53 | func NewCFAuthMiddlewareProvider( 54 | oauth2Reader Oauth2ClientReader, 55 | logAuthorizer LogAuthorizer, 56 | metaFetcher MetaFetcher, 57 | promQLSourceIdExtractor PromQLSourceIdExtractor, appNameTranslator AppNameTranslator, 58 | ) CFAuthMiddlewareProvider { 59 | return CFAuthMiddlewareProvider{ 60 | oauth2Reader: oauth2Reader, 61 | logAuthorizer: logAuthorizer, 62 | metaFetcher: metaFetcher, 63 | promQLSourceIdExtractor: promQLSourceIdExtractor, 64 | appNameTranslator: appNameTranslator, 65 | } 66 | } 67 | 68 | type promqlErrorBody struct { 69 | Status string `json:"status"` 70 | ErrorType string `json:"errorType"` 71 | Error string `json:"error"` 72 | } 73 | 74 | func (m CFAuthMiddlewareProvider) Middleware(h http.Handler) http.Handler { 75 | router := mux.NewRouter() 76 | 77 | router.HandleFunc("/api/v1/read/{sourceID:.*}", func(w http.ResponseWriter, r *http.Request) { 78 | sourceID, ok := mux.Vars(r)["sourceID"] 79 | if !ok { 80 | w.WriteHeader(http.StatusNotFound) 81 | return 82 | } 83 | 84 | authToken := r.Header.Get("Authorization") 85 | if authToken == "" { 86 | w.WriteHeader(http.StatusNotFound) 87 | return 88 | } 89 | 90 | userContext, err := m.oauth2Reader.Read(authToken) 91 | if err != nil { 92 | log.Printf("failed to read from Oauth2 server: %s", err) 93 | w.WriteHeader(http.StatusNotFound) 94 | return 95 | } 96 | 97 | if !userContext.IsAdmin { 98 | if !m.logAuthorizer.IsAuthorized(sourceID, userContext.Token) { 99 | w.WriteHeader(http.StatusNotFound) 100 | return 101 | } 102 | } 103 | 104 | h.ServeHTTP(w, r) 105 | }) 106 | 107 | router.HandleFunc("/api/v1/{subpath:query|query_range}", func(w http.ResponseWriter, r *http.Request) { 108 | authToken := r.Header.Get("Authorization") 109 | if authToken == "" { 110 | w.WriteHeader(http.StatusNotFound) 111 | return 112 | } 113 | 114 | query := r.URL.Query().Get("query") 115 | sourceIds, err := m.promQLSourceIdExtractor(query) 116 | if err != nil { 117 | w.Header().Set("Content-Type", "application/json") 118 | w.WriteHeader(http.StatusBadRequest) 119 | 120 | json.NewEncoder(w).Encode(&promqlErrorBody{ 121 | Status: "error", 122 | ErrorType: "bad_data", 123 | Error: err.Error(), 124 | }) 125 | 126 | return 127 | } 128 | 129 | if len(sourceIds) == 0 { 130 | w.Header().Set("Content-Type", "application/json") 131 | w.WriteHeader(http.StatusBadRequest) 132 | 133 | json.NewEncoder(w).Encode(&promqlErrorBody{ 134 | Status: "error", 135 | ErrorType: "bad_data", 136 | Error: "query does not request any source_ids", 137 | }) 138 | 139 | return 140 | } 141 | 142 | c, err := m.oauth2Reader.Read(authToken) 143 | if err != nil { 144 | log.Printf("failed to read from Oauth2 server: %s", err) 145 | w.WriteHeader(http.StatusNotFound) 146 | return 147 | } 148 | 149 | relatedSourceIds := m.appNameTranslator.GetRelatedSourceIds(sourceIds, authToken) 150 | if relatedSourceIds == nil { 151 | w.WriteHeader(http.StatusNotFound) 152 | return 153 | } 154 | 155 | for _, sourceId := range sourceIds { 156 | sourceIdSet := append(relatedSourceIds[sourceId], sourceId) 157 | 158 | if !c.IsAdmin { 159 | sourceIdSet = m.authorizeSourceIds(sourceIdSet, c) 160 | 161 | if len(sourceIdSet) == 0 { 162 | w.WriteHeader(http.StatusNotFound) 163 | return 164 | } 165 | } 166 | 167 | relatedSourceIds[sourceId] = sourceIdSet 168 | } 169 | 170 | modifiedQuery, err := promql.ReplaceSourceIdSets(query, relatedSourceIds) 171 | if err != nil { 172 | log.Printf("failed to expand source IDs: %s", err) 173 | w.WriteHeader(http.StatusNotFound) 174 | return 175 | } 176 | 177 | q := r.URL.Query() 178 | q.Set("query", modifiedQuery) 179 | r.URL.RawQuery = q.Encode() 180 | 181 | h.ServeHTTP(w, r) 182 | }) 183 | 184 | router.HandleFunc("/api/v1/meta", func(w http.ResponseWriter, r *http.Request) { 185 | authToken := r.Header.Get("Authorization") 186 | if authToken == "" { 187 | w.WriteHeader(http.StatusNotFound) 188 | return 189 | } 190 | 191 | c, err := m.oauth2Reader.Read(authToken) 192 | if err != nil { 193 | log.Printf("failed to read from Oauth2 server: %s", err) 194 | w.WriteHeader(http.StatusNotFound) 195 | return 196 | } 197 | 198 | meta, err := m.metaFetcher.Meta(r.Context()) 199 | if err != nil { 200 | log.Printf("failed to fetch meta information: %s", err) 201 | w.WriteHeader(http.StatusBadGateway) 202 | return 203 | } 204 | 205 | // We don't care if writing to the client fails. They can come back and ask again. 206 | _ = m.marshaller.Marshal(w, &rpc.MetaResponse{ 207 | Meta: m.onlyAuthorized(authToken, meta, c), 208 | }) 209 | w.Write([]byte("\n")) 210 | }) 211 | 212 | router.HandleFunc("/api/v1/info", h.ServeHTTP) 213 | 214 | return router 215 | } 216 | 217 | func (m CFAuthMiddlewareProvider) authorizeSourceIds(sourceIds []string, c Oauth2ClientContext) []string { 218 | var authorizedSourceIds []string 219 | 220 | for _, sourceId := range sourceIds { 221 | if m.logAuthorizer.IsAuthorized(sourceId, c.Token) { 222 | authorizedSourceIds = append(authorizedSourceIds, sourceId) 223 | } 224 | } 225 | 226 | return authorizedSourceIds 227 | } 228 | 229 | func (m CFAuthMiddlewareProvider) onlyAuthorized(authToken string, meta map[string]*rpc.MetaInfo, c Oauth2ClientContext) map[string]*rpc.MetaInfo { 230 | if c.IsAdmin { 231 | return meta 232 | } 233 | 234 | authorized := m.logAuthorizer.AvailableSourceIDs(authToken) 235 | intersection := make(map[string]*rpc.MetaInfo) 236 | for _, id := range authorized { 237 | if v, ok := meta[id]; ok { 238 | intersection[id] = v 239 | } 240 | } 241 | 242 | return intersection 243 | } 244 | -------------------------------------------------------------------------------- /internal/blackbox/blackbox_suite_test.go: -------------------------------------------------------------------------------- 1 | package blackbox_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestBlackbox(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Blackbox Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/cache/log_cache_suite_test.go: -------------------------------------------------------------------------------- 1 | package cache_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestLogCache(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "LogCache Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/cache/memory_analyzer.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "runtime" 5 | 6 | "sync" 7 | 8 | "code.cloudfoundry.org/go-loggregator/metrics" 9 | "github.com/cloudfoundry/gosigar" 10 | ) 11 | 12 | // MemoryAnalyzer reports the available and total memory. 13 | type MemoryAnalyzer struct { 14 | // metrics 15 | setAvail metrics.Gauge 16 | setTotal metrics.Gauge 17 | setHeap metrics.Gauge 18 | 19 | // cache 20 | avail uint64 21 | total uint64 22 | heap uint64 23 | 24 | sync.Mutex 25 | } 26 | 27 | // NewMemoryAnalyzer creates and returns a new MemoryAnalyzer. 28 | func NewMemoryAnalyzer(m Metrics) *MemoryAnalyzer { 29 | return &MemoryAnalyzer{ 30 | setAvail: m.NewGauge("log_cache_available_system_memory", metrics.WithMetricTags(map[string]string{"unit":"bytes"})), 31 | setHeap: m.NewGauge("log_cache_heap_in_use_memory", metrics.WithMetricTags(map[string]string{"unit":"bytes"})), 32 | setTotal: m.NewGauge("log_cache_total_system_memory", metrics.WithMetricTags(map[string]string{"unit":"bytes"})), 33 | } 34 | } 35 | 36 | // Memory returns the heap memory and total system memory. 37 | func (a *MemoryAnalyzer) Memory() (heapInUse, available, total uint64) { 38 | a.Lock() 39 | defer a.Unlock() 40 | 41 | var m sigar.Mem 42 | m.Get() 43 | 44 | a.avail = m.ActualFree 45 | a.total = m.Total 46 | 47 | a.setAvail.Set(float64(m.ActualFree)) 48 | a.setTotal.Set(float64(m.Total)) 49 | 50 | var rm runtime.MemStats 51 | runtime.ReadMemStats(&rm) 52 | 53 | a.heap = rm.HeapInuse 54 | a.setHeap.Set(float64(a.heap)) 55 | 56 | return a.heap, a.avail, a.total 57 | } 58 | -------------------------------------------------------------------------------- /internal/cache/store/prune_consultant.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "code.cloudfoundry.org/go-loggregator/metrics" 4 | 5 | // PruneConsultant keeps track of the available memory on the system and tries 6 | // to utilize as much memory as possible while not being a bad neighbor. 7 | type PruneConsultant struct { 8 | m Memory 9 | 10 | percentToFill float64 11 | stepBy int 12 | reportMemory metrics.Gauge 13 | } 14 | 15 | // Memory is used to give information about system memory. 16 | type Memory interface { 17 | // Memory returns in-use heap memory and total system memory. 18 | Memory() (heap, avail, total uint64) 19 | } 20 | 21 | // NewPruneConsultant returns a new PruneConsultant. 22 | func NewPruneConsultant(stepBy int, percentToFill float64, m Memory) *PruneConsultant { 23 | return &PruneConsultant{ 24 | m: m, 25 | percentToFill: percentToFill, 26 | stepBy: stepBy, 27 | } 28 | } 29 | 30 | func (pc *PruneConsultant) SetMemoryReporter(mr metrics.Gauge) { 31 | pc.reportMemory = mr 32 | } 33 | 34 | // Prune reports how many entries should be removed. 35 | func (pc *PruneConsultant) GetQuantityToPrune(storeCount int64) int { 36 | heap, _, total := pc.m.Memory() 37 | 38 | heapPercentage := float64(heap*100) / float64(total) 39 | 40 | if pc.reportMemory != nil { 41 | pc.reportMemory.Set(heapPercentage) 42 | } 43 | 44 | if heapPercentage > pc.percentToFill { 45 | percentageToPrune := (heapPercentage - pc.percentToFill) / heapPercentage 46 | return int(float64(storeCount) * percentageToPrune) 47 | } 48 | 49 | return 0 50 | } 51 | -------------------------------------------------------------------------------- /internal/cache/store/prune_consultant_test.go: -------------------------------------------------------------------------------- 1 | package store_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/log-cache/internal/cache/store" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var _ = Describe("PruneConsultant", func() { 11 | var ( 12 | sm *spyMemory 13 | c *store.PruneConsultant 14 | storeCount int64 15 | ) 16 | 17 | BeforeEach(func() { 18 | sm = newSpyMemory() 19 | c = store.NewPruneConsultant(5, 70, sm) 20 | storeCount = 1000 21 | }) 22 | 23 | It("does not prune any entries if memory utilization is under allotment", func() { 24 | sm.avail = 30 25 | sm.heap = 70 26 | sm.total = 100 27 | 28 | Expect(c.GetQuantityToPrune(storeCount)).To(Equal(0)) 29 | }) 30 | 31 | It("prunes entries if memory utilization is over allotment", func() { 32 | sm.avail = 100 33 | sm.heap = 71 34 | sm.total = 100 35 | 36 | Expect(c.GetQuantityToPrune(storeCount)).To(Equal(14)) 37 | }) 38 | }) 39 | 40 | type spyMemory struct { 41 | heap, avail, total uint64 42 | } 43 | 44 | func newSpyMemory() *spyMemory { 45 | return &spyMemory{} 46 | } 47 | 48 | func (s *spyMemory) Memory() (uint64, uint64, uint64) { 49 | return s.heap, s.avail, s.total 50 | } 51 | -------------------------------------------------------------------------------- /internal/cache/store/store_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package store_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "context" 6 | "fmt" 7 | "math/rand" 8 | "testing" 9 | "time" 10 | 11 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 12 | "code.cloudfoundry.org/log-cache/internal/cache/store" 13 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 14 | ) 15 | 16 | const ( 17 | MaxPerSource = 1000000 18 | ) 19 | 20 | var ( 21 | MinTime = time.Unix(0, 0) 22 | MaxTime = time.Unix(0, 9223372036854775807) 23 | gen = randEnvGen() 24 | sourceIDs = []string{"0", "1", "2", "3", "4"} 25 | results []*loggregator_v2.Envelope 26 | metaResults map[string]logcache_v1.MetaInfo 27 | ) 28 | 29 | func BenchmarkStoreWrite(b *testing.B) { 30 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 31 | 32 | b.ResetTimer() 33 | for i := 0; i < b.N; i++ { 34 | e := gen() 35 | s.Put(e, e.GetSourceId()) 36 | } 37 | } 38 | 39 | func BenchmarkStoreTruncationOnWrite(b *testing.B) { 40 | s := store.NewStore(100, &staticPruner{}, nopMetrics{}) 41 | 42 | b.ResetTimer() 43 | for i := 0; i < b.N; i++ { 44 | e := gen() 45 | s.Put(e, e.GetSourceId()) 46 | } 47 | } 48 | 49 | func BenchmarkStoreWriteParallel(b *testing.B) { 50 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 51 | 52 | b.ResetTimer() 53 | 54 | b.RunParallel(func(pb *testing.PB) { 55 | for pb.Next() { 56 | e := gen() 57 | s.Put(e, e.GetSourceId()) 58 | } 59 | }) 60 | } 61 | 62 | func BenchmarkStoreGetTime5MinRange(b *testing.B) { 63 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 64 | 65 | for i := 0; i < MaxPerSource/10; i++ { 66 | e := gen() 67 | s.Put(e, e.GetSourceId()) 68 | } 69 | now := time.Now() 70 | fiveMinAgo := now.Add(-5 * time.Minute) 71 | 72 | b.ResetTimer() 73 | for i := 0; i < b.N; i++ { 74 | results = s.Get(sourceIDs[i%len(sourceIDs)], fiveMinAgo, now, nil, nil, b.N, false) 75 | } 76 | } 77 | 78 | func BenchmarkStoreGetLogType(b *testing.B) { 79 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 80 | 81 | for i := 0; i < MaxPerSource/10; i++ { 82 | e := gen() 83 | s.Put(e, e.GetSourceId()) 84 | } 85 | 86 | logType := []logcache_v1.EnvelopeType{logcache_v1.EnvelopeType_LOG} 87 | 88 | b.ResetTimer() 89 | for i := 0; i < b.N; i++ { 90 | results = s.Get(sourceIDs[i%len(sourceIDs)], MinTime, MaxTime, logType, nil, b.N, false) 91 | } 92 | } 93 | 94 | func BenchmarkMeta(b *testing.B) { 95 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 96 | 97 | for i := 0; i < b.N; i++ { 98 | e := gen() 99 | s.Put(e, e.GetSourceId()) 100 | } 101 | 102 | b.ResetTimer() 103 | for i := 0; i < b.N; i++ { 104 | metaResults = s.Meta() 105 | } 106 | } 107 | 108 | func BenchmarkMetaWhileWriting(b *testing.B) { 109 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 110 | 111 | ready := make(chan struct{}, 1) 112 | go func() { 113 | close(ready) 114 | for i := 0; i < b.N; i++ { 115 | e := gen() 116 | s.Put(e, e.GetSourceId()) 117 | } 118 | }() 119 | <-ready 120 | 121 | b.ResetTimer() 122 | for i := 0; i < b.N; i++ { 123 | metaResults = s.Meta() 124 | } 125 | } 126 | 127 | func BenchmarkMetaWhileReading(b *testing.B) { 128 | s := store.NewStore(MaxPerSource, &staticPruner{}, nopMetrics{}) 129 | 130 | for i := 0; i < b.N; i++ { 131 | e := gen() 132 | s.Put(e, e.GetSourceId()) 133 | } 134 | now := time.Now() 135 | fiveMinAgo := now.Add(-5 * time.Minute) 136 | ready := make(chan struct{}, 1) 137 | go func() { 138 | close(ready) 139 | for i := 0; i < b.N; i++ { 140 | results = s.Get(sourceIDs[i%len(sourceIDs)], fiveMinAgo, now, nil, nil, b.N, false) 141 | } 142 | }() 143 | <-ready 144 | 145 | b.ResetTimer() 146 | for i := 0; i < b.N; i++ { 147 | metaResults = s.Meta() 148 | } 149 | } 150 | 151 | func contextIsDone(ctx context.Context) bool { 152 | select { 153 | case <-ctx.Done(): 154 | return true 155 | default: 156 | return false 157 | } 158 | } 159 | 160 | func randEnvGen() func() *loggregator_v2.Envelope { 161 | var s []*loggregator_v2.Envelope 162 | fiveMinAgo := time.Now().Add(-5 * time.Minute) 163 | for i := 0; i < 10000; i++ { 164 | s = append(s, benchBuildLog( 165 | fmt.Sprintf("%d", i%len(sourceIDs)), 166 | fiveMinAgo.Add(time.Duration(i)*time.Millisecond).UnixNano(), 167 | )) 168 | } 169 | 170 | var i int 171 | return func() *loggregator_v2.Envelope { 172 | i++ 173 | return s[i%len(s)] 174 | } 175 | } 176 | 177 | func benchBuildLog(appID string, ts int64) *loggregator_v2.Envelope { 178 | return &loggregator_v2.Envelope{ 179 | SourceId: appID, 180 | // Timestamp: ts, 181 | Timestamp: time.Now().Add(time.Duration(rand.Int63n(50)-100) * time.Microsecond).UnixNano(), 182 | Message: &loggregator_v2.Envelope_Log{ 183 | Log: &loggregator_v2.Log{}, 184 | }, 185 | } 186 | } 187 | 188 | type nopMetrics struct{} 189 | 190 | type nopCounter struct {} 191 | func (nc *nopCounter) Add(float64) {} 192 | 193 | type nopGauge struct {} 194 | func (ng *nopGauge) Add(float64) {} 195 | func (ng *nopGauge) Set(float64) {} 196 | 197 | func (n nopMetrics) NewCounter(name string, opts ...metrics.MetricOption) metrics.Counter { 198 | return &nopCounter{} 199 | } 200 | 201 | func (n nopMetrics) NewGauge(name string, opts ...metrics.MetricOption) metrics.Gauge { 202 | return &nopGauge{} 203 | } 204 | 205 | type staticPruner struct { 206 | size int 207 | } 208 | 209 | func (s *staticPruner) GetQuantityToPrune(int64) int { 210 | return s.size 211 | } 212 | 213 | func (s *staticPruner) SetMemoryReporter(metrics.Gauge) {} 214 | -------------------------------------------------------------------------------- /internal/cache/store/store_load_test.go: -------------------------------------------------------------------------------- 1 | package store_test 2 | 3 | import ( 4 | 5 | "code.cloudfoundry.org/go-loggregator/metrics/testhelpers" 6 | "fmt" 7 | 8 | "sync" 9 | "sync/atomic" 10 | "time" 11 | 12 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 13 | "code.cloudfoundry.org/log-cache/internal/cache/store" 14 | . "github.com/onsi/ginkgo" 15 | . "github.com/onsi/gomega" 16 | ) 17 | 18 | var _ = Describe("store under high concurrent load", func() { 19 | timeoutInSeconds := 300 20 | 21 | It("", func(done Done) { 22 | var wg sync.WaitGroup 23 | 24 | sp := newSpyPruner() 25 | sp.numberToPrune = 128 26 | sm := testhelpers.NewMetricsRegistry() 27 | 28 | loadStore := store.NewStore(2500, sp, sm) 29 | start := time.Now() 30 | var envelopesWritten uint64 31 | 32 | // 10 writers per sourceId, 10k envelopes per writer 33 | for sourceId := 0; sourceId < 10; sourceId++ { 34 | for writers := 0; writers < 10; writers++ { 35 | wg.Add(1) 36 | go func(sourceId string) { 37 | defer wg.Done() 38 | 39 | for envelopes := 0; envelopes < 500; envelopes++ { 40 | e := buildTypedEnvelope(time.Now().UnixNano(), sourceId, &loggregator_v2.Log{}) 41 | loadStore.Put(e, sourceId) 42 | atomic.AddUint64(&envelopesWritten, 1) 43 | time.Sleep(50 * time.Microsecond) 44 | } 45 | }(fmt.Sprintf("index-%d", sourceId)) 46 | } 47 | } 48 | 49 | for readers := 0; readers < 10; readers++ { 50 | go func() { 51 | for i := 0; i < 100; i++ { 52 | loadStore.Meta() 53 | time.Sleep(10 * time.Millisecond) 54 | } 55 | }() 56 | } 57 | 58 | go func() { 59 | wg.Wait() 60 | // fmt.Printf("Finished writing %d envelopes in %s\n", atomic.LoadUint64(&envelopesWritten), time.Since(start)) 61 | close(done) 62 | }() 63 | 64 | Consistently(func() int64 { 65 | envelopes := loadStore.Get("index-9", start, time.Now(), nil, nil, 100000, false) 66 | return int64(len(envelopes)) 67 | }, timeoutInSeconds).Should(BeNumerically("<=", 2500)) 68 | 69 | }, float64(timeoutInSeconds)) 70 | }) 71 | -------------------------------------------------------------------------------- /internal/cache/store/store_suite_test.go: -------------------------------------------------------------------------------- 1 | package store_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestStore(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Store Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/cfauthproxy/cf_auth_proxy.go: -------------------------------------------------------------------------------- 1 | package cfauthproxy 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "log" 8 | "net" 9 | "net/http" 10 | "net/http/httputil" 11 | "net/url" 12 | "time" 13 | 14 | "code.cloudfoundry.org/log-cache/internal/auth" 15 | sharedtls "code.cloudfoundry.org/log-cache/internal/tls" 16 | ) 17 | 18 | type CFAuthProxy struct { 19 | blockOnStart bool 20 | ln net.Listener 21 | 22 | gatewayURL *url.URL 23 | addr string 24 | certPath string 25 | keyPath string 26 | proxyCACertPool *x509.CertPool 27 | 28 | authMiddleware func(http.Handler) http.Handler 29 | accessMiddleware func(http.Handler) *auth.AccessHandler 30 | } 31 | 32 | func NewCFAuthProxy(gatewayAddr, addr, certPath, keyPath string, proxyCACertPool *x509.CertPool, opts ...CFAuthProxyOption) *CFAuthProxy { 33 | gatewayURL, err := url.Parse(gatewayAddr) 34 | if err != nil { 35 | panic(fmt.Sprintf("Couldn't parse gateway address: %s", err)) 36 | } 37 | 38 | p := &CFAuthProxy{ 39 | gatewayURL: gatewayURL, 40 | addr: addr, 41 | certPath: certPath, 42 | keyPath: keyPath, 43 | proxyCACertPool: proxyCACertPool, 44 | authMiddleware: func(h http.Handler) http.Handler { 45 | return h 46 | }, 47 | accessMiddleware: auth.NewNullAccessMiddleware(), 48 | } 49 | 50 | for _, o := range opts { 51 | o(p) 52 | } 53 | 54 | return p 55 | } 56 | 57 | // CFAuthProxyOption configures a CFAuthProxy 58 | type CFAuthProxyOption func(*CFAuthProxy) 59 | 60 | // WithCFAuthProxyBlock returns a CFAuthProxyOption that determines if Start 61 | // launches a go-routine or not. It defaults to launching a go-routine. If 62 | // this is set, start will block on serving the HTTP endpoint. 63 | func WithCFAuthProxyBlock() CFAuthProxyOption { 64 | return func(p *CFAuthProxy) { 65 | p.blockOnStart = true 66 | } 67 | } 68 | 69 | // WithAuthMiddleware returns a CFAuthProxyOption that sets the CFAuthProxy's 70 | // authentication and authorization middleware. 71 | func WithAuthMiddleware(authMiddleware func(http.Handler) http.Handler) CFAuthProxyOption { 72 | return func(p *CFAuthProxy) { 73 | p.authMiddleware = authMiddleware 74 | } 75 | } 76 | 77 | func WithAccessMiddleware(accessMiddleware func(http.Handler) *auth.AccessHandler) CFAuthProxyOption { 78 | return func(p *CFAuthProxy) { 79 | p.accessMiddleware = accessMiddleware 80 | } 81 | } 82 | 83 | // Start starts the HTTP listener and serves the HTTP server. If the 84 | // CFAuthProxy was initialized with the WithCFAuthProxyBlock option this 85 | // method will block. 86 | func (p *CFAuthProxy) Start() { 87 | ln, err := net.Listen("tcp", p.addr) 88 | if err != nil { 89 | log.Fatalf("failed to start listener: %s", err) 90 | } 91 | 92 | p.ln = ln 93 | 94 | server := http.Server{ 95 | Handler: p.accessMiddleware(p.authMiddleware(p.reverseProxy())), 96 | TLSConfig: sharedtls.NewBaseTLSConfig(), 97 | } 98 | 99 | if p.blockOnStart { 100 | log.Fatal(server.ServeTLS(ln, p.certPath, p.keyPath)) 101 | } 102 | 103 | go func() { 104 | log.Fatal(server.ServeTLS(ln, p.certPath, p.keyPath)) 105 | }() 106 | } 107 | 108 | // Addr returns the listener address. This must be called after calling Start. 109 | func (p *CFAuthProxy) Addr() string { 110 | return p.ln.Addr().String() 111 | } 112 | 113 | func (p *CFAuthProxy) reverseProxy() *httputil.ReverseProxy { 114 | proxy := httputil.NewSingleHostReverseProxy(p.gatewayURL) 115 | proxy.Transport = NewTransportWithRootCA(p.proxyCACertPool) 116 | return proxy 117 | } 118 | 119 | func NewTransportWithRootCA(rootCACertPool *x509.CertPool) *http.Transport { 120 | // Aside from the Root CA for the gateway, these values are defaults 121 | // from Golang's http.DefaultTransport 122 | return &http.Transport{ 123 | Proxy: http.ProxyFromEnvironment, 124 | DialContext: (&net.Dialer{ 125 | Timeout: 30 * time.Second, 126 | KeepAlive: 30 * time.Second, 127 | DualStack: true, 128 | }).DialContext, 129 | MaxIdleConns: 100, 130 | IdleConnTimeout: 90 * time.Second, 131 | TLSHandshakeTimeout: 10 * time.Second, 132 | ExpectContinueTimeout: 1 * time.Second, 133 | TLSClientConfig: &tls.Config{ 134 | RootCAs: rootCACertPool, 135 | }, 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /internal/cfauthproxy/cf_auth_proxy_test.go: -------------------------------------------------------------------------------- 1 | package cfauthproxy_test 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "net/http/httptest" 11 | "net/url" 12 | 13 | "code.cloudfoundry.org/log-cache/internal/auth" 14 | . "code.cloudfoundry.org/log-cache/internal/cfauthproxy" 15 | "code.cloudfoundry.org/log-cache/internal/testing" 16 | 17 | . "github.com/onsi/ginkgo" 18 | . "github.com/onsi/gomega" 19 | ) 20 | 21 | var _ = Describe("CFAuthProxy", func() { 22 | It("only proxies requests to a secure log cache gateway", func() { 23 | gateway := startSecureGateway("Hello World!") 24 | defer gateway.Close() 25 | 26 | proxy := newCFAuthProxy(gateway.URL) 27 | proxy.Start() 28 | 29 | resp, err := makeTLSReq("https", proxy.Addr()) 30 | Expect(err).ToNot(HaveOccurred()) 31 | 32 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 33 | body, _ := ioutil.ReadAll(resp.Body) 34 | Expect(body).To(Equal([]byte("Hello World!"))) 35 | }) 36 | 37 | It("returns an error when proxying requests to an insecure log cache gateway", func() { 38 | // suppress tls error in test 39 | log.SetOutput(ioutil.Discard) 40 | 41 | testServer := httptest.NewServer( 42 | http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 43 | w.Write([]byte("Insecure Gateway not allowed, lol")) 44 | })) 45 | defer testServer.Close() 46 | 47 | testGatewayURL, _ := url.Parse(testServer.URL) 48 | testGatewayURL.Scheme = "https" 49 | 50 | proxy := newCFAuthProxy(testGatewayURL.String()) 51 | proxy.Start() 52 | 53 | resp, err := makeTLSReq("https", proxy.Addr()) 54 | Expect(err).ToNot(HaveOccurred()) 55 | 56 | Expect(resp.StatusCode).To(Equal(http.StatusBadGateway)) 57 | }) 58 | 59 | It("delegates to the auth middleware", func() { 60 | var middlewareCalled bool 61 | middleware := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 62 | middlewareCalled = true 63 | w.WriteHeader(http.StatusNotFound) 64 | }) 65 | 66 | proxy := newCFAuthProxy( 67 | "https://127.0.0.1", 68 | WithAuthMiddleware(func(http.Handler) http.Handler { 69 | return middleware 70 | }), 71 | ) 72 | proxy.Start() 73 | 74 | resp, err := makeTLSReq("https", proxy.Addr()) 75 | Expect(err).ToNot(HaveOccurred()) 76 | 77 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 78 | Expect(middlewareCalled).To(BeTrue()) 79 | }) 80 | 81 | It("delegates to the access middleware", func() { 82 | var middlewareCalled bool 83 | middleware := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 84 | middlewareCalled = true 85 | w.WriteHeader(http.StatusNotFound) 86 | }) 87 | 88 | proxy := newCFAuthProxy( 89 | "https://127.0.0.1", 90 | WithAccessMiddleware(func(http.Handler) *auth.AccessHandler { 91 | return auth.NewAccessHandler(middleware, auth.NewNullAccessLogger(), "0.0.0.0", "1234") 92 | }), 93 | ) 94 | proxy.Start() 95 | 96 | resp, err := makeTLSReq("https", proxy.Addr()) 97 | Expect(err).ToNot(HaveOccurred()) 98 | 99 | Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) 100 | Expect(middlewareCalled).To(BeTrue()) 101 | }) 102 | 103 | It("does not accept unencrypted connections", func() { 104 | testServer := httptest.NewTLSServer( 105 | http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {}), 106 | ) 107 | proxy := newCFAuthProxy(testServer.URL) 108 | proxy.Start() 109 | 110 | resp, err := makeTLSReq("http", proxy.Addr()) 111 | Expect(err).NotTo(HaveOccurred()) 112 | 113 | Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) 114 | }) 115 | }) 116 | 117 | func newCFAuthProxy(gatewayURL string, opts ...CFAuthProxyOption) *CFAuthProxy { 118 | proxyCACert, err := ioutil.ReadFile(testing.Cert("localhost.crt")) 119 | if err != nil { 120 | panic("couldn't parse proxyCACert") 121 | } 122 | 123 | proxyCACertPool := x509.NewCertPool() 124 | ok := proxyCACertPool.AppendCertsFromPEM(proxyCACert) 125 | if !ok { 126 | panic("couldn't create proxyCACertPool") 127 | } 128 | 129 | parsedURL, err := url.Parse(gatewayURL) 130 | if err != nil { 131 | panic("couldn't parse gateway URL") 132 | } 133 | 134 | return NewCFAuthProxy( 135 | parsedURL.String(), 136 | "127.0.0.1:0", 137 | testing.Cert("localhost.crt"), 138 | testing.Cert("localhost.key"), 139 | proxyCACertPool, 140 | opts..., 141 | ) 142 | } 143 | 144 | func startSecureGateway(responseBody string) *httptest.Server { 145 | testGateway := httptest.NewTLSServer( 146 | http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 147 | w.Write([]byte(responseBody)) 148 | }), 149 | ) 150 | return testGateway 151 | } 152 | 153 | func makeTLSReq(scheme, addr string) (*http.Response, error) { 154 | req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s://%s", scheme, addr), nil) 155 | Expect(err).ToNot(HaveOccurred()) 156 | 157 | tr := &http.Transport{ 158 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 159 | } 160 | client := &http.Client{Transport: tr} 161 | 162 | return client.Do(req) 163 | } 164 | -------------------------------------------------------------------------------- /internal/cfauthproxy/proxy_suite_test.go: -------------------------------------------------------------------------------- 1 | package cfauthproxy_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestProxy(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "CF Auth Proxy Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/end2end/end2end_suite_test.go: -------------------------------------------------------------------------------- 1 | package end2end_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestEnd2end(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "End2end Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/end2end/log_cache_test.go: -------------------------------------------------------------------------------- 1 | package end2end_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "context" 6 | "fmt" 7 | "log" 8 | "time" 9 | 10 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 11 | "code.cloudfoundry.org/log-cache/internal/cache" 12 | "code.cloudfoundry.org/log-cache/internal/scheduler" 13 | "code.cloudfoundry.org/log-cache/pkg/client" 14 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 15 | "google.golang.org/grpc" 16 | 17 | . "github.com/onsi/ginkgo" 18 | . "github.com/onsi/gomega" 19 | ) 20 | 21 | var _ = Describe("LogCache", func() { 22 | var ( 23 | lc_addrs []string 24 | node1 *cache.LogCache 25 | node2 *cache.LogCache 26 | lc_scheduler *scheduler.Scheduler 27 | lc_client *client.Client 28 | 29 | // run is used to set varying port numbers 30 | // it is incremented for each spec 31 | run int 32 | runIncBy = 3 33 | ) 34 | 35 | BeforeEach(func() { 36 | run++ 37 | lc_addrs = []string{ 38 | fmt.Sprintf("127.0.0.1:%d", 9999+(run*runIncBy)), 39 | fmt.Sprintf("127.0.0.1:%d", 10000+(run*runIncBy)), 40 | } 41 | 42 | logger := log.New(GinkgoWriter, "", 0) 43 | m := metrics.NewRegistry(logger) 44 | node1 = cache.New( 45 | m, 46 | logger, 47 | cache.WithAddr(lc_addrs[0]), 48 | cache.WithClustered(0, lc_addrs, grpc.WithInsecure()), 49 | ) 50 | 51 | node2 = cache.New( 52 | m, 53 | logger, 54 | cache.WithAddr(lc_addrs[1]), 55 | cache.WithClustered(1, lc_addrs, grpc.WithInsecure()), 56 | ) 57 | 58 | lc_scheduler = scheduler.NewScheduler( 59 | lc_addrs, 60 | scheduler.WithSchedulerInterval(50*time.Millisecond), 61 | ) 62 | 63 | node1.Start() 64 | node2.Start() 65 | lc_scheduler.Start() 66 | 67 | lc_client = client.NewClient(lc_addrs[0], client.WithViaGRPC(grpc.WithInsecure())) 68 | }) 69 | 70 | AfterEach(func() { 71 | node1.Close() 72 | node2.Close() 73 | }) 74 | 75 | It("reads data from Log Cache", func() { 76 | Eventually(func() []int64 { 77 | ic1, cleanup1 := ingressClient(node1.Addr()) 78 | defer cleanup1() 79 | 80 | ic2, cleanup2 := ingressClient(node2.Addr()) 81 | defer cleanup2() 82 | 83 | _, err := ic1.Send(context.Background(), &rpc.SendRequest{ 84 | Envelopes: &loggregator_v2.EnvelopeBatch{ 85 | Batch: []*loggregator_v2.Envelope{ 86 | {SourceId: "a", Timestamp: 1}, 87 | {SourceId: "a", Timestamp: 2}, 88 | {SourceId: "b", Timestamp: 3}, 89 | {SourceId: "b", Timestamp: 4}, 90 | {SourceId: "c", Timestamp: 5}, 91 | }, 92 | }, 93 | }) 94 | 95 | Expect(err).ToNot(HaveOccurred()) 96 | 97 | _, err = ic2.Send(context.Background(), &rpc.SendRequest{ 98 | Envelopes: &loggregator_v2.EnvelopeBatch{ 99 | Batch: []*loggregator_v2.Envelope{ 100 | {SourceId: "a", Timestamp: 1000000006}, 101 | {SourceId: "a", Timestamp: 1000000007}, 102 | {SourceId: "b", Timestamp: 1000000008}, 103 | {SourceId: "b", Timestamp: 1000000009}, 104 | {SourceId: "c", Timestamp: 1000000010}, 105 | }, 106 | }, 107 | }) 108 | 109 | Expect(err).ToNot(HaveOccurred()) 110 | es, err := lc_client.Read(context.Background(), "a", time.Unix(0, 0), client.WithLimit(500)) 111 | if err != nil { 112 | return nil 113 | } 114 | 115 | var result []int64 116 | for _, e := range es { 117 | result = append(result, e.GetTimestamp()) 118 | } 119 | return result 120 | 121 | }, 5).Should(And( 122 | ContainElement(int64(1)), 123 | ContainElement(int64(2)), 124 | ContainElement(int64(1000000006)), 125 | ContainElement(int64(1000000007)), 126 | )) 127 | }) 128 | }) 129 | 130 | func ingressClient(addr string) (client rpc.IngressClient, cleanup func()) { 131 | conn, err := grpc.Dial(addr, grpc.WithInsecure()) 132 | if err != nil { 133 | panic(err) 134 | } 135 | 136 | return rpc.NewIngressClient(conn), func() { 137 | conn.Close() 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /internal/gateway/gateway.go: -------------------------------------------------------------------------------- 1 | package gateway 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net" 9 | "net/http" 10 | 11 | "github.com/grpc-ecosystem/grpc-gateway/runtime" 12 | "github.com/shirou/gopsutil/host" 13 | "golang.org/x/net/context" 14 | "google.golang.org/grpc" 15 | 16 | "code.cloudfoundry.org/log-cache/pkg/marshaler" 17 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 18 | ) 19 | 20 | // Gateway provides a RESTful API into LogCache's gRPC API. 21 | type Gateway struct { 22 | log *log.Logger 23 | 24 | logCacheAddr string 25 | logCacheVersion string 26 | uptimeFn func() int64 27 | 28 | gatewayAddr string 29 | lis net.Listener 30 | blockOnStart bool 31 | logCacheDialOpts []grpc.DialOption 32 | certPath string 33 | keyPath string 34 | } 35 | 36 | // NewGateway creates a new Gateway. It will listen on the gatewayAddr and 37 | // submit requests via gRPC to the LogCache on logCacheAddr. Start() must be 38 | // invoked before using the Gateway. 39 | func NewGateway(logCacheAddr, gatewayAddr, certPath, keyPath string, opts ...GatewayOption) *Gateway { 40 | g := &Gateway{ 41 | log: log.New(ioutil.Discard, "", 0), 42 | logCacheAddr: logCacheAddr, 43 | gatewayAddr: gatewayAddr, 44 | uptimeFn: uptimeInSeconds, 45 | certPath: certPath, 46 | keyPath: keyPath, 47 | } 48 | 49 | for _, o := range opts { 50 | o(g) 51 | } 52 | 53 | return g 54 | } 55 | 56 | // GatewayOption configures a Gateway. 57 | type GatewayOption func(*Gateway) 58 | 59 | // WithGatewayLogger returns a GatewayOption that configures the logger for 60 | // the Gateway. It defaults to no logging. 61 | func WithGatewayLogger(l *log.Logger) GatewayOption { 62 | return func(g *Gateway) { 63 | g.log = l 64 | } 65 | } 66 | 67 | // WithGatewayBlock returns a GatewayOption that determines if Start launches 68 | // a go-routine or not. It defaults to launching a go-routine. If this is set, 69 | // start will block on serving the HTTP endpoint. 70 | func WithGatewayBlock() GatewayOption { 71 | return func(g *Gateway) { 72 | g.blockOnStart = true 73 | } 74 | } 75 | 76 | // WithGatewayLogCacheDialOpts returns a GatewayOption that sets grpc.DialOptions on the 77 | // log-cache dial 78 | func WithGatewayLogCacheDialOpts(opts ...grpc.DialOption) GatewayOption { 79 | return func(g *Gateway) { 80 | g.logCacheDialOpts = opts 81 | } 82 | } 83 | 84 | // WithGatewayLogCacheDialOpts returns a GatewayOption that the log-cache 85 | // version returned by the info endpoint. 86 | func WithGatewayVersion(version string) GatewayOption { 87 | return func(g *Gateway) { 88 | g.logCacheVersion = version 89 | } 90 | } 91 | 92 | // WithGatewayLogCacheDialOpts returns a GatewayOption that the log-cache 93 | // version returned by the info endpoint. 94 | func WithGatewayVMUptimeFn(uptimeFn func() int64) GatewayOption { 95 | return func(g *Gateway) { 96 | g.uptimeFn = uptimeFn 97 | } 98 | } 99 | 100 | // Start starts the gateway to start receiving and forwarding requests. It 101 | // does not block unless WithGatewayBlock was set. 102 | func (g *Gateway) Start() { 103 | lis, err := net.Listen("tcp", g.gatewayAddr) 104 | if err != nil { 105 | g.log.Fatalf("failed to listen on addr %s: %s", g.gatewayAddr, err) 106 | } 107 | g.lis = lis 108 | g.log.Printf("listening on %s...", lis.Addr().String()) 109 | 110 | if g.blockOnStart { 111 | g.listenAndServe() 112 | return 113 | } 114 | 115 | go g.listenAndServe() 116 | } 117 | 118 | // Addr returns the address the gateway is listening on. Start must be called 119 | // first. 120 | func (g *Gateway) Addr() string { 121 | return g.lis.Addr().String() 122 | } 123 | 124 | func (g *Gateway) listenAndServe() { 125 | mux := runtime.NewServeMux( 126 | runtime.WithMarshalerOption( 127 | runtime.MIMEWildcard, marshaler.NewPromqlMarshaler(&runtime.JSONPb{OrigName: true, EmitDefaults: true}), 128 | ), 129 | ) 130 | 131 | runtime.HTTPError = g.httpErrorHandler 132 | 133 | conn, err := grpc.Dial(g.logCacheAddr, g.logCacheDialOpts...) 134 | if err != nil { 135 | g.log.Fatalf("failed to dial Log Cache: %s", err) 136 | } 137 | 138 | err = logcache_v1.RegisterEgressHandlerClient( 139 | context.Background(), 140 | mux, 141 | logcache_v1.NewEgressClient(conn), 142 | ) 143 | if err != nil { 144 | g.log.Fatalf("failed to register LogCache handler: %s", err) 145 | } 146 | 147 | err = logcache_v1.RegisterPromQLQuerierHandlerClient( 148 | context.Background(), 149 | mux, 150 | logcache_v1.NewPromQLQuerierClient(conn), 151 | ) 152 | if err != nil { 153 | g.log.Fatalf("failed to register PromQLQuerier handler: %s", err) 154 | } 155 | 156 | topLevelMux := http.NewServeMux() 157 | topLevelMux.HandleFunc("/api/v1/info", g.handleInfoEndpoint) 158 | topLevelMux.Handle("/", mux) 159 | 160 | server := &http.Server{Handler: topLevelMux} 161 | if err := server.ServeTLS(g.lis, g.certPath, g.keyPath); err != nil { 162 | g.log.Fatalf("failed to serve HTTPS endpoint: %s", err) 163 | } 164 | } 165 | 166 | func (g *Gateway) handleInfoEndpoint(w http.ResponseWriter, r *http.Request) { 167 | w.Write([]byte(fmt.Sprintf(`{"version":"%s","vm_uptime":"%d"}`+"\n", g.logCacheVersion, g.uptimeFn()))) 168 | } 169 | 170 | func uptimeInSeconds() int64 { 171 | hostStats, _ := host.Info() 172 | return int64(hostStats.Uptime) 173 | } 174 | 175 | type errorBody struct { 176 | Status string `json:"status"` 177 | ErrorType string `json:"errorType"` 178 | Error string `json:"error"` 179 | } 180 | 181 | func (g *Gateway) httpErrorHandler( 182 | ctx context.Context, 183 | mux *runtime.ServeMux, 184 | marshaler runtime.Marshaler, 185 | w http.ResponseWriter, 186 | r *http.Request, 187 | err error, 188 | ) { 189 | if r.URL.Path != "/api/v1/query" && r.URL.Path != "/api/v1/query_range" { 190 | runtime.DefaultHTTPError(ctx, mux, marshaler, w, r, err) 191 | return 192 | } 193 | 194 | const fallback = `{"error": "failed to marshal error message"}` 195 | 196 | w.Header().Del("Trailer") 197 | w.Header().Set("Content-Type", marshaler.ContentType()) 198 | 199 | body := &errorBody{ 200 | Status: "error", 201 | ErrorType: "internal", 202 | Error: grpc.ErrorDesc(err), 203 | } 204 | 205 | buf, merr := marshaler.Marshal(body) 206 | if merr != nil { 207 | g.log.Printf("Failed to marshal error message %q: %v", body, merr) 208 | w.WriteHeader(http.StatusInternalServerError) 209 | if _, err := io.WriteString(w, fallback); err != nil { 210 | g.log.Printf("Failed to write response: %v", err) 211 | } 212 | return 213 | } 214 | 215 | w.WriteHeader(runtime.HTTPStatusFromCode(grpc.Code(err))) 216 | if _, err := w.Write(buf); err != nil { 217 | g.log.Printf("Failed to write response: %v", err) 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /internal/gateway/gateway_suite_test.go: -------------------------------------------------------------------------------- 1 | package gateway_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestGateway(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Gateway Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/gateway/gateway_test.go: -------------------------------------------------------------------------------- 1 | package gateway_test 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "strings" 10 | 11 | . "code.cloudfoundry.org/log-cache/internal/gateway" 12 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials" 15 | 16 | "code.cloudfoundry.org/log-cache/internal/testing" 17 | . "github.com/onsi/ginkgo" 18 | . "github.com/onsi/ginkgo/extensions/table" 19 | . "github.com/onsi/gomega" 20 | ) 21 | 22 | var _ = Describe("Gateway", func() { 23 | var ( 24 | spyLogCache *testing.SpyLogCache 25 | gw *Gateway 26 | ) 27 | 28 | BeforeEach(func() { 29 | tlsConfig, err := testing.NewTLSConfig( 30 | testing.Cert("log-cache-ca.crt"), 31 | testing.Cert("log-cache.crt"), 32 | testing.Cert("log-cache.key"), 33 | "log-cache", 34 | ) 35 | Expect(err).ToNot(HaveOccurred()) 36 | 37 | spyLogCache = testing.NewSpyLogCache(tlsConfig) 38 | logCacheAddr := spyLogCache.Start() 39 | 40 | gw = NewGateway( 41 | logCacheAddr, 42 | "localhost:0", 43 | testing.Cert("localhost.crt"), 44 | testing.Cert("localhost.key"), 45 | WithGatewayLogCacheDialOpts( 46 | grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), 47 | ), 48 | WithGatewayVersion("1.2.3"), 49 | WithGatewayVMUptimeFn(testing.StubUptimeFn), 50 | ) 51 | gw.Start() 52 | }) 53 | 54 | DescribeTable("upgrades HTTPS requests for LogCache into gRPC requests", func(pathSourceID, expectedSourceID string) { 55 | path := fmt.Sprintf("api/v1/read/%s?start_time=99&end_time=101&limit=103&envelope_types=LOG&envelope_types=GAUGE", pathSourceID) 56 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 57 | resp, err := makeTLSReq("https", URL) 58 | Expect(err).ToNot(HaveOccurred()) 59 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 60 | 61 | reqs := spyLogCache.GetReadRequests() 62 | Expect(reqs).To(HaveLen(1)) 63 | Expect(reqs[0].SourceId).To(Equal(expectedSourceID)) 64 | Expect(reqs[0].StartTime).To(Equal(int64(99))) 65 | Expect(reqs[0].EndTime).To(Equal(int64(101))) 66 | Expect(reqs[0].Limit).To(Equal(int64(103))) 67 | Expect(reqs[0].EnvelopeTypes).To(ConsistOf(rpc.EnvelopeType_LOG, rpc.EnvelopeType_GAUGE)) 68 | }, 69 | Entry("URL encoded", "some-source%2Fid", "some-source/id"), 70 | Entry("with slash", "some-source/id", "some-source/id"), 71 | Entry("with dash", "some-source-id", "some-source-id"), 72 | ) 73 | 74 | It("adds newlines to the end of HTTPS responses", func() { 75 | path := `api/v1/meta` 76 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 77 | resp, err := makeTLSReq("https", URL) 78 | Expect(err).ToNot(HaveOccurred()) 79 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 80 | 81 | respBytes, err := ioutil.ReadAll(resp.Body) 82 | Expect(string(respBytes)).To(MatchRegexp(`\n$`)) 83 | }) 84 | 85 | It("upgrades HTTPS requests for instant queries via PromQLQuerier GETs into gRPC requests", func() { 86 | path := `api/v1/query?query=metric{source_id="some-id"}&time=1234.000` 87 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 88 | resp, err := makeTLSReq("https", URL) 89 | Expect(err).ToNot(HaveOccurred()) 90 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 91 | 92 | reqs := spyLogCache.GetQueryRequests() 93 | Expect(reqs).To(HaveLen(1)) 94 | Expect(reqs[0].Query).To(Equal(`metric{source_id="some-id"}`)) 95 | Expect(reqs[0].Time).To(Equal("1234.000")) 96 | }) 97 | 98 | It("upgrades HTTPS requests for range queries via PromQLQuerier GETs into gRPC requests", func() { 99 | path := `api/v1/query_range?query=metric{source_id="some-id"}&start=1234.000&end=5678.000&step=30s` 100 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 101 | resp, err := makeTLSReq("https", URL) 102 | Expect(err).ToNot(HaveOccurred()) 103 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 104 | 105 | reqs := spyLogCache.GetRangeQueryRequests() 106 | Expect(reqs).To(HaveLen(1)) 107 | Expect(reqs[0].Query).To(Equal(`metric{source_id="some-id"}`)) 108 | Expect(reqs[0].Start).To(Equal("1234.000")) 109 | Expect(reqs[0].End).To(Equal("5678.000")) 110 | Expect(reqs[0].Step).To(Equal("30s")) 111 | }) 112 | 113 | It("outputs json with zero-value points and correct Prometheus API fields", func() { 114 | path := `api/v1/query?query=metric{source_id="some-id"}&time=1234` 115 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 116 | spyLogCache.SetValue(0) 117 | 118 | resp, err := makeTLSReq("https", URL) 119 | Expect(err).ToNot(HaveOccurred()) 120 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 121 | 122 | body, _ := ioutil.ReadAll(resp.Body) 123 | Expect(body).To(MatchJSON(`{"status":"success","data":{"resultType":"scalar","result":[99,"0"]}}`)) 124 | }) 125 | 126 | It("returns version information from an info endpoint", func() { 127 | path := `api/v1/info` 128 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 129 | resp, err := makeTLSReq("https", URL) 130 | Expect(err).ToNot(HaveOccurred()) 131 | Expect(resp.StatusCode).To(Equal(http.StatusOK)) 132 | 133 | respBytes, err := ioutil.ReadAll(resp.Body) 134 | Expect(err).ToNot(HaveOccurred()) 135 | Expect(respBytes).To(MatchJSON( 136 | `{ 137 | "version":"1.2.3", 138 | "vm_uptime":"789" 139 | }`)) 140 | Expect(strings.HasSuffix(string(respBytes), "\n")).To(BeTrue()) 141 | }) 142 | 143 | It("does not accept unencrypted connections", func() { 144 | resp, err := makeTLSReq("http", fmt.Sprintf("%s/api/v1/info", gw.Addr())) 145 | Expect(err).NotTo(HaveOccurred()) 146 | Expect(resp.StatusCode).To(Equal(http.StatusBadRequest)) 147 | }) 148 | 149 | Context("errors", func() { 150 | It("passes through content-type correctly on errors", func() { 151 | path := `api/v1/query?query=metric{source_id="some-id"}&time=1234` 152 | spyLogCache.QueryError = errors.New("expected error") 153 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 154 | 155 | resp, err := makeTLSReq("https", URL) 156 | Expect(err).ToNot(HaveOccurred()) 157 | Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) 158 | Expect(resp.Header).To(HaveKeyWithValue("Content-Type", []string{"application/json"})) 159 | }) 160 | 161 | It("adds necessary fields to match Prometheus API", func() { 162 | path := `api/v1/query?query=metric{source_id="some-id"}&time=1234` 163 | spyLogCache.QueryError = errors.New("expected error") 164 | URL := fmt.Sprintf("%s/%s", gw.Addr(), path) 165 | 166 | resp, err := makeTLSReq("https", URL) 167 | Expect(err).ToNot(HaveOccurred()) 168 | Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) 169 | 170 | body, _ := ioutil.ReadAll(resp.Body) 171 | Expect(body).To(MatchJSON(`{ 172 | "status": "error", 173 | 174 | "errorType": "internal", 175 | "error": "expected error" 176 | }`)) 177 | }) 178 | }) 179 | }) 180 | 181 | func makeTLSReq(scheme, addr string) (*http.Response, error) { 182 | req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s://%s", scheme, addr), nil) 183 | Expect(err).ToNot(HaveOccurred()) 184 | 185 | tr := &http.Transport{ 186 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 187 | } 188 | client := &http.Client{Transport: tr} 189 | 190 | return client.Do(req) 191 | } 192 | -------------------------------------------------------------------------------- /internal/matchers/contain_metric.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/onsi/gomega/types" 7 | "github.com/prometheus/client_golang/prometheus" 8 | io_prometheus_client "github.com/prometheus/client_model/go" 9 | ) 10 | 11 | func ContainCounterMetric(expectedName string, expectedValue float64) types.GomegaMatcher { 12 | return &containMetricMatcher{ 13 | expectedName: expectedName, 14 | expectedValue: expectedValue, 15 | valueExtractor: func(metric *io_prometheus_client.Metric) float64 { 16 | return metric.GetCounter().GetValue() 17 | }, 18 | } 19 | } 20 | 21 | func ContainGaugeMetric(expectedName, expectedUnit string, expectedValue float64) types.GomegaMatcher { 22 | return &containMetricMatcher{ 23 | expectedName: expectedName, 24 | expectedValue: expectedValue, 25 | expectedUnit: expectedUnit, 26 | valueExtractor: func(metric *io_prometheus_client.Metric) float64 { 27 | return metric.GetGauge().GetValue() 28 | }, 29 | } 30 | } 31 | 32 | type containMetricMatcher struct { 33 | expectedName string 34 | expectedValue float64 35 | expectedUnit string 36 | valueExtractor func(*io_prometheus_client.Metric) float64 37 | } 38 | 39 | func (matcher *containMetricMatcher) Match(actual interface{}) (success bool, err error) { 40 | metrics, err := normalizeMetrics(actual) 41 | if err != nil { 42 | return false, err 43 | } 44 | 45 | for _, metricFamily := range metrics { 46 | if metricFamily.GetName() != matcher.expectedName { 47 | continue 48 | } 49 | for _, metric := range metricFamily.GetMetric() { 50 | if matcher.valueExtractor(metric) != matcher.expectedValue { 51 | continue 52 | } 53 | 54 | if matcher.expectedUnit != "" { 55 | for _, labelPair := range metric.GetLabel() { 56 | if labelPair.GetName() == "unit" && labelPair.GetValue() == matcher.expectedUnit { 57 | return true, nil 58 | } 59 | } 60 | 61 | continue 62 | } 63 | 64 | return true, nil 65 | } 66 | } 67 | 68 | return false, nil 69 | } 70 | 71 | func (matcher *containMetricMatcher) FailureMessage(actual interface{}) (message string) { 72 | metrics, err := normalizeMetrics(actual) 73 | if err != nil { 74 | return fmt.Sprintf("Expected registry to contain the metric\n\t%s: %f", matcher.expectedName, matcher.expectedValue) 75 | } 76 | 77 | var actualOutput string 78 | for _, a := range metrics { 79 | actualOutput += fmt.Sprintf("\t%+v\n", a) 80 | } 81 | expectedOutput := fmt.Sprintf("\tname: %s, value: %f", matcher.expectedName, matcher.expectedValue) 82 | if matcher.expectedUnit != "" { 83 | expectedOutput += fmt.Sprintf(", unit: %s", matcher.expectedUnit) 84 | } 85 | expectedOutput += "\n" 86 | return fmt.Sprintf("Expected\n%s\nto contain the metric \n%s", actualOutput, expectedOutput) 87 | } 88 | 89 | func (matcher *containMetricMatcher) NegatedFailureMessage(actual interface{}) (message string) { 90 | metrics, err := normalizeMetrics(actual) 91 | if err != nil { 92 | return fmt.Sprintf("Expected registry not to contain the metric\n\t%s: %f", matcher.expectedName, matcher.expectedValue) 93 | } 94 | 95 | var actualOutput string 96 | for _, a := range metrics { 97 | actualOutput += fmt.Sprintf("\t%+v\n", a) 98 | } 99 | expectedOutput := fmt.Sprintf("\tname: %s, value: %f", matcher.expectedName, matcher.expectedValue) 100 | if matcher.expectedUnit != "" { 101 | expectedOutput += fmt.Sprintf(", unit: %s", matcher.expectedUnit) 102 | } 103 | expectedOutput += "\n" 104 | return fmt.Sprintf("Expected\n%s\nnot to contain the metric \n%s", actualOutput, expectedOutput) 105 | } 106 | 107 | func normalizeMetrics(actual interface{}) ([]*io_prometheus_client.MetricFamily, error) { 108 | var metrics []*io_prometheus_client.MetricFamily 109 | var err error 110 | 111 | switch v := actual.(type) { 112 | case *prometheus.Registry: 113 | metrics, err = v.Gather() 114 | if err != nil { 115 | return metrics, err 116 | } 117 | 118 | case map[string]*io_prometheus_client.MetricFamily: 119 | for _, m := range v { 120 | metrics = append(metrics, m) 121 | } 122 | 123 | default: 124 | return metrics, fmt.Errorf("unexpected actual: %+v", actual) 125 | } 126 | 127 | return metrics, nil 128 | } 129 | -------------------------------------------------------------------------------- /internal/nozzle/nozzle.go: -------------------------------------------------------------------------------- 1 | package nozzle 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "log" 6 | "runtime" 7 | "time" 8 | 9 | diodes "code.cloudfoundry.org/go-diodes" 10 | "code.cloudfoundry.org/go-loggregator" 11 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 12 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 13 | "golang.org/x/net/context" 14 | "google.golang.org/grpc" 15 | ) 16 | 17 | type Metrics interface { 18 | NewCounter(name string, opts ...metrics.MetricOption) metrics.Counter 19 | NewGauge(name string, opts ...metrics.MetricOption) metrics.Gauge 20 | } 21 | 22 | // Nozzle reads envelopes and writes them to LogCache. 23 | type Nozzle struct { 24 | log *log.Logger 25 | s StreamConnector 26 | metrics Metrics 27 | shardId string 28 | selectors []string 29 | streamBuffer *diodes.OneToOne 30 | 31 | ingressInc metrics.Counter 32 | egressInc metrics.Counter 33 | errInc metrics.Counter 34 | 35 | // LogCache 36 | addr string 37 | opts []grpc.DialOption 38 | } 39 | 40 | const ( 41 | BATCH_FLUSH_INTERVAL = 500 * time.Millisecond 42 | BATCH_CHANNEL_SIZE = 512 43 | ) 44 | 45 | // StreamConnector reads envelopes from the the logs provider. 46 | type StreamConnector interface { 47 | // Stream creates a EnvelopeStream for the given request. 48 | Stream(ctx context.Context, req *loggregator_v2.EgressBatchRequest) loggregator.EnvelopeStream 49 | } 50 | 51 | // NewNozzle creates a new Nozzle. 52 | func NewNozzle(c StreamConnector, logCacheAddr, shardId string, m Metrics, logger *log.Logger, opts ...NozzleOption) *Nozzle { 53 | n := &Nozzle{ 54 | s: c, 55 | addr: logCacheAddr, 56 | opts: []grpc.DialOption{grpc.WithInsecure()}, 57 | log: logger, 58 | metrics: m, 59 | shardId: shardId, 60 | selectors: []string{}, 61 | } 62 | 63 | for _, o := range opts { 64 | o(n) 65 | } 66 | 67 | n.streamBuffer = diodes.NewOneToOne(100000, diodes.AlertFunc(func(missed int) { 68 | n.log.Printf("stream buffer dropped %d points", missed) 69 | })) 70 | 71 | return n 72 | } 73 | 74 | // NozzleOption configures a Nozzle. 75 | type NozzleOption func(*Nozzle) 76 | 77 | // WithDialOpts returns a NozzleOption that configures the dial options 78 | // for dialing the LogCache. It defaults to grpc.WithInsecure(). 79 | func WithDialOpts(opts ...grpc.DialOption) NozzleOption { 80 | return func(n *Nozzle) { 81 | n.opts = opts 82 | } 83 | } 84 | 85 | func WithSelectors(selectors ...string) NozzleOption { 86 | return func(n *Nozzle) { 87 | n.selectors = selectors 88 | } 89 | } 90 | 91 | // Start starts reading envelopes from the logs provider and writes them to 92 | // LogCache. It blocks indefinitely. 93 | func (n *Nozzle) Start() { 94 | rx := n.s.Stream(context.Background(), n.buildBatchReq()) 95 | 96 | conn, err := grpc.Dial(n.addr, n.opts...) 97 | if err != nil { 98 | log.Fatalf("failed to dial %s: %s", n.addr, err) 99 | } 100 | client := logcache_v1.NewIngressClient(conn) 101 | 102 | n.ingressInc = n.metrics.NewCounter("nozzle_ingress") 103 | n.egressInc = n.metrics.NewCounter("nozzle_egress") 104 | n.errInc = n.metrics.NewCounter("nozzle_err") 105 | 106 | go n.envelopeReader(rx) 107 | 108 | ch := make(chan []*loggregator_v2.Envelope, BATCH_CHANNEL_SIZE) 109 | 110 | log.Printf("Starting %d nozzle workers...", 2*runtime.NumCPU()) 111 | for i := 0; i < 2*runtime.NumCPU(); i++ { 112 | go n.envelopeWriter(ch, client) 113 | } 114 | 115 | // The batcher will block indefinitely. 116 | n.envelopeBatcher(ch) 117 | } 118 | 119 | func (n *Nozzle) envelopeBatcher(ch chan []*loggregator_v2.Envelope) { 120 | poller := diodes.NewPoller(n.streamBuffer) 121 | envelopes := make([]*loggregator_v2.Envelope, 0) 122 | t := time.NewTimer(BATCH_FLUSH_INTERVAL) 123 | for { 124 | data, found := poller.TryNext() 125 | 126 | if found { 127 | envelopes = append(envelopes, (*loggregator_v2.Envelope)(data)) 128 | } 129 | 130 | select { 131 | case <-t.C: 132 | if len(envelopes) > 0 { 133 | select { 134 | case ch <- envelopes: 135 | envelopes = make([]*loggregator_v2.Envelope, 0) 136 | default: 137 | // if we can't write into the channel, it must be full, so 138 | // we probably need to drop these envelopes on the floor 139 | envelopes = envelopes[:0] 140 | } 141 | } 142 | t.Reset(BATCH_FLUSH_INTERVAL) 143 | default: 144 | if len(envelopes) >= BATCH_CHANNEL_SIZE { 145 | select { 146 | case ch <- envelopes: 147 | envelopes = make([]*loggregator_v2.Envelope, 0) 148 | default: 149 | envelopes = envelopes[:0] 150 | } 151 | t.Reset(BATCH_FLUSH_INTERVAL) 152 | } 153 | if !found { 154 | time.Sleep(time.Millisecond) 155 | } 156 | } 157 | } 158 | } 159 | 160 | func (n *Nozzle) envelopeWriter(ch chan []*loggregator_v2.Envelope, client logcache_v1.IngressClient) { 161 | for { 162 | envelopes := <-ch 163 | 164 | ctx, _ := context.WithTimeout(context.Background(), 3*time.Second) 165 | _, err := client.Send(ctx, &logcache_v1.SendRequest{ 166 | Envelopes: &loggregator_v2.EnvelopeBatch{ 167 | Batch: envelopes, 168 | }, 169 | }) 170 | 171 | if err != nil { 172 | n.errInc.Add(1) 173 | continue 174 | } 175 | 176 | n.egressInc.Add(float64(len(envelopes))) 177 | } 178 | } 179 | 180 | func (n *Nozzle) envelopeReader(rx loggregator.EnvelopeStream) { 181 | for { 182 | envelopeBatch := rx() 183 | for _, envelope := range envelopeBatch { 184 | n.streamBuffer.Set(diodes.GenericDataType(envelope)) 185 | n.ingressInc.Add(1) 186 | } 187 | } 188 | } 189 | 190 | var selectorTypes = map[string]*loggregator_v2.Selector{ 191 | "log": { 192 | Message: &loggregator_v2.Selector_Log{ 193 | Log: &loggregator_v2.LogSelector{}, 194 | }, 195 | }, 196 | "gauge": { 197 | Message: &loggregator_v2.Selector_Gauge{ 198 | Gauge: &loggregator_v2.GaugeSelector{}, 199 | }, 200 | }, 201 | "counter": { 202 | Message: &loggregator_v2.Selector_Counter{ 203 | Counter: &loggregator_v2.CounterSelector{}, 204 | }, 205 | }, 206 | "timer": { 207 | Message: &loggregator_v2.Selector_Timer{ 208 | Timer: &loggregator_v2.TimerSelector{}, 209 | }, 210 | }, 211 | "event": { 212 | Message: &loggregator_v2.Selector_Event{ 213 | Event: &loggregator_v2.EventSelector{}, 214 | }, 215 | }, 216 | } 217 | 218 | func (n *Nozzle) buildBatchReq() *loggregator_v2.EgressBatchRequest { 219 | var selectors []*loggregator_v2.Selector 220 | 221 | for _, selectorType := range n.selectors { 222 | selectors = append(selectors, selectorTypes[selectorType]) 223 | } 224 | 225 | return &loggregator_v2.EgressBatchRequest{ 226 | ShardId: n.shardId, 227 | UsePreferredTags: true, 228 | Selectors: selectors, 229 | } 230 | } 231 | -------------------------------------------------------------------------------- /internal/nozzle/nozzle_suite_test.go: -------------------------------------------------------------------------------- 1 | package nozzle_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestNozzle(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Nozzle Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/nozzle/nozzle_test.go: -------------------------------------------------------------------------------- 1 | package nozzle_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics/testhelpers" 5 | "log" 6 | "sync" 7 | 8 | "code.cloudfoundry.org/go-loggregator" 9 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 10 | . "code.cloudfoundry.org/log-cache/internal/nozzle" 11 | "golang.org/x/net/context" 12 | "google.golang.org/grpc" 13 | "google.golang.org/grpc/credentials" 14 | 15 | "code.cloudfoundry.org/log-cache/internal/testing" 16 | . "github.com/onsi/ginkgo" 17 | . "github.com/onsi/gomega" 18 | ) 19 | 20 | var _ = Describe("Nozzle", func() { 21 | var ( 22 | n *Nozzle 23 | streamConnector *spyStreamConnector 24 | logCache *testing.SpyLogCache 25 | spyMetrics *testhelpers.SpyMetricsRegistry 26 | logger *log.Logger 27 | ) 28 | 29 | Context("With custom envelope selectors", func() { 30 | BeforeEach(func() { 31 | tlsConfig, err := testing.NewTLSConfig( 32 | testing.Cert("log-cache-ca.crt"), 33 | testing.Cert("log-cache.crt"), 34 | testing.Cert("log-cache.key"), 35 | "log-cache", 36 | ) 37 | Expect(err).ToNot(HaveOccurred()) 38 | streamConnector = newSpyStreamConnector() 39 | spyMetrics = testhelpers.NewMetricsRegistry() 40 | logCache = testing.NewSpyLogCache(tlsConfig) 41 | logger = log.New(GinkgoWriter, "", log.LstdFlags) 42 | addr := logCache.Start() 43 | 44 | n = NewNozzle(streamConnector, addr, "log-cache", spyMetrics, logger, 45 | WithDialOpts(grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))), 46 | WithSelectors("gauge", "timer", "event"), 47 | ) 48 | go n.Start() 49 | }) 50 | 51 | It("only asks for the requested selectors", func() { 52 | Eventually(streamConnector.requests).Should(HaveLen(1)) 53 | Expect(streamConnector.requests()[0].Selectors).To(ConsistOf( 54 | []*loggregator_v2.Selector{ 55 | { 56 | Message: &loggregator_v2.Selector_Gauge{ 57 | Gauge: &loggregator_v2.GaugeSelector{}, 58 | }, 59 | }, 60 | { 61 | Message: &loggregator_v2.Selector_Timer{ 62 | Timer: &loggregator_v2.TimerSelector{}, 63 | }, 64 | }, 65 | { 66 | Message: &loggregator_v2.Selector_Event{ 67 | Event: &loggregator_v2.EventSelector{}, 68 | }, 69 | }, 70 | }, 71 | )) 72 | 73 | Eventually(streamConnector.envelopes).Should(HaveLen(0)) 74 | }) 75 | }) 76 | 77 | Context("With default envelope selectors", func() { 78 | BeforeEach(func() { 79 | tlsConfig, err := testing.NewTLSConfig( 80 | testing.Cert("log-cache-ca.crt"), 81 | testing.Cert("log-cache.crt"), 82 | testing.Cert("log-cache.key"), 83 | "log-cache", 84 | ) 85 | Expect(err).ToNot(HaveOccurred()) 86 | streamConnector = newSpyStreamConnector() 87 | spyMetrics = testhelpers.NewMetricsRegistry() 88 | logCache = testing.NewSpyLogCache(tlsConfig) 89 | logger = log.New(GinkgoWriter, "", log.LstdFlags) 90 | addr := logCache.Start() 91 | 92 | n = NewNozzle(streamConnector, addr, "log-cache", spyMetrics, logger, 93 | WithDialOpts(grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))), 94 | WithSelectors("log", "gauge", "counter", "timer", "event"), 95 | ) 96 | go n.Start() 97 | }) 98 | 99 | It("connects and reads from a logs provider server", func() { 100 | addEnvelope(1, "some-source-id", streamConnector) 101 | addEnvelope(2, "some-source-id", streamConnector) 102 | addEnvelope(3, "some-source-id", streamConnector) 103 | 104 | Eventually(streamConnector.requests).Should(HaveLen(1)) 105 | Expect(streamConnector.requests()[0].ShardId).To(Equal("log-cache")) 106 | Expect(streamConnector.requests()[0].UsePreferredTags).To(BeTrue()) 107 | Expect(streamConnector.requests()[0].Selectors).To(HaveLen(5)) 108 | 109 | Expect(streamConnector.requests()[0].Selectors).To(ConsistOf( 110 | []*loggregator_v2.Selector{ 111 | { 112 | Message: &loggregator_v2.Selector_Log{ 113 | Log: &loggregator_v2.LogSelector{}, 114 | }, 115 | }, 116 | { 117 | Message: &loggregator_v2.Selector_Gauge{ 118 | Gauge: &loggregator_v2.GaugeSelector{}, 119 | }, 120 | }, 121 | { 122 | Message: &loggregator_v2.Selector_Counter{ 123 | Counter: &loggregator_v2.CounterSelector{}, 124 | }, 125 | }, 126 | { 127 | Message: &loggregator_v2.Selector_Timer{ 128 | Timer: &loggregator_v2.TimerSelector{}, 129 | }, 130 | }, 131 | { 132 | Message: &loggregator_v2.Selector_Event{ 133 | Event: &loggregator_v2.EventSelector{}, 134 | }, 135 | }, 136 | }, 137 | )) 138 | 139 | Eventually(streamConnector.envelopes).Should(HaveLen(0)) 140 | }) 141 | 142 | It("writes each envelope to the LogCache", func() { 143 | addEnvelope(1, "some-source-id", streamConnector) 144 | addEnvelope(2, "some-source-id", streamConnector) 145 | addEnvelope(3, "some-source-id", streamConnector) 146 | 147 | Eventually(logCache.GetEnvelopes).Should(HaveLen(3)) 148 | Expect(logCache.GetEnvelopes()[0].Timestamp).To(Equal(int64(1))) 149 | Expect(logCache.GetEnvelopes()[1].Timestamp).To(Equal(int64(2))) 150 | Expect(logCache.GetEnvelopes()[2].Timestamp).To(Equal(int64(3))) 151 | }) 152 | 153 | It("writes Ingress, Egress and Err metrics", func() { 154 | addEnvelope(1, "some-source-id", streamConnector) 155 | addEnvelope(2, "some-source-id", streamConnector) 156 | addEnvelope(3, "some-source-id", streamConnector) 157 | 158 | Eventually(logCache.GetEnvelopes).Should(HaveLen(3)) 159 | Eventually(func() float64 { 160 | return spyMetrics.GetMetricValue("nozzle_ingress", nil) 161 | }).Should(Equal(3.0)) 162 | Eventually(func() float64 { 163 | return spyMetrics.GetMetricValue("nozzle_egress", nil) 164 | }).Should(Equal(3.0)) 165 | Eventually(func() float64 { 166 | return spyMetrics.GetMetricValue("nozzle_err", nil) 167 | }).Should(BeZero()) 168 | }) 169 | }) 170 | }) 171 | 172 | func addEnvelope(timestamp int64, sourceID string, c *spyStreamConnector) { 173 | c.envelopes <- []*loggregator_v2.Envelope{ 174 | { 175 | Timestamp: timestamp, 176 | SourceId: sourceID, 177 | }, 178 | } 179 | } 180 | 181 | type spyStreamConnector struct { 182 | mu sync.Mutex 183 | requests_ []*loggregator_v2.EgressBatchRequest 184 | envelopes chan []*loggregator_v2.Envelope 185 | } 186 | 187 | func newSpyStreamConnector() *spyStreamConnector { 188 | return &spyStreamConnector{ 189 | envelopes: make(chan []*loggregator_v2.Envelope, 100), 190 | } 191 | } 192 | 193 | func (s *spyStreamConnector) Stream(ctx context.Context, req *loggregator_v2.EgressBatchRequest) loggregator.EnvelopeStream { 194 | s.mu.Lock() 195 | defer s.mu.Unlock() 196 | s.requests_ = append(s.requests_, req) 197 | 198 | return func() []*loggregator_v2.Envelope { 199 | select { 200 | case e := <-s.envelopes: 201 | return e 202 | default: 203 | return nil 204 | } 205 | } 206 | } 207 | 208 | func (s *spyStreamConnector) requests() []*loggregator_v2.EgressBatchRequest { 209 | s.mu.Lock() 210 | defer s.mu.Unlock() 211 | 212 | reqs := make([]*loggregator_v2.EgressBatchRequest, len(s.requests_)) 213 | copy(reqs, s.requests_) 214 | 215 | return reqs 216 | } 217 | -------------------------------------------------------------------------------- /internal/promql/data_reader/data_reader_suite_test.go: -------------------------------------------------------------------------------- 1 | package data_reader_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestDataReader(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "DataReader Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/promql/data_reader/walking_data_reader.go: -------------------------------------------------------------------------------- 1 | package data_reader 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 8 | "code.cloudfoundry.org/log-cache/pkg/client" 9 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 10 | ) 11 | 12 | type WalkingDataReader struct { 13 | r client.Reader 14 | } 15 | 16 | func NewWalkingDataReader(reader client.Reader) *WalkingDataReader { 17 | return &WalkingDataReader{ 18 | r: reader, 19 | } 20 | } 21 | 22 | func (r *WalkingDataReader) Read( 23 | ctx context.Context, 24 | in *logcache_v1.ReadRequest, 25 | ) (*logcache_v1.ReadResponse, error) { 26 | 27 | var result []*loggregator_v2.Envelope 28 | 29 | client.Walk(ctx, in.GetSourceId(), func(es []*loggregator_v2.Envelope) bool { 30 | result = append(result, es...) 31 | return true 32 | }, r.r, 33 | client.WithWalkStartTime(time.Unix(0, in.GetStartTime())), 34 | client.WithWalkEndTime(time.Unix(0, in.GetEndTime())), 35 | client.WithWalkLimit(int(in.GetLimit())), 36 | client.WithWalkEnvelopeTypes(in.GetEnvelopeTypes()...), 37 | client.WithWalkBackoff(client.NewRetryBackoffOnErr(time.Second, 5)), 38 | ) 39 | 40 | if err := ctx.Err(); err != nil { 41 | return nil, err 42 | } 43 | 44 | return &logcache_v1.ReadResponse{ 45 | Envelopes: &loggregator_v2.EnvelopeBatch{ 46 | Batch: result, 47 | }, 48 | }, nil 49 | } 50 | -------------------------------------------------------------------------------- /internal/promql/data_reader/walking_data_reader_test.go: -------------------------------------------------------------------------------- 1 | package data_reader_test 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 8 | "code.cloudfoundry.org/log-cache/internal/promql/data_reader" 9 | "code.cloudfoundry.org/log-cache/pkg/client" 10 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 11 | 12 | . "github.com/onsi/ginkgo" 13 | . "github.com/onsi/gomega" 14 | ) 15 | 16 | var _ = Describe("WalkingDataReader", func() { 17 | var ( 18 | spyLogCache *spyLogCache 19 | r *data_reader.WalkingDataReader 20 | ) 21 | 22 | BeforeEach(func() { 23 | spyLogCache = newSpyLogCache() 24 | r = data_reader.NewWalkingDataReader(spyLogCache.Read) 25 | }) 26 | 27 | It("returns the error from the context", func() { 28 | ctx, cancel := context.WithCancel(context.Background()) 29 | cancel() 30 | _, err := r.Read(ctx, &logcache_v1.ReadRequest{}) 31 | Expect(err).To(HaveOccurred()) 32 | }) 33 | }) 34 | 35 | type spyLogCache struct { 36 | results []*loggregator_v2.Envelope 37 | err error 38 | } 39 | 40 | func newSpyLogCache() *spyLogCache { 41 | return &spyLogCache{} 42 | } 43 | 44 | func (s *spyLogCache) Read( 45 | ctx context.Context, 46 | sourceID string, 47 | start time.Time, 48 | opts ...client.ReadOption, 49 | ) ([]*loggregator_v2.Envelope, error) { 50 | return s.results, s.err 51 | } 52 | -------------------------------------------------------------------------------- /internal/promql/parsing.go: -------------------------------------------------------------------------------- 1 | package promql 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/prometheus/common/model" 10 | ) 11 | 12 | func ParseStep(param string) (time.Duration, error) { 13 | if step, err := strconv.ParseFloat(param, 64); err == nil { 14 | stepInNanoSeconds := step * float64(time.Second) 15 | if stepInNanoSeconds > float64(math.MaxInt64) || stepInNanoSeconds < float64(math.MinInt64) { 16 | return 0, fmt.Errorf("cannot parse %q to a valid step. It overflows int64", param) 17 | } 18 | return time.Duration(stepInNanoSeconds), nil 19 | } 20 | if step, err := ParseDuration(param); err == nil { 21 | return step, nil 22 | } 23 | return 0, fmt.Errorf("cannot parse %q to a valid step", param) 24 | } 25 | 26 | func ParseDuration(param string) (time.Duration, error) { 27 | duration, err := model.ParseDuration(param) 28 | if err != nil { 29 | return 0, fmt.Errorf("cannot parse %q to a valid duration", param) 30 | } 31 | 32 | return time.Duration(duration), nil 33 | } 34 | 35 | func ParseTime(param string) (time.Time, error) { 36 | if decimalTime, err := strconv.ParseFloat(param, 64); err == nil { 37 | return time.Unix(0, int64(decimalTime*1e9)), nil 38 | } 39 | if t, err := time.Parse(time.RFC3339Nano, param); err == nil { 40 | return t, nil 41 | } 42 | return time.Unix(0, 0), fmt.Errorf("cannot parse %q to a valid Unix or RFC3339 timestamp", param) 43 | } 44 | -------------------------------------------------------------------------------- /internal/promql/parsing_test.go: -------------------------------------------------------------------------------- 1 | package promql_test 2 | 3 | import ( 4 | "math" 5 | "strconv" 6 | "time" 7 | 8 | "code.cloudfoundry.org/log-cache/internal/promql" 9 | 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/ginkgo/extensions/table" 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | var _ = Describe("PromQL Parsing", func() { 16 | Describe("ParseStep", func() { 17 | DescribeTable("it supports all Prometheus-accepted units", 18 | func(stringStep string, expectedStep time.Duration) { 19 | step, err := promql.ParseStep(stringStep) 20 | 21 | Expect(err).ToNot(HaveOccurred()) 22 | Expect(step).To(Equal(expectedStep)) 23 | }, 24 | 25 | Entry("unlabeled second", "1", time.Second), 26 | Entry("float second", "1.5", 1500*time.Millisecond), 27 | Entry("second", "1s", time.Second), 28 | Entry("minute", "1m", time.Minute), 29 | Entry("hour", "1h", time.Hour), 30 | Entry("day", "1d", typicalNumberOfHoursPerDay*time.Hour), 31 | Entry("week", "1w", typicalNumberOfHoursPerWeek*time.Hour), 32 | Entry("year", "1y", typicalNumberOfHoursPerYear*time.Hour), 33 | ) 34 | 35 | DescribeTable("it handles errors", 36 | func(stringStep string) { 37 | _, err := promql.ParseStep(stringStep) 38 | 39 | Expect(err).To(HaveOccurred()) 40 | }, 41 | 42 | Entry("empty step", ""), 43 | Entry("invalid unit", "4q"), 44 | Entry("overflows int64", strconv.Itoa(math.MaxInt64)+"0"), 45 | ) 46 | }) 47 | 48 | Describe("ParseDuration", func() { 49 | DescribeTable("it supports all Prometheus-accepted units", 50 | func(stringDuration string, expectedDuration time.Duration) { 51 | step, err := promql.ParseDuration(stringDuration) 52 | 53 | Expect(err).ToNot(HaveOccurred()) 54 | Expect(step).To(Equal(expectedDuration)) 55 | }, 56 | 57 | Entry("second", "1s", time.Second), 58 | Entry("minute", "1m", time.Minute), 59 | Entry("hour", "1h", time.Hour), 60 | Entry("day", "1d", typicalNumberOfHoursPerDay*time.Hour), 61 | Entry("week", "1w", typicalNumberOfHoursPerWeek*time.Hour), 62 | Entry("year", "1y", typicalNumberOfHoursPerYear*time.Hour), 63 | ) 64 | 65 | DescribeTable("it handles errors", 66 | func(stringDuration string) { 67 | _, err := promql.ParseDuration(stringDuration) 68 | 69 | Expect(err).To(HaveOccurred()) 70 | }, 71 | 72 | Entry("unlabeled second", "1"), 73 | Entry("float second", "1.5"), 74 | Entry("empty step", ""), 75 | Entry("invalid unit", "4q"), 76 | Entry("overflows int64", strconv.Itoa(math.MaxInt64)+"0"), 77 | ) 78 | }) 79 | 80 | Describe("ParseTime", func() { 81 | DescribeTable("it supports all Prometheus-accepted formats", 82 | func(inputTime string, expectedResult time.Time) { 83 | result, err := promql.ParseTime(inputTime) 84 | 85 | Expect(err).ToNot(HaveOccurred()) 86 | Expect(result).To(BeTemporally("==", expectedResult)) 87 | }, 88 | 89 | Entry("whole seconds", "123456", time.Unix(123456, 0)), 90 | Entry("decimal seconds", "123456.789", time.Unix(123456, 789000000)), 91 | Entry("RFC3339 with UTC", "2015-07-01T20:10:30.781Z", time.Date(2015, 7, 1, 20, 10, 30, 781000000, time.UTC)), 92 | Entry("RFC3339 with MST", "2015-07-01T20:10:30.781-06:00", time.Date(2015, 7, 1, 20, 10, 30, 781000000, time.FixedZone("MST", -6*60*60))), 93 | ) 94 | 95 | DescribeTable("it handles errors", 96 | func(inputTime string) { 97 | _, err := promql.ParseTime(inputTime) 98 | 99 | Expect(err).To(HaveOccurred()) 100 | }, 101 | 102 | Entry("empty time", ""), 103 | Entry("not a number", "potato"), 104 | Entry("RFC3339 without timezone", "2015-07-01T20:10:30.781"), 105 | ) 106 | }) 107 | }) 108 | 109 | const ( 110 | typicalNumberOfHoursPerDay = 24 111 | typicalNumberOfHoursPerWeek = typicalNumberOfHoursPerDay * 7 112 | typicalNumberOfHoursPerYear = typicalNumberOfHoursPerDay * 365 113 | ) 114 | -------------------------------------------------------------------------------- /internal/promql/promql_suite_test.go: -------------------------------------------------------------------------------- 1 | package promql_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestPromql(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Promql Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/routing/batched_ingress_client.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "log" 6 | "time" 7 | 8 | batching "code.cloudfoundry.org/go-batching" 9 | diodes "code.cloudfoundry.org/go-diodes" 10 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 11 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 12 | "golang.org/x/net/context" 13 | "google.golang.org/grpc" 14 | ) 15 | 16 | // BatchedIngressClient batches envelopes before sending it. Each invocation 17 | // to Send is async. 18 | type BatchedIngressClient struct { 19 | c rpc.IngressClient 20 | 21 | buffer *diodes.OneToOne 22 | size int 23 | interval time.Duration 24 | log *log.Logger 25 | } 26 | 27 | // NewBatchedIngressClient returns a new BatchedIngressClient. 28 | func NewBatchedIngressClient( 29 | size int, 30 | interval time.Duration, 31 | c rpc.IngressClient, 32 | incDroppedMetric metrics.Counter, 33 | log *log.Logger, 34 | ) *BatchedIngressClient { 35 | b := &BatchedIngressClient{ 36 | c: c, 37 | size: size, 38 | interval: interval, 39 | log: log, 40 | 41 | buffer: diodes.NewOneToOne(10000, diodes.AlertFunc(func(dropped int) { 42 | log.Printf("dropped %d envelopes", dropped) 43 | incDroppedMetric.Add(float64(dropped)) 44 | })), 45 | } 46 | 47 | go b.start() 48 | 49 | return b 50 | } 51 | 52 | // Send batches envelopes before shipping them to the client. 53 | func (b *BatchedIngressClient) Send(ctx context.Context, in *rpc.SendRequest, opts ...grpc.CallOption) (*rpc.SendResponse, error) { 54 | for i := range in.GetEnvelopes().GetBatch() { 55 | b.buffer.Set(diodes.GenericDataType(in.Envelopes.Batch[i])) 56 | } 57 | 58 | return &rpc.SendResponse{}, nil 59 | } 60 | 61 | func (b *BatchedIngressClient) start() { 62 | batcher := batching.NewBatcher(b.size, b.interval, batching.WriterFunc(b.write)) 63 | for { 64 | e, ok := b.buffer.TryNext() 65 | if !ok { 66 | batcher.Flush() 67 | time.Sleep(50 * time.Millisecond) 68 | continue 69 | } 70 | batcher.Write((*loggregator_v2.Envelope)(e)) 71 | } 72 | } 73 | 74 | func (b *BatchedIngressClient) write(batch []interface{}) { 75 | var e []*loggregator_v2.Envelope 76 | for _, i := range batch { 77 | e = append(e, i.(*loggregator_v2.Envelope)) 78 | } 79 | 80 | ctx, _ := context.WithTimeout(context.Background(), 3*time.Second) 81 | _, err := b.c.Send(ctx, &rpc.SendRequest{ 82 | LocalOnly: true, 83 | Envelopes: &loggregator_v2.EnvelopeBatch{Batch: e}, 84 | }) 85 | 86 | if err != nil { 87 | b.log.Printf("failed to write envelope: %s", err) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /internal/routing/batched_ingress_client_test.go: -------------------------------------------------------------------------------- 1 | package routing_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/go-loggregator/metrics" 5 | "code.cloudfoundry.org/go-loggregator/metrics/testhelpers" 6 | "io/ioutil" 7 | "log" 8 | "time" 9 | 10 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 11 | "code.cloudfoundry.org/log-cache/internal/routing" 12 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 13 | "golang.org/x/net/context" 14 | 15 | . "github.com/onsi/ginkgo" 16 | . "github.com/onsi/gomega" 17 | ) 18 | 19 | var _ = Describe("BatchedIngressClient", func() { 20 | var ( 21 | m *testhelpers.SpyMetricsRegistry 22 | spyDropped metrics.Counter 23 | ingressClient *spyIngressClient 24 | c *routing.BatchedIngressClient 25 | ) 26 | 27 | BeforeEach(func() { 28 | m = testhelpers.NewMetricsRegistry() 29 | spyDropped = m.NewCounter("nodeX_dropped") 30 | ingressClient = newSpyIngressClient() 31 | c = routing.NewBatchedIngressClient(5, time.Hour, ingressClient, spyDropped, log.New(ioutil.Discard, "", 0)) 32 | }) 33 | 34 | It("sends envelopes with LocalOnly set to true", func() { 35 | for i := 0; i < 5; i++ { 36 | _, err := c.Send(context.Background(), &rpc.SendRequest{ 37 | Envelopes: &loggregator_v2.EnvelopeBatch{ 38 | Batch: []*loggregator_v2.Envelope{ 39 | {Timestamp: int64(i)}, 40 | }, 41 | }, 42 | }) 43 | Expect(err).ToNot(HaveOccurred()) 44 | } 45 | 46 | Eventually(ingressClient.Requests).Should(HaveLen(1)) 47 | Expect(ingressClient.Requests()[0].Envelopes.Batch).To(HaveLen(5)) 48 | Expect(ingressClient.Requests()[0].LocalOnly).To(BeTrue()) 49 | 50 | for i, e := range ingressClient.Requests()[0].Envelopes.Batch { 51 | Expect(e).To(Equal( 52 | &loggregator_v2.Envelope{ 53 | Timestamp: int64(i), 54 | }, 55 | )) 56 | } 57 | }) 58 | 59 | It("sends envelopes by batches because of size", func() { 60 | for i := 0; i < 5; i++ { 61 | _, err := c.Send(context.Background(), &rpc.SendRequest{ 62 | Envelopes: &loggregator_v2.EnvelopeBatch{ 63 | Batch: []*loggregator_v2.Envelope{ 64 | {Timestamp: int64(i)}, 65 | }, 66 | }, 67 | }) 68 | Expect(err).ToNot(HaveOccurred()) 69 | } 70 | 71 | Eventually(ingressClient.Requests).Should(HaveLen(1)) 72 | Expect(ingressClient.Requests()[0].Envelopes.Batch).To(HaveLen(5)) 73 | }) 74 | 75 | It("sends envelopes by batches because of interval", func() { 76 | c = routing.NewBatchedIngressClient(5, time.Microsecond, ingressClient, spyDropped, log.New(ioutil.Discard, "", 0)) 77 | _, err := c.Send(context.Background(), &rpc.SendRequest{ 78 | Envelopes: &loggregator_v2.EnvelopeBatch{ 79 | Batch: []*loggregator_v2.Envelope{ 80 | {Timestamp: 1}, 81 | }, 82 | }, 83 | }) 84 | Expect(err).ToNot(HaveOccurred()) 85 | 86 | Eventually(ingressClient.Requests).Should(HaveLen(1)) 87 | Expect(ingressClient.Requests()[0].Envelopes.Batch).To(HaveLen(1)) 88 | }) 89 | 90 | It("increments a dropped counter", func() { 91 | go func(ingressClient *spyIngressClient) { 92 | for { 93 | // Force ingress client to block 100ms 94 | ingressClient.mu.Lock() 95 | time.Sleep(100 * time.Millisecond) 96 | ingressClient.mu.Unlock() 97 | } 98 | }(ingressClient) 99 | 100 | for i := 0; i < 25000; i++ { 101 | c.Send(context.Background(), &rpc.SendRequest{ 102 | Envelopes: &loggregator_v2.EnvelopeBatch{ 103 | Batch: []*loggregator_v2.Envelope{ 104 | {Timestamp: 1}, 105 | }, 106 | }, 107 | }) 108 | } 109 | 110 | Eventually(func() float64 { 111 | return m.GetMetricValue("nodeX_dropped", nil) 112 | }).ShouldNot(BeZero()) 113 | }) 114 | }) 115 | -------------------------------------------------------------------------------- /internal/routing/egress_reverse_proxy.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | "math/rand" 8 | "sync/atomic" 9 | "time" 10 | "unsafe" 11 | 12 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/codes" 15 | ) 16 | 17 | // EgressReverseProxy is a reverse proxy for Egress requests. 18 | type EgressReverseProxy struct { 19 | clients []rpc.EgressClient 20 | l Lookup 21 | localIdx int 22 | log *log.Logger 23 | 24 | remoteMetaCache unsafe.Pointer 25 | localMetaCache unsafe.Pointer 26 | metaCacheDuration time.Duration 27 | } 28 | 29 | // NewEgressReverseProxy returns a new EgressReverseProxy. LocalIdx is 30 | // required to know where to find the local node for meta lookups. 31 | func NewEgressReverseProxy( 32 | l Lookup, 33 | clients []rpc.EgressClient, 34 | localIdx int, 35 | log *log.Logger, 36 | opts ...EgressReverseProxyOption, 37 | ) *EgressReverseProxy { 38 | e := &EgressReverseProxy{ 39 | l: l, 40 | clients: clients, 41 | localIdx: localIdx, 42 | log: log, 43 | metaCacheDuration: time.Second, 44 | } 45 | 46 | for _, o := range opts { 47 | o(e) 48 | } 49 | 50 | return e 51 | } 52 | 53 | // Read will either read from the local node or remote nodes. 54 | func (e *EgressReverseProxy) Read(ctx context.Context, in *rpc.ReadRequest) (*rpc.ReadResponse, error) { 55 | idx := e.l(in.GetSourceId()) 56 | if len(idx) == 0 { 57 | return nil, grpc.Errorf(codes.Unavailable, "failed to find route for request. please try again") 58 | } 59 | for _, i := range idx { 60 | if i == e.localIdx { 61 | return e.clients[e.localIdx].Read(ctx, in) 62 | } 63 | } 64 | return e.clients[idx[rand.Intn(len(idx))]].Read(ctx, in) 65 | } 66 | 67 | // Meta will gather meta from the local store and remote nodes. 68 | func (e *EgressReverseProxy) Meta(ctx context.Context, in *rpc.MetaRequest) (*rpc.MetaResponse, error) { 69 | if in.LocalOnly { 70 | return e.localMeta(ctx, in) 71 | } 72 | return e.remoteMeta(ctx, in) 73 | } 74 | 75 | func (e *EgressReverseProxy) localMeta(ctx context.Context, in *rpc.MetaRequest) (*rpc.MetaResponse, error) { 76 | cache := (*metaCache)(atomic.LoadPointer(&e.localMetaCache)) 77 | if !cache.expired() { 78 | return cache.metaResp, nil 79 | } 80 | 81 | metaInfo, err := e.clients[e.localIdx].Meta(ctx, in) 82 | if err != nil { 83 | return nil, err 84 | } 85 | 86 | atomic.StorePointer(&e.localMetaCache, unsafe.Pointer(&metaCache{ 87 | duration: e.metaCacheDuration, 88 | timestamp: time.Now(), 89 | metaResp: metaInfo, 90 | })) 91 | 92 | return metaInfo, nil 93 | } 94 | 95 | func (e *EgressReverseProxy) remoteMeta(ctx context.Context, in *rpc.MetaRequest) (*rpc.MetaResponse, error) { 96 | cache := (*metaCache)(atomic.LoadPointer(&e.remoteMetaCache)) 97 | if !cache.expired() { 98 | return cache.metaResp, nil 99 | } 100 | 101 | // Each remote should only fetch their local meta data. 102 | req := &rpc.MetaRequest{ 103 | LocalOnly: true, 104 | } 105 | 106 | result := &rpc.MetaResponse{ 107 | Meta: make(map[string]*rpc.MetaInfo), 108 | } 109 | 110 | var errs []error 111 | for _, c := range e.clients { 112 | resp, err := c.Meta(ctx, req) 113 | if err != nil { 114 | // TODO: Metric 115 | e.log.Printf("failed to read meta data from remote node: %s", err) 116 | errs = append(errs, err) 117 | continue 118 | } 119 | 120 | for sourceID, mi := range resp.Meta { 121 | result.Meta[sourceID] = mi 122 | } 123 | } 124 | 125 | if len(errs) == len(e.clients) { 126 | return nil, errors.New("failed to read meta data from remote node") 127 | } 128 | 129 | atomic.StorePointer(&e.remoteMetaCache, unsafe.Pointer(&metaCache{ 130 | duration: e.metaCacheDuration, 131 | timestamp: time.Now(), 132 | metaResp: result, 133 | })) 134 | 135 | return result, nil 136 | } 137 | 138 | type EgressReverseProxyOption func(e *EgressReverseProxy) 139 | 140 | // WithMetaCacheDuration is a EgressReverseProxyOption to configure how long 141 | // to cache results from the Meta endpoint. 142 | func WithMetaCacheDuration(d time.Duration) EgressReverseProxyOption { 143 | return func(e *EgressReverseProxy) { 144 | e.metaCacheDuration = d 145 | } 146 | } 147 | 148 | type metaCache struct { 149 | duration time.Duration 150 | timestamp time.Time 151 | metaResp *rpc.MetaResponse 152 | } 153 | 154 | func (c *metaCache) expired() bool { 155 | if c == nil { 156 | return true 157 | } 158 | 159 | return time.Now().After(c.timestamp.Add(c.duration)) 160 | } 161 | -------------------------------------------------------------------------------- /internal/routing/ingress_reverse_proxy.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 8 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 9 | "google.golang.org/grpc" 10 | ) 11 | 12 | // IngressReverseProxy is a reverse proxy for Ingress requests. 13 | type IngressReverseProxy struct { 14 | clients []rpc.IngressClient 15 | localIdx int 16 | l Lookup 17 | log *log.Logger 18 | } 19 | 20 | // Lookup is used to find which Clients a source ID should be routed to. 21 | type Lookup func(sourceID string) []int 22 | 23 | // NewIngressReverseProxy returns a new IngressReverseProxy. 24 | func NewIngressReverseProxy( 25 | l Lookup, 26 | clients []rpc.IngressClient, 27 | localIdx int, 28 | log *log.Logger, 29 | ) *IngressReverseProxy { 30 | 31 | return &IngressReverseProxy{ 32 | clients: clients, 33 | localIdx: localIdx, 34 | l: l, 35 | log: log, 36 | } 37 | } 38 | 39 | // Send will send to either the local node or the correct remote node 40 | // according to its source ID. 41 | func (p *IngressReverseProxy) Send(ctx context.Context, r *rpc.SendRequest) (*rpc.SendResponse, error) { 42 | if r.LocalOnly { 43 | return p.clients[p.localIdx].Send(ctx, r) 44 | } 45 | 46 | envelopesByNode := make(map[int][]*loggregator_v2.Envelope) 47 | 48 | for _, e := range r.Envelopes.Batch { 49 | for _, idx := range p.l(e.GetSourceId()) { 50 | envelopesByNode[idx] = append(envelopesByNode[idx], e) 51 | } 52 | } 53 | 54 | for idx, envelopes := range envelopesByNode { 55 | _, err := p.clients[idx].Send(ctx, &rpc.SendRequest{ 56 | LocalOnly: true, 57 | Envelopes: &loggregator_v2.EnvelopeBatch{ 58 | Batch: envelopes, 59 | }, 60 | }) 61 | 62 | if err != nil { 63 | p.log.Printf("ingress reverse proxy: failed to write to client: %s", err) 64 | continue 65 | } 66 | } 67 | 68 | return &rpc.SendResponse{}, nil 69 | } 70 | 71 | // IngressClientFunc transforms a function into an IngressClient. 72 | type IngressClientFunc func(ctx context.Context, r *rpc.SendRequest, opts ...grpc.CallOption) (*rpc.SendResponse, error) 73 | 74 | // Send implements an IngressClient. 75 | func (f IngressClientFunc) Send(ctx context.Context, r *rpc.SendRequest, opts ...grpc.CallOption) (*rpc.SendResponse, error) { 76 | return f(ctx, r, opts...) 77 | } 78 | -------------------------------------------------------------------------------- /internal/routing/ingress_reverse_proxy_test.go: -------------------------------------------------------------------------------- 1 | package routing_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "io/ioutil" 7 | "log" 8 | "sync" 9 | 10 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 11 | "code.cloudfoundry.org/log-cache/internal/routing" 12 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 13 | "google.golang.org/grpc" 14 | 15 | . "github.com/onsi/ginkgo" 16 | . "github.com/onsi/gomega" 17 | ) 18 | 19 | var _ = Describe("IngressReverseProxy", func() { 20 | var ( 21 | spyLookup *spyLookup 22 | spyIngressRemoteClient *spyIngressClient 23 | spyIngressLocalClient *spyIngressClient 24 | p *routing.IngressReverseProxy 25 | ) 26 | 27 | BeforeEach(func() { 28 | spyLookup = newSpyLookup() 29 | spyIngressRemoteClient = newSpyIngressClient() 30 | spyIngressLocalClient = newSpyIngressClient() 31 | p = routing.NewIngressReverseProxy(spyLookup.Lookup, []rpc.IngressClient{ 32 | spyIngressRemoteClient, 33 | spyIngressLocalClient, 34 | }, 35 | 1, // Point local at spyIngressLocalClient 36 | log.New(ioutil.Discard, "", 0)) 37 | }) 38 | 39 | It("uses the correct clients", func() { 40 | spyLookup.results["a"] = []int{0} 41 | spyLookup.results["b"] = []int{1} 42 | spyLookup.results["c"] = []int{0, 1} 43 | 44 | _, err := p.Send(context.Background(), &rpc.SendRequest{ 45 | Envelopes: &loggregator_v2.EnvelopeBatch{ 46 | Batch: []*loggregator_v2.Envelope{ 47 | {SourceId: "a", Timestamp: 1}, 48 | {SourceId: "b", Timestamp: 2}, 49 | {SourceId: "c", Timestamp: 3}, 50 | }, 51 | }, 52 | }) 53 | Expect(err).ToNot(HaveOccurred()) 54 | 55 | Expect(spyLookup.sourceIDs).To(ConsistOf("a", "b", "c")) 56 | 57 | Expect(spyIngressRemoteClient.reqs).To(ConsistOf( 58 | &rpc.SendRequest{ 59 | LocalOnly: true, 60 | Envelopes: &loggregator_v2.EnvelopeBatch{ 61 | Batch: []*loggregator_v2.Envelope{ 62 | {SourceId: "a", Timestamp: 1}, 63 | {SourceId: "c", Timestamp: 3}, 64 | }, 65 | }, 66 | }, 67 | )) 68 | 69 | Expect(spyIngressLocalClient.reqs).To(ConsistOf( 70 | &rpc.SendRequest{ 71 | LocalOnly: true, 72 | Envelopes: &loggregator_v2.EnvelopeBatch{ 73 | Batch: []*loggregator_v2.Envelope{ 74 | {SourceId: "b", Timestamp: 2}, 75 | {SourceId: "c", Timestamp: 3}, 76 | }, 77 | }, 78 | }, 79 | )) 80 | }) 81 | 82 | It("routes local_only requests only to local client", func() { 83 | spyLookup.results["a"] = []int{0, 1} 84 | _, err := p.Send(context.Background(), &rpc.SendRequest{ 85 | LocalOnly: true, 86 | Envelopes: &loggregator_v2.EnvelopeBatch{ 87 | Batch: []*loggregator_v2.Envelope{ 88 | {SourceId: "a", Timestamp: 1}, 89 | }, 90 | }, 91 | }) 92 | Expect(err).ToNot(HaveOccurred()) 93 | 94 | Expect(spyIngressLocalClient.reqs).To(ConsistOf( 95 | &rpc.SendRequest{ 96 | LocalOnly: true, 97 | Envelopes: &loggregator_v2.EnvelopeBatch{ 98 | Batch: []*loggregator_v2.Envelope{ 99 | {SourceId: "a", Timestamp: 1}, 100 | }, 101 | }, 102 | }, 103 | )) 104 | Expect(spyIngressRemoteClient.reqs).To(BeEmpty()) 105 | }) 106 | 107 | It("survives an unroutable request", func() { 108 | spyLookup.results["b"] = []int{1} 109 | 110 | _, err := p.Send(context.Background(), &rpc.SendRequest{ 111 | Envelopes: &loggregator_v2.EnvelopeBatch{ 112 | Batch: []*loggregator_v2.Envelope{ 113 | {SourceId: "a", Timestamp: 1}, 114 | {SourceId: "b", Timestamp: 2}, 115 | }, 116 | }, 117 | }) 118 | Expect(err).ToNot(HaveOccurred()) 119 | 120 | Expect(spyIngressLocalClient.reqs).To(ConsistOf(&rpc.SendRequest{ 121 | LocalOnly: true, 122 | Envelopes: &loggregator_v2.EnvelopeBatch{ 123 | Batch: []*loggregator_v2.Envelope{ 124 | {SourceId: "b", Timestamp: 2}, 125 | }, 126 | }, 127 | })) 128 | }) 129 | 130 | It("uses the given context", func() { 131 | spyLookup.results["a"] = []int{0} 132 | spyLookup.results["b"] = []int{1} 133 | 134 | ctx, cancel := context.WithCancel(context.Background()) 135 | cancel() 136 | 137 | _, err := p.Send(ctx, &rpc.SendRequest{ 138 | Envelopes: &loggregator_v2.EnvelopeBatch{ 139 | Batch: []*loggregator_v2.Envelope{ 140 | {SourceId: "a", Timestamp: 1}, 141 | {SourceId: "b", Timestamp: 2}, 142 | }, 143 | }, 144 | }) 145 | Expect(err).ToNot(HaveOccurred()) 146 | 147 | Expect(spyIngressRemoteClient.ctxs).ToNot(BeEmpty()) 148 | Expect(spyIngressRemoteClient.ctxs[0].Done()).To(BeClosed()) 149 | }) 150 | 151 | It("does not return an error if one of the clients returns an error", func() { 152 | spyIngressRemoteClient.err = errors.New("some-error") 153 | 154 | spyLookup.results["a"] = []int{0} 155 | spyLookup.results["b"] = []int{1} 156 | 157 | _, err := p.Send(context.Background(), &rpc.SendRequest{ 158 | Envelopes: &loggregator_v2.EnvelopeBatch{ 159 | Batch: []*loggregator_v2.Envelope{ 160 | {SourceId: "a", Timestamp: 1}, 161 | {SourceId: "b", Timestamp: 2}, 162 | }, 163 | }, 164 | }) 165 | Expect(err).ToNot(HaveOccurred()) 166 | }) 167 | }) 168 | 169 | type spyLookup struct { 170 | sourceIDs []string 171 | results map[string][]int 172 | } 173 | 174 | func newSpyLookup() *spyLookup { 175 | return &spyLookup{ 176 | results: make(map[string][]int), 177 | } 178 | } 179 | 180 | func (s *spyLookup) Lookup(sourceID string) []int { 181 | s.sourceIDs = append(s.sourceIDs, sourceID) 182 | return s.results[sourceID] 183 | } 184 | 185 | type spyIngressClient struct { 186 | mu sync.Mutex 187 | ctxs []context.Context 188 | reqs []*rpc.SendRequest 189 | err error 190 | } 191 | 192 | func newSpyIngressClient() *spyIngressClient { 193 | return &spyIngressClient{} 194 | } 195 | 196 | func (s *spyIngressClient) Send(ctx context.Context, in *rpc.SendRequest, opts ...grpc.CallOption) (*rpc.SendResponse, error) { 197 | s.mu.Lock() 198 | defer s.mu.Unlock() 199 | 200 | s.ctxs = append(s.ctxs, ctx) 201 | s.reqs = append(s.reqs, in) 202 | return &rpc.SendResponse{}, s.err 203 | } 204 | 205 | func (s *spyIngressClient) Requests() []*rpc.SendRequest { 206 | s.mu.Lock() 207 | defer s.mu.Unlock() 208 | 209 | r := make([]*rpc.SendRequest, len(s.reqs)) 210 | copy(r, s.reqs) 211 | return r 212 | } 213 | -------------------------------------------------------------------------------- /internal/routing/local_store_reader.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "time" 7 | 8 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 9 | "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 10 | "golang.org/x/net/context" 11 | "google.golang.org/grpc" 12 | ) 13 | 14 | // LocalStoreReader accesses a store via gRPC calls. It handles converting the 15 | // requests into a form that the store understands for reading. 16 | type LocalStoreReader struct { 17 | s StoreReader 18 | } 19 | 20 | // StoreReader proxies to the log cache for getting envelopes or Log Cache 21 | // Metadata. 22 | type StoreReader interface { 23 | // Gets envelopes from a local or remote Log Cache. 24 | Get( 25 | sourceID string, 26 | start time.Time, 27 | end time.Time, 28 | envelopeTypes []logcache_v1.EnvelopeType, 29 | nameFilter *regexp.Regexp, 30 | limit int, 31 | descending bool, 32 | ) []*loggregator_v2.Envelope 33 | 34 | // Meta gets the metadata from Log Cache instances in the cluster. 35 | Meta() map[string]logcache_v1.MetaInfo 36 | } 37 | 38 | // NewLocalStoreReader creates and returns a new LocalStoreReader. 39 | func NewLocalStoreReader(s StoreReader) *LocalStoreReader { 40 | return &LocalStoreReader{ 41 | s: s, 42 | } 43 | } 44 | 45 | // Read returns data from the store. 46 | func (r *LocalStoreReader) Read(ctx context.Context, req *logcache_v1.ReadRequest, opts ...grpc.CallOption) (*logcache_v1.ReadResponse, error) { 47 | if req.EndTime != 0 && req.StartTime > req.EndTime { 48 | return nil, fmt.Errorf("StartTime (%d) must be before EndTime (%d)", req.StartTime, req.EndTime) 49 | } 50 | 51 | if req.Limit > 1000 { 52 | return nil, fmt.Errorf("Limit (%d) must be 1000 or less", req.Limit) 53 | } 54 | 55 | if req.Limit < 0 { 56 | return nil, fmt.Errorf("Limit (%d) must be greater than zero", req.Limit) 57 | } 58 | 59 | if req.EndTime == 0 { 60 | req.EndTime = time.Now().UnixNano() 61 | } 62 | 63 | if req.Limit == 0 { 64 | req.Limit = 100 65 | } 66 | 67 | var nameFilter *regexp.Regexp 68 | var err error 69 | if req.NameFilter != "" { 70 | nameFilter, err = regexp.Compile(req.NameFilter) 71 | if err != nil { 72 | return nil, fmt.Errorf("Name filter must be a valid regular expression: %s", err) 73 | } 74 | } 75 | 76 | var envelopeTypes []logcache_v1.EnvelopeType 77 | for _, e := range req.GetEnvelopeTypes() { 78 | if e != logcache_v1.EnvelopeType_ANY { 79 | envelopeTypes = append(envelopeTypes, e) 80 | } 81 | } 82 | envs := r.s.Get( 83 | req.SourceId, 84 | time.Unix(0, req.StartTime), 85 | time.Unix(0, req.EndTime), 86 | envelopeTypes, 87 | nameFilter, 88 | int(req.Limit), 89 | req.Descending, 90 | ) 91 | resp := &logcache_v1.ReadResponse{ 92 | Envelopes: &loggregator_v2.EnvelopeBatch{ 93 | Batch: envs, 94 | }, 95 | } 96 | 97 | return resp, nil 98 | } 99 | 100 | func (r *LocalStoreReader) Meta(ctx context.Context, req *logcache_v1.MetaRequest, opts ...grpc.CallOption) (*logcache_v1.MetaResponse, error) { 101 | sourceIds := r.s.Meta() 102 | 103 | metaInfo := make(map[string]*logcache_v1.MetaInfo) 104 | for sourceId, m := range sourceIds { 105 | // Shadow m so that the range function does not mess with the 106 | // instance. 107 | m := m 108 | metaInfo[sourceId] = &m 109 | } 110 | 111 | return &logcache_v1.MetaResponse{ 112 | Meta: metaInfo, 113 | }, nil 114 | } 115 | -------------------------------------------------------------------------------- /internal/routing/orchestrator.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 8 | ) 9 | 10 | // OrchestratorAgent manages the Log Cache node's routes. 11 | type OrchestratorAgent struct { 12 | mu sync.RWMutex 13 | ranges []*rpc.Range 14 | 15 | s RangeSetter 16 | } 17 | 18 | type RangeSetter interface { 19 | // SetRanges is used as a pass through for the orchestration service's 20 | // SetRanges method. 21 | SetRanges(ctx context.Context, in *rpc.SetRangesRequest) (*rpc.SetRangesResponse, error) 22 | } 23 | 24 | // NewOrchestratorAgent returns a new OrchestratorAgent. 25 | func NewOrchestratorAgent(s RangeSetter) *OrchestratorAgent { 26 | return &OrchestratorAgent{ 27 | s: s, 28 | } 29 | } 30 | 31 | // AddRange adds a range (from the scheduler) for data to be routed to. 32 | func (o *OrchestratorAgent) AddRange(ctx context.Context, r *rpc.AddRangeRequest) (*rpc.AddRangeResponse, error) { 33 | o.mu.Lock() 34 | defer o.mu.Unlock() 35 | 36 | o.ranges = append(o.ranges, r.Range) 37 | 38 | return &rpc.AddRangeResponse{}, nil 39 | } 40 | 41 | // RemoveRange removes a range (form the scheduler) for the data to be routed 42 | // to. 43 | func (o *OrchestratorAgent) RemoveRange(ctx context.Context, req *rpc.RemoveRangeRequest) (*rpc.RemoveRangeResponse, error) { 44 | o.mu.Lock() 45 | defer o.mu.Unlock() 46 | 47 | for i, r := range o.ranges { 48 | if r.Start == req.Range.Start && r.End == req.Range.End { 49 | o.ranges = append(o.ranges[:i], o.ranges[i+1:]...) 50 | break 51 | } 52 | } 53 | 54 | return &rpc.RemoveRangeResponse{}, nil 55 | } 56 | 57 | // ListRanges returns all the ranges that are currently active. 58 | func (o *OrchestratorAgent) ListRanges(ctx context.Context, r *rpc.ListRangesRequest) (*rpc.ListRangesResponse, error) { 59 | o.mu.RLock() 60 | defer o.mu.RUnlock() 61 | 62 | return &rpc.ListRangesResponse{ 63 | Ranges: o.ranges, 64 | }, nil 65 | } 66 | 67 | // SetRanges passes them along to the RangeSetter. 68 | func (o *OrchestratorAgent) SetRanges(ctx context.Context, in *rpc.SetRangesRequest) (*rpc.SetRangesResponse, error) { 69 | return o.s.SetRanges(ctx, in) 70 | } 71 | -------------------------------------------------------------------------------- /internal/routing/orchestrator_test.go: -------------------------------------------------------------------------------- 1 | package routing_test 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "code.cloudfoundry.org/log-cache/internal/routing" 8 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 9 | 10 | . "github.com/onsi/ginkgo" 11 | . "github.com/onsi/gomega" 12 | ) 13 | 14 | var _ = Describe("OrchestratorAgent", func() { 15 | var ( 16 | spyRangeSetter *spyRangeSetter 17 | o *routing.OrchestratorAgent 18 | ) 19 | 20 | BeforeEach(func() { 21 | spyRangeSetter = newSpyRangeSetter() 22 | o = routing.NewOrchestratorAgent(spyRangeSetter) 23 | }) 24 | 25 | It("keeps track of the ranges", func() { 26 | _, err := o.AddRange(context.Background(), &rpc.AddRangeRequest{ 27 | Range: &rpc.Range{ 28 | Start: 1, 29 | End: 2, 30 | }, 31 | }) 32 | Expect(err).ToNot(HaveOccurred()) 33 | 34 | _, err = o.AddRange(context.Background(), &rpc.AddRangeRequest{ 35 | Range: &rpc.Range{ 36 | Start: 3, 37 | End: 4, 38 | }, 39 | }) 40 | Expect(err).ToNot(HaveOccurred()) 41 | 42 | resp, err := o.ListRanges(context.Background(), &rpc.ListRangesRequest{}) 43 | Expect(err).ToNot(HaveOccurred()) 44 | Expect(resp.Ranges).To(ConsistOf( 45 | &rpc.Range{ 46 | Start: 1, 47 | End: 2, 48 | }, 49 | 50 | &rpc.Range{ 51 | Start: 3, 52 | End: 4, 53 | }, 54 | )) 55 | 56 | _, err = o.RemoveRange(context.Background(), &rpc.RemoveRangeRequest{ 57 | Range: &rpc.Range{ 58 | Start: 1, 59 | End: 2, 60 | }, 61 | }) 62 | Expect(err).ToNot(HaveOccurred()) 63 | 64 | resp, err = o.ListRanges(context.Background(), &rpc.ListRangesRequest{}) 65 | Expect(err).ToNot(HaveOccurred()) 66 | Expect(resp.Ranges).To(ConsistOf( 67 | &rpc.Range{ 68 | Start: 3, 69 | End: 4, 70 | }, 71 | )) 72 | }) 73 | 74 | It("survives the race detector", func() { 75 | var wg sync.WaitGroup 76 | wg.Add(2) 77 | go func(o *routing.OrchestratorAgent) { 78 | wg.Done() 79 | for i := 0; i < 100; i++ { 80 | o.ListRanges(context.Background(), &rpc.ListRangesRequest{}) 81 | } 82 | }(o) 83 | 84 | go func(o *routing.OrchestratorAgent) { 85 | wg.Done() 86 | for i := 0; i < 100; i++ { 87 | o.RemoveRange(context.Background(), &rpc.RemoveRangeRequest{ 88 | Range: &rpc.Range{ 89 | Start: 1, 90 | End: 2, 91 | }, 92 | }) 93 | } 94 | }(o) 95 | 96 | wg.Wait() 97 | 98 | for i := 0; i < 100; i++ { 99 | o.AddRange(context.Background(), &rpc.AddRangeRequest{ 100 | Range: &rpc.Range{ 101 | Start: 1, 102 | End: 2, 103 | }, 104 | }) 105 | } 106 | }) 107 | 108 | It("passes through SetRanges requests", func() { 109 | expected := &rpc.SetRangesRequest{ 110 | Ranges: map[string]*rpc.Ranges{ 111 | "a": { 112 | Ranges: []*rpc.Range{ 113 | { 114 | Start: 1, 115 | }, 116 | }, 117 | }, 118 | }, 119 | } 120 | o.SetRanges(context.Background(), expected) 121 | Expect(spyRangeSetter.requests).To(ConsistOf(expected)) 122 | }) 123 | }) 124 | 125 | type spyMetaFetcher struct { 126 | results []string 127 | } 128 | 129 | func newSpyMetaFetcher() *spyMetaFetcher { 130 | return &spyMetaFetcher{} 131 | } 132 | 133 | func (s *spyMetaFetcher) Meta() []string { 134 | return s.results 135 | } 136 | 137 | type spyHasher struct { 138 | mu sync.Mutex 139 | ids []string 140 | results []uint64 141 | } 142 | 143 | func newSpyHasher() *spyHasher { 144 | return &spyHasher{} 145 | } 146 | 147 | func (s *spyHasher) Hash(id string) uint64 { 148 | s.mu.Lock() 149 | defer s.mu.Unlock() 150 | s.ids = append(s.ids, id) 151 | 152 | if len(s.results) == 0 { 153 | return 0 154 | } 155 | 156 | r := s.results[0] 157 | s.results = s.results[1:] 158 | return r 159 | } 160 | 161 | type spyRangeSetter struct { 162 | mu sync.Mutex 163 | requests []*rpc.SetRangesRequest 164 | } 165 | 166 | func newSpyRangeSetter() *spyRangeSetter { 167 | return &spyRangeSetter{} 168 | } 169 | 170 | func (s *spyRangeSetter) SetRanges(ctx context.Context, in *rpc.SetRangesRequest) (*rpc.SetRangesResponse, error) { 171 | s.mu.Lock() 172 | defer s.mu.Unlock() 173 | s.requests = append(s.requests, in) 174 | return &rpc.SetRangesResponse{}, nil 175 | } 176 | -------------------------------------------------------------------------------- /internal/routing/routing_suite_test.go: -------------------------------------------------------------------------------- 1 | package routing_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestRouting(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Routing Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/routing/routing_table.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "context" 5 | "sort" 6 | "sync" 7 | 8 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 9 | ) 10 | 11 | // RoutingTable makes decisions for where a item should be routed. 12 | type RoutingTable struct { 13 | mu sync.RWMutex 14 | addrs map[string]int 15 | h func(string) uint64 16 | latestTerm uint64 17 | 18 | table []rangeInfo 19 | } 20 | 21 | // NewRoutingTable returns a new RoutingTable. 22 | func NewRoutingTable(addrs []string, hasher func(string) uint64) *RoutingTable { 23 | a := make(map[string]int) 24 | for i, addr := range addrs { 25 | a[addr] = i 26 | } 27 | 28 | return &RoutingTable{ 29 | addrs: a, 30 | h: hasher, 31 | } 32 | } 33 | 34 | // Lookup takes a item, hash it and determine what node it should be 35 | // routed to. 36 | func (t *RoutingTable) Lookup(item string) []int { 37 | h := t.h(item) 38 | t.mu.RLock() 39 | defer t.mu.RUnlock() 40 | 41 | var result []int 42 | for _, r := range t.table { 43 | if h < r.r.Start || h > r.r.End { 44 | // Outside of range 45 | continue 46 | } 47 | result = append(result, r.idx) 48 | } 49 | 50 | return result 51 | } 52 | 53 | // LookupAll returns every index that has a range where the item would 54 | // fall under. 55 | func (t *RoutingTable) LookupAll(item string) []int { 56 | h := t.h(item) 57 | 58 | t.mu.RLock() 59 | defer t.mu.RUnlock() 60 | 61 | var result []int 62 | ranges := t.table 63 | 64 | for { 65 | i := t.findRange(h, ranges) 66 | if i < 0 { 67 | break 68 | } 69 | result = append(result, ranges[i].idx) 70 | ranges = ranges[i+1:] 71 | } 72 | 73 | return result 74 | } 75 | 76 | // SetRanges sets the routing table. 77 | func (t *RoutingTable) SetRanges(ctx context.Context, in *rpc.SetRangesRequest) (*rpc.SetRangesResponse, error) { 78 | t.mu.Lock() 79 | defer t.mu.Unlock() 80 | 81 | t.table = nil 82 | for addr, ranges := range in.Ranges { 83 | for _, r := range ranges.Ranges { 84 | var sr Range 85 | sr.CloneRpcRange(r) 86 | 87 | t.table = append(t.table, rangeInfo{ 88 | idx: t.addrs[addr], 89 | r: sr, 90 | }) 91 | } 92 | } 93 | 94 | sort.Sort(rangeInfos(t.table)) 95 | 96 | return &rpc.SetRangesResponse{}, nil 97 | } 98 | 99 | func (t *RoutingTable) findRange(h uint64, rs []rangeInfo) int { 100 | for i, r := range rs { 101 | if h < r.r.Start || h > r.r.End { 102 | // Outside of range 103 | continue 104 | } 105 | return i 106 | } 107 | 108 | return -1 109 | } 110 | 111 | type Range struct { 112 | Start uint64 113 | End uint64 114 | } 115 | 116 | func (sr *Range) CloneRpcRange(r *rpc.Range) { 117 | sr.Start = r.Start 118 | sr.End = r.End 119 | } 120 | 121 | func (sr *Range) ToRpcRange() *rpc.Range { 122 | return &rpc.Range{ 123 | Start: sr.Start, 124 | End: sr.End, 125 | } 126 | } 127 | 128 | type rangeInfo struct { 129 | r Range 130 | idx int 131 | } 132 | 133 | type rangeInfos []rangeInfo 134 | 135 | func (r rangeInfos) Len() int { 136 | return len(r) 137 | } 138 | 139 | func (r rangeInfos) Less(i, j int) bool { 140 | if r[i].r.Start == r[j].r.Start { 141 | return r[i].idx > r[j].idx 142 | } 143 | 144 | return r[i].r.Start < r[j].r.Start 145 | } 146 | 147 | func (r rangeInfos) Swap(i, j int) { 148 | tmp := r[i] 149 | r[i] = r[j] 150 | r[j] = tmp 151 | } 152 | -------------------------------------------------------------------------------- /internal/routing/routing_table_test.go: -------------------------------------------------------------------------------- 1 | package routing_test 2 | 3 | import ( 4 | "context" 5 | 6 | "code.cloudfoundry.org/log-cache/internal/routing" 7 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 8 | 9 | . "github.com/onsi/ginkgo" 10 | . "github.com/onsi/gomega" 11 | ) 12 | 13 | var _ = Describe("RoutingTable", func() { 14 | var ( 15 | spyHasher *spyHasher 16 | r *routing.RoutingTable 17 | ) 18 | 19 | BeforeEach(func() { 20 | spyHasher = newSpyHasher() 21 | r = routing.NewRoutingTable([]string{"a", "b", "c", "d"}, spyHasher.Hash) 22 | }) 23 | 24 | It("returns the correct index for the node", func() { 25 | r.SetRanges(context.Background(), &rpc.SetRangesRequest{ 26 | Ranges: map[string]*rpc.Ranges{ 27 | "a": { 28 | Ranges: []*rpc.Range{ 29 | {Start: 0, End: 100}, 30 | }, 31 | }, 32 | "b": { 33 | Ranges: []*rpc.Range{ 34 | {Start: 101, End: 200}, 35 | }, 36 | }, 37 | "c": { 38 | Ranges: []*rpc.Range{ 39 | {Start: 201, End: 300}, 40 | }, 41 | }, 42 | "d": { 43 | Ranges: []*rpc.Range{ 44 | {Start: 101, End: 200}, 45 | }, 46 | }, 47 | }, 48 | }) 49 | 50 | spyHasher.results = []uint64{200} 51 | 52 | i := r.Lookup("some-id") 53 | Expect(spyHasher.ids).To(ConsistOf("some-id")) 54 | Expect(i).To(Equal([]int{3, 1})) 55 | }) 56 | 57 | It("returns the correct index for the node", func() { 58 | r.SetRanges(context.Background(), &rpc.SetRangesRequest{ 59 | Ranges: map[string]*rpc.Ranges{ 60 | "a": { 61 | Ranges: []*rpc.Range{ 62 | {Start: 0, End: 100}, 63 | {Start: 101, End: 200}, 64 | }, 65 | }, 66 | "b": { 67 | Ranges: []*rpc.Range{ 68 | {Start: 101, End: 200}, 69 | }, 70 | }, 71 | "c": { 72 | Ranges: []*rpc.Range{ 73 | {Start: 201, End: 300}, 74 | }, 75 | }, 76 | }, 77 | }) 78 | 79 | spyHasher.results = []uint64{200} 80 | 81 | i := r.LookupAll("some-id") 82 | Expect(spyHasher.ids).To(ConsistOf("some-id")) 83 | Expect(i).To(Equal([]int{1, 0})) 84 | }) 85 | 86 | It("returns an empty slice for a non-routable hash", func() { 87 | i := r.Lookup("some-id") 88 | Expect(i).To(BeEmpty()) 89 | }) 90 | 91 | It("survives the race detector", func() { 92 | go func(r *routing.RoutingTable) { 93 | for i := 0; i < 100; i++ { 94 | r.Lookup("a") 95 | } 96 | }(r) 97 | 98 | go func(r *routing.RoutingTable) { 99 | for i := 0; i < 100; i++ { 100 | r.LookupAll("a") 101 | } 102 | }(r) 103 | 104 | for i := 0; i < 100; i++ { 105 | r.SetRanges(context.Background(), &rpc.SetRangesRequest{}) 106 | } 107 | }) 108 | }) 109 | -------------------------------------------------------------------------------- /internal/routing/static_lookup.go: -------------------------------------------------------------------------------- 1 | package routing 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/emirpasic/gods/trees/avltree" 7 | "github.com/emirpasic/gods/utils" 8 | ) 9 | 10 | // StaticLookup is used to do lookup for static routes. 11 | type StaticLookup struct { 12 | hash func(string) uint64 13 | t *avltree.Tree 14 | } 15 | 16 | // NewStaticLookup creates and returns a StaticLookup. 17 | func NewStaticLookup(numOfRoutes int, hasher func(string) uint64) *StaticLookup { 18 | if numOfRoutes <= 0 { 19 | log.Panicf("Invalid number of routes: %d", numOfRoutes) 20 | } 21 | 22 | t := avltree.NewWith(utils.UInt64Comparator) 23 | 24 | // NOTE: 18446744073709551615 is 0xFFFFFFFFFFFFFFFF or the max value of a 25 | // uint64. 26 | x := (18446744073709551615 / uint64(numOfRoutes)) 27 | for i := uint64(0); i < uint64(numOfRoutes); i++ { 28 | t.Put(i*x, i) 29 | } 30 | 31 | return &StaticLookup{ 32 | hash: hasher, 33 | t: t, 34 | } 35 | } 36 | 37 | // Lookup hashes the SourceId and then returns the index that is in range of 38 | // the hash. 39 | func (l *StaticLookup) Lookup(sourceID string) int { 40 | h := l.hash(sourceID) 41 | n, _ := l.t.Floor(h) 42 | return int(n.Value.(uint64)) 43 | } 44 | -------------------------------------------------------------------------------- /internal/routing/static_lookup_test.go: -------------------------------------------------------------------------------- 1 | package routing_test 2 | 3 | import ( 4 | "code.cloudfoundry.org/log-cache/internal/routing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | var _ = Describe("StaticLookup", func() { 11 | var ( 12 | l *routing.StaticLookup 13 | 14 | sourceId string 15 | hash uint64 16 | ) 17 | 18 | BeforeEach(func() { 19 | l = routing.NewStaticLookup(4, func(s string) uint64 { 20 | sourceId = s 21 | return hash 22 | }) 23 | }) 24 | 25 | It("associates indexes for each route", func() { 26 | // range #0 -> 0 - 4611686018427387902 27 | // range #1 -> 4611686018427387903 - 9223372036854775805 28 | // range #2 -> 9223372036854775806 - 13835058055282163708 29 | // range #3 -> 13835058055282163709 - 18446744073709551615 30 | 31 | // Range #0 32 | hash = 0 33 | i := l.Lookup("source-a") 34 | Expect(i).To(Equal(0)) 35 | 36 | hash = 4611686018427387902 37 | i = l.Lookup("source-a") 38 | Expect(i).To(Equal(0)) 39 | 40 | // Range #1 41 | hash = 4611686018427387903 42 | i = l.Lookup("source-a") 43 | Expect(i).To(Equal(1)) 44 | 45 | hash = 9223372036854775805 46 | i = l.Lookup("source-a") 47 | Expect(i).To(Equal(1)) 48 | 49 | // Range #2 50 | hash = 9223372036854775806 51 | i = l.Lookup("source-a") 52 | Expect(i).To(Equal(2)) 53 | 54 | hash = 13835058055282163708 55 | i = l.Lookup("source-a") 56 | Expect(i).To(Equal(2)) 57 | 58 | // Range #3 59 | hash = 13835058055282163709 60 | i = l.Lookup("source-a") 61 | Expect(i).To(Equal(3)) 62 | 63 | hash = 18446744073709551615 64 | i = l.Lookup("source-a") 65 | Expect(i).To(Equal(3)) 66 | 67 | Expect(sourceId).To(Equal("source-a")) 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /internal/scheduler/scheduler_suite_test.go: -------------------------------------------------------------------------------- 1 | package scheduler_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestScheduler(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Scheduler Suite") 13 | } 14 | -------------------------------------------------------------------------------- /internal/syslog/syslog_suite_test.go: -------------------------------------------------------------------------------- 1 | package syslog_test 2 | 3 | import ( 4 | "log" 5 | "testing" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | func TestSyslog(t *testing.T) { 12 | log.SetOutput(GinkgoWriter) 13 | RegisterFailHandler(Fail) 14 | RunSpecs(t, "Syslog Suite") 15 | } 16 | -------------------------------------------------------------------------------- /internal/testing/acceptance.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "net" 5 | 6 | . "github.com/onsi/gomega" 7 | ) 8 | 9 | func GetFreePort() int { 10 | addr, err := net.ResolveTCPAddr("tcp", "localhost:0") 11 | if err != nil { 12 | Expect(err).ToNot(HaveOccurred()) 13 | } 14 | 15 | l, err := net.ListenTCP("tcp", addr) 16 | if err != nil { 17 | Expect(err).ToNot(HaveOccurred()) 18 | } 19 | defer l.Close() 20 | return l.Addr().(*net.TCPAddr).Port 21 | } 22 | -------------------------------------------------------------------------------- /internal/testing/auth.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "strings" 10 | "time" 11 | 12 | . "github.com/onsi/gomega" 13 | ) 14 | 15 | func NewServerRequest(method, uri string, body io.Reader) (*http.Request, error) { 16 | req, err := http.NewRequest(method, uri, body) 17 | if err != nil { 18 | return nil, err 19 | } 20 | req.Host = req.URL.Host 21 | // set req.TLS if request is https 22 | if req.URL.Scheme == "https" { 23 | req.TLS = &tls.ConnectionState{} 24 | } 25 | // zero out fields that are not available to the server 26 | req.URL = &url.URL{ 27 | Path: req.URL.Path, 28 | RawQuery: req.URL.RawQuery, 29 | } 30 | return req, nil 31 | } 32 | 33 | func BuildRequest(method, url, remoteAddr, requestId, forwardedFor string) *http.Request { 34 | req, err := NewServerRequest(method, url, nil) 35 | Expect(err).ToNot(HaveOccurred()) 36 | req.Header.Add("X-Vcap-Request-ID", requestId) 37 | req.Header.Add("X-Forwarded-For", forwardedFor) 38 | req.RemoteAddr = remoteAddr 39 | return req 40 | } 41 | 42 | func BuildExpectedLog(timestamp time.Time, requestId, method, path, sourceHost, sourcePort, dstHost, dstPort string) string { 43 | extensions := []string{ 44 | fmt.Sprintf("rt=%d", timestamp.UnixNano()/int64(time.Millisecond)), 45 | "cs1Label=userAuthenticationMechanism", 46 | "cs1=oauth-access-token", 47 | "cs2Label=vcapRequestId", 48 | "cs2=" + requestId, 49 | "request=" + path, 50 | "requestMethod=" + method, 51 | "src=" + sourceHost, 52 | "spt=" + sourcePort, 53 | "dst=" + dstHost, 54 | "dpt=" + dstPort, 55 | } 56 | fields := []string{ 57 | "0", 58 | "cloud_foundry", 59 | "log_cache", 60 | "1.0", 61 | method + " " + path, 62 | method + " " + path, 63 | "0", 64 | strings.Join(extensions, " "), 65 | } 66 | return "CEF:" + strings.Join(fields, "|") 67 | } 68 | -------------------------------------------------------------------------------- /internal/testing/certificates.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | //go:generate $GOPATH/scripts/generate-certs 4 | //go:generate go-bindata -nocompress -pkg testing -o bindata.go -prefix certs/ certs/ 5 | //go:generate rm -rf certs/ 6 | 7 | import ( 8 | "io/ioutil" 9 | "log" 10 | ) 11 | 12 | func Cert(filename string) string { 13 | var contents []byte 14 | switch filename { 15 | case "localhost.crt": 16 | contents = localhostCert 17 | case "localhost.key": 18 | contents = localhostKey 19 | default: 20 | contents = MustAsset(filename) 21 | } 22 | 23 | tmpfile, err := ioutil.TempFile("", "") 24 | 25 | if err != nil { 26 | log.Fatal(err) 27 | } 28 | 29 | if _, err := tmpfile.Write(contents); err != nil { 30 | log.Fatal(err) 31 | } 32 | if err := tmpfile.Close(); err != nil { 33 | log.Fatal(err) 34 | } 35 | 36 | return tmpfile.Name() 37 | } 38 | 39 | var localhostCert = []byte(`-----BEGIN CERTIFICATE----- 40 | MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS 41 | MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw 42 | MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB 43 | iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4 44 | iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul 45 | rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO 46 | BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw 47 | AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA 48 | AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9 49 | tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs 50 | h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM 51 | fblo6RBxUQ== 52 | -----END CERTIFICATE-----`) 53 | 54 | var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- 55 | MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 56 | SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB 57 | l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB 58 | AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet 59 | 3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb 60 | uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H 61 | qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp 62 | jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY 63 | fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U 64 | fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU 65 | y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX 66 | qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo 67 | f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== 68 | -----END RSA PRIVATE KEY-----`) 69 | -------------------------------------------------------------------------------- /internal/testing/spy_log_cache.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "log" 7 | "net" 8 | "sync" 9 | 10 | "google.golang.org/grpc" 11 | "google.golang.org/grpc/credentials" 12 | 13 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 14 | rpc "code.cloudfoundry.org/log-cache/pkg/rpc/logcache_v1" 15 | ) 16 | 17 | type SpyAgent struct { 18 | mu sync.Mutex 19 | envelopes []*loggregator_v2.Envelope 20 | tlsConfig *tls.Config 21 | } 22 | 23 | func NewSpyAgent(tlsConfig *tls.Config) *SpyAgent { 24 | return &SpyAgent{ 25 | tlsConfig: tlsConfig, 26 | } 27 | } 28 | 29 | func (s *SpyAgent) Start() string { 30 | lis, err := net.Listen("tcp", ":0") 31 | if err != nil { 32 | log.Fatalf("failed to listen: %v", err) 33 | } 34 | 35 | srv := grpc.NewServer( 36 | grpc.Creds(credentials.NewTLS(s.tlsConfig)), 37 | ) 38 | loggregator_v2.RegisterIngressServer(srv, s) 39 | go srv.Serve(lis) 40 | 41 | return lis.Addr().String() 42 | 43 | } 44 | 45 | func (s *SpyAgent) Sender(_ loggregator_v2.Ingress_SenderServer) error { 46 | return nil 47 | } 48 | 49 | func (s *SpyAgent) BatchSender(_ loggregator_v2.Ingress_BatchSenderServer) error { 50 | return nil 51 | } 52 | 53 | func (s *SpyAgent) Send(ctx context.Context, batch *loggregator_v2.EnvelopeBatch) (*loggregator_v2.SendResponse, error) { 54 | s.mu.Lock() 55 | defer s.mu.Unlock() 56 | 57 | s.envelopes = append(s.envelopes, batch.GetBatch()...) 58 | 59 | return &loggregator_v2.SendResponse{}, nil 60 | } 61 | 62 | func (s *SpyAgent) GetEnvelopes() []*loggregator_v2.Envelope { 63 | s.mu.Lock() 64 | defer s.mu.Unlock() 65 | r := make([]*loggregator_v2.Envelope, len(s.envelopes)) 66 | copy(r, s.envelopes) 67 | return r 68 | } 69 | 70 | type SpyLogCache struct { 71 | mu sync.Mutex 72 | localOnlyValues []bool 73 | envelopes []*loggregator_v2.Envelope 74 | readRequests []*rpc.ReadRequest 75 | queryRequests []*rpc.PromQL_InstantQueryRequest 76 | QueryError error 77 | rangeQueryRequests []*rpc.PromQL_RangeQueryRequest 78 | ReadEnvelopes map[string]func() []*loggregator_v2.Envelope 79 | MetaResponses map[string]*rpc.MetaInfo 80 | tlsConfig *tls.Config 81 | value float64 82 | } 83 | 84 | func (s *SpyLogCache) SetValue(value float64) { 85 | s.mu.Lock() 86 | defer s.mu.Unlock() 87 | s.value = value 88 | } 89 | 90 | func NewSpyLogCache(tlsConfig *tls.Config) *SpyLogCache { 91 | return &SpyLogCache{ 92 | ReadEnvelopes: make(map[string]func() []*loggregator_v2.Envelope), 93 | tlsConfig: tlsConfig, 94 | value: 101, 95 | } 96 | } 97 | 98 | func (s *SpyLogCache) Start() string { 99 | lis, err := net.Listen("tcp", ":0") 100 | if err != nil { 101 | log.Fatalf("failed to listen: %v", err) 102 | } 103 | 104 | srv := grpc.NewServer( 105 | grpc.Creds(credentials.NewTLS(s.tlsConfig)), 106 | ) 107 | rpc.RegisterIngressServer(srv, s) 108 | rpc.RegisterEgressServer(srv, s) 109 | rpc.RegisterPromQLQuerierServer(srv, s) 110 | go srv.Serve(lis) 111 | 112 | return lis.Addr().String() 113 | } 114 | 115 | func (s *SpyLogCache) GetEnvelopes() []*loggregator_v2.Envelope { 116 | s.mu.Lock() 117 | defer s.mu.Unlock() 118 | r := make([]*loggregator_v2.Envelope, len(s.envelopes)) 119 | copy(r, s.envelopes) 120 | return r 121 | } 122 | 123 | func (s *SpyLogCache) GetLocalOnlyValues() []bool { 124 | s.mu.Lock() 125 | defer s.mu.Unlock() 126 | r := make([]bool, len(s.localOnlyValues)) 127 | copy(r, s.localOnlyValues) 128 | return r 129 | } 130 | 131 | func (s *SpyLogCache) GetReadRequests() []*rpc.ReadRequest { 132 | s.mu.Lock() 133 | defer s.mu.Unlock() 134 | r := make([]*rpc.ReadRequest, len(s.readRequests)) 135 | copy(r, s.readRequests) 136 | return r 137 | } 138 | 139 | func (s *SpyLogCache) Send(ctx context.Context, r *rpc.SendRequest) (*rpc.SendResponse, error) { 140 | s.mu.Lock() 141 | defer s.mu.Unlock() 142 | 143 | s.localOnlyValues = append(s.localOnlyValues, r.LocalOnly) 144 | 145 | for _, e := range r.Envelopes.Batch { 146 | s.envelopes = append(s.envelopes, e) 147 | } 148 | 149 | return &rpc.SendResponse{}, nil 150 | } 151 | 152 | func (s *SpyLogCache) Read(ctx context.Context, r *rpc.ReadRequest) (*rpc.ReadResponse, error) { 153 | s.mu.Lock() 154 | defer s.mu.Unlock() 155 | 156 | s.readRequests = append(s.readRequests, r) 157 | 158 | b := s.ReadEnvelopes[r.GetSourceId()] 159 | 160 | var batch []*loggregator_v2.Envelope 161 | if b != nil { 162 | batch = b() 163 | } 164 | 165 | return &rpc.ReadResponse{ 166 | Envelopes: &loggregator_v2.EnvelopeBatch{ 167 | Batch: batch, 168 | }, 169 | }, nil 170 | } 171 | 172 | func (s *SpyLogCache) Meta(ctx context.Context, r *rpc.MetaRequest) (*rpc.MetaResponse, error) { 173 | return &rpc.MetaResponse{ 174 | Meta: s.MetaResponses, 175 | }, nil 176 | } 177 | 178 | func (s *SpyLogCache) InstantQuery(ctx context.Context, r *rpc.PromQL_InstantQueryRequest) (*rpc.PromQL_InstantQueryResult, error) { 179 | s.mu.Lock() 180 | defer s.mu.Unlock() 181 | 182 | s.queryRequests = append(s.queryRequests, r) 183 | 184 | return &rpc.PromQL_InstantQueryResult{ 185 | Result: &rpc.PromQL_InstantQueryResult_Scalar{ 186 | Scalar: &rpc.PromQL_Scalar{ 187 | Time: "99.000", 188 | Value: s.value, 189 | }, 190 | }, 191 | }, s.QueryError 192 | } 193 | 194 | func (s *SpyLogCache) GetQueryRequests() []*rpc.PromQL_InstantQueryRequest { 195 | s.mu.Lock() 196 | defer s.mu.Unlock() 197 | 198 | r := make([]*rpc.PromQL_InstantQueryRequest, len(s.queryRequests)) 199 | copy(r, s.queryRequests) 200 | 201 | return r 202 | } 203 | 204 | func (s *SpyLogCache) RangeQuery(ctx context.Context, r *rpc.PromQL_RangeQueryRequest) (*rpc.PromQL_RangeQueryResult, error) { 205 | s.mu.Lock() 206 | defer s.mu.Unlock() 207 | 208 | s.rangeQueryRequests = append(s.rangeQueryRequests, r) 209 | 210 | return &rpc.PromQL_RangeQueryResult{ 211 | Result: &rpc.PromQL_RangeQueryResult_Matrix{ 212 | Matrix: &rpc.PromQL_Matrix{ 213 | Series: []*rpc.PromQL_Series{ 214 | { 215 | Metric: map[string]string{ 216 | "__name__": "test", 217 | }, 218 | Points: []*rpc.PromQL_Point{ 219 | { 220 | Time: "99.000", 221 | Value: s.value, 222 | }, 223 | }, 224 | }, 225 | }, 226 | }, 227 | }, 228 | }, nil 229 | } 230 | 231 | func (s *SpyLogCache) GetRangeQueryRequests() []*rpc.PromQL_RangeQueryRequest { 232 | s.mu.Lock() 233 | defer s.mu.Unlock() 234 | 235 | r := make([]*rpc.PromQL_RangeQueryRequest, len(s.rangeQueryRequests)) 236 | copy(r, s.rangeQueryRequests) 237 | 238 | return r 239 | } 240 | 241 | func StubUptimeFn() int64 { 242 | return 789 243 | } 244 | -------------------------------------------------------------------------------- /internal/testing/spy_metrics.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | const ( 9 | UNDEFINED_METRIC float64 = 99999999999 10 | ) 11 | 12 | type SpyMetrics struct { 13 | values map[string]float64 14 | units map[string]string 15 | summaryObservations map[string][]float64 16 | sync.Mutex 17 | } 18 | 19 | func NewSpyMetrics() *SpyMetrics { 20 | return &SpyMetrics{ 21 | values: make(map[string]float64), 22 | units: make(map[string]string), 23 | summaryObservations: make(map[string][]float64), 24 | } 25 | } 26 | 27 | func (s *SpyMetrics) NewCounter(name string) func(uint64) { 28 | s.Lock() 29 | defer s.Unlock() 30 | s.values[name] = 0 31 | 32 | return func(value uint64) { 33 | s.Lock() 34 | defer s.Unlock() 35 | 36 | s.values[name] += float64(value) 37 | } 38 | } 39 | 40 | func (s *SpyMetrics) NewPerNodeCounter(name string, nodeIndex int) func(uint64) { 41 | s.Lock() 42 | defer s.Unlock() 43 | metricName := fmt.Sprintf("%s-node%d", name, nodeIndex) 44 | s.values[metricName] = 0 45 | 46 | return func(value uint64) { 47 | s.Lock() 48 | defer s.Unlock() 49 | 50 | s.values[metricName] += float64(value) 51 | } 52 | } 53 | 54 | func (s *SpyMetrics) NewGauge(name, unit string) func(float64) { 55 | s.Lock() 56 | defer s.Unlock() 57 | s.values[name] = 0 58 | s.units[name] = unit 59 | 60 | return func(value float64) { 61 | s.Lock() 62 | defer s.Unlock() 63 | 64 | s.values[name] = value 65 | } 66 | } 67 | 68 | func (s *SpyMetrics) NewSummary(name, unit string) func(float64) { 69 | s.Lock() 70 | defer s.Unlock() 71 | s.summaryObservations[name] = make([]float64, 0) 72 | s.units[name] = unit 73 | 74 | return func(value float64) { 75 | s.Lock() 76 | defer s.Unlock() 77 | 78 | s.summaryObservations[name] = append(s.summaryObservations[name], value) 79 | } 80 | } 81 | 82 | func (s *SpyMetrics) Getter(name string) func() float64 { 83 | return func() float64 { 84 | s.Lock() 85 | defer s.Unlock() 86 | 87 | value, ok := s.values[name] 88 | if !ok { 89 | return UNDEFINED_METRIC 90 | } 91 | return value 92 | } 93 | } 94 | 95 | func (s *SpyMetrics) Get(name string) float64 { 96 | return s.Getter(name)() 97 | } 98 | 99 | func (s *SpyMetrics) GetUnit(name string) string { 100 | s.Lock() 101 | defer s.Unlock() 102 | 103 | return s.units[name] 104 | } 105 | 106 | func (s *SpyMetrics) SummaryObservationsGetter(name string) func() []float64 { 107 | return func() []float64 { 108 | s.Lock() 109 | defer s.Unlock() 110 | 111 | value := s.summaryObservations[name] 112 | return value 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /internal/testing/time.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | 8 | . "github.com/onsi/gomega" 9 | ) 10 | 11 | func FormatTimeWithDecimalMillis(t time.Time) string { 12 | return fmt.Sprintf("%.3f", float64(t.UnixNano())/1e9) 13 | } 14 | 15 | func ParseTimeWithDecimalMillis(t string) time.Time { 16 | decimalTime, err := strconv.ParseFloat(t, 64) 17 | Expect(err).ToNot(HaveOccurred()) 18 | return time.Unix(0, int64(decimalTime*1e9)) 19 | } 20 | -------------------------------------------------------------------------------- /internal/testing/tls_config.go: -------------------------------------------------------------------------------- 1 | package testing 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "errors" 7 | "io/ioutil" 8 | ) 9 | 10 | func NewTLSConfig(caPath, certPath, keyPath, cn string) (*tls.Config, error) { 11 | cert, err := tls.LoadX509KeyPair(certPath, keyPath) 12 | if err != nil { 13 | return nil, err 14 | } 15 | 16 | tlsConfig := &tls.Config{ 17 | ServerName: cn, 18 | Certificates: []tls.Certificate{cert}, 19 | InsecureSkipVerify: false, 20 | } 21 | 22 | caCertBytes, err := ioutil.ReadFile(caPath) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | caCertPool := x509.NewCertPool() 28 | if ok := caCertPool.AppendCertsFromPEM(caCertBytes); !ok { 29 | return nil, errors.New("cannot parse ca cert") 30 | } 31 | 32 | tlsConfig.RootCAs = caCertPool 33 | 34 | return tlsConfig, nil 35 | } 36 | -------------------------------------------------------------------------------- /internal/tls/tls.go: -------------------------------------------------------------------------------- 1 | package tls 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "errors" 7 | "io/ioutil" 8 | "log" 9 | 10 | "google.golang.org/grpc/credentials" 11 | ) 12 | 13 | type TLS struct { 14 | CAPath string `env:"CA_PATH, required, report"` 15 | CertPath string `env:"CERT_PATH, required, report"` 16 | KeyPath string `env:"KEY_PATH, required, report"` 17 | } 18 | 19 | func (t TLS) Credentials(cn string) credentials.TransportCredentials { 20 | creds, err := NewTLSCredentials(t.CAPath, t.CertPath, t.KeyPath, cn) 21 | if err != nil { 22 | log.Fatalf("failed to load TLS config: %s", err) 23 | } 24 | 25 | return creds 26 | } 27 | 28 | func NewTLSCredentials( 29 | caPath string, 30 | certPath string, 31 | keyPath string, 32 | cn string, 33 | ) (credentials.TransportCredentials, error) { 34 | cfg, err := NewMutualTLSConfig(caPath, certPath, keyPath, cn) 35 | if err != nil { 36 | return nil, err 37 | } 38 | 39 | return credentials.NewTLS(cfg), nil 40 | } 41 | 42 | func NewMutualTLSConfig(caPath, certPath, keyPath, cn string) (*tls.Config, error) { 43 | cert, err := tls.LoadX509KeyPair(certPath, keyPath) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | tlsConfig := NewBaseTLSConfig() 49 | tlsConfig.ServerName = cn 50 | tlsConfig.Certificates = []tls.Certificate{cert} 51 | 52 | caCertBytes, err := ioutil.ReadFile(caPath) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | caCertPool := x509.NewCertPool() 58 | if ok := caCertPool.AppendCertsFromPEM(caCertBytes); !ok { 59 | return nil, errors.New("cannot parse ca cert") 60 | } 61 | 62 | tlsConfig.RootCAs = caCertPool 63 | 64 | return tlsConfig, nil 65 | } 66 | 67 | func NewBaseTLSConfig() *tls.Config { 68 | return &tls.Config{ 69 | InsecureSkipVerify: false, 70 | MinVersion: tls.VersionTLS12, 71 | CipherSuites: supportedCipherSuites, 72 | } 73 | } 74 | 75 | var supportedCipherSuites = []uint16{ 76 | tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 77 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 78 | } 79 | -------------------------------------------------------------------------------- /pkg/client/.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /pkg/client/NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-Present CloudFoundry.org Foundation, Inc. All Rights Reserved. 2 | 3 | This product is licensed to you under the Apache License, Version 2.0 (the "License"). 4 | You may not use this product except in compliance with the License. 5 | 6 | This product may include a number of subcomponents with separate copyright notices 7 | and license terms. Your use of these subcomponents is subject to the terms and 8 | conditions of the subcomponent's license, as noted in the LICENSE file. 9 | -------------------------------------------------------------------------------- /pkg/client/README.md: -------------------------------------------------------------------------------- 1 | # Log Cache Client 2 | 3 | [![GoDoc][go-doc-badge]][go-doc] [![slack.cloudfoundry.org][slack-badge]][log-cache-slack] 4 | 5 | This is a golang client library for [Log-Cache][log-cache]. 6 | 7 | ## Usage 8 | 9 | This repository should be imported as: 10 | 11 | `import logcache "code.cloudfoundry.org/log-cache/pkg/client"` 12 | 13 | **NOTE**: This client library is compatible with `log-cache` versions 1.5.x and 14 | above. For versions 1.4.x and below, please use [`go-log-cache`][go-log-cache]. 15 | 16 | Also, the master branch of `log-cache/pkg/client` reflects current (potentially 17 | development) state of `log-cache-release`. Branches are provided for 18 | compatibility with released versions of `log-cache`. 19 | 20 | [slack-badge]: https://slack.cloudfoundry.org/badge.svg 21 | [log-cache-slack]: https://cloudfoundry.slack.com/archives/log-cache 22 | [log-cache]: https://code.cloudfoundry.org/log-cache 23 | [go-doc-badge]: https://godoc.org/code.cloudfoundry.org/log-cache/client?status.svg 24 | [go-doc]: https://godoc.org/code.cloudfoundry.org/log-cache/pkg/client 25 | [go-log-cache]: https://github.com/cloudfoundry/go-log-cache 26 | -------------------------------------------------------------------------------- /pkg/client/client_suite_test.go: -------------------------------------------------------------------------------- 1 | package client_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | // TODO - the tests here are not actually ginkgo; this is just to get 11 | // scripts/test to work temporarily, and not fail on an unknown ginkgo flag 12 | // [#160701220] 13 | 14 | func TestClient(t *testing.T) { 15 | RegisterFailHandler(Fail) 16 | RunSpecs(t, "Client Suite") 17 | } 18 | -------------------------------------------------------------------------------- /pkg/client/examples/cf_user_auth/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "os" 8 | "strings" 9 | 10 | "code.cloudfoundry.org/log-cache/pkg/client" 11 | ) 12 | 13 | func main() { 14 | 15 | logCacheAddr := os.Getenv("LOG_CACHE_ADDR") 16 | uaaAddr := os.Getenv("UAA_ADDR") 17 | uaaClient := os.Getenv("UAA_CLIENT") 18 | uaaClientSecret := os.Getenv("UAA_CLIENT_SECRET") 19 | username := os.Getenv("USERNAME") 20 | password := os.Getenv("PASSWORD") 21 | 22 | var missing []string 23 | 24 | if logCacheAddr == "" { 25 | missing = append(missing, "LOG_CACHE_ADDR") 26 | } 27 | if uaaAddr == "" { 28 | missing = append(missing, "UAA_ADDR") 29 | } 30 | if uaaClient == "" { 31 | missing = append(missing, "UAA_CLIENT") 32 | } 33 | if username == "" { 34 | missing = append(missing, "USERNAME") 35 | } 36 | if password == "" { 37 | missing = append(missing, "PASSWORD") 38 | } 39 | 40 | if len(missing) > 0 { 41 | panic(fmt.Sprintf("missing required environment variables: %s", strings.Join(missing, ", "))) 42 | } 43 | 44 | c := client.NewOauth2HTTPClient(uaaAddr, uaaClient, uaaClientSecret, 45 | client.WithOauth2HTTPUser(username, password), 46 | ) 47 | 48 | req, err := http.NewRequest(http.MethodGet, logCacheAddr+"/api/v1/meta", nil) 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | resp, err := c.Do(req) 54 | if err != nil { 55 | panic(err) 56 | } 57 | 58 | b, err := ioutil.ReadAll(resp.Body) 59 | if err != nil { 60 | panic(err) 61 | } 62 | 63 | fmt.Println(string(b)) 64 | } 65 | 66 | func verifyPresent(name, envVar string) { 67 | if envVar == "" { 68 | panic(fmt.Errorf("")) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /pkg/client/examples/promql/.gitignore: -------------------------------------------------------------------------------- 1 | promql 2 | -------------------------------------------------------------------------------- /pkg/client/examples/promql/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "errors" 8 | "fmt" 9 | "io/ioutil" 10 | "log" 11 | "os" 12 | 13 | envstruct "code.cloudfoundry.org/go-envstruct" 14 | "code.cloudfoundry.org/log-cache/pkg/client" 15 | "github.com/golang/protobuf/jsonpb" 16 | "google.golang.org/grpc" 17 | "google.golang.org/grpc/credentials" 18 | ) 19 | 20 | func main() { 21 | if len(os.Args) != 2 { 22 | log.Fatalf("usage: %s ", os.Args[0]) 23 | } 24 | 25 | cfg, err := LoadConfig() 26 | if err != nil { 27 | log.Fatalf("invalid configuration: %s", err) 28 | } 29 | 30 | client := client.NewClient( 31 | cfg.LogCacheAddr, 32 | client.WithViaGRPC( 33 | grpc.WithTransportCredentials(cfg.TLS.Credentials("log-cache")), 34 | ), 35 | ) 36 | 37 | result, err := client.PromQL(context.Background(), os.Args[1]) 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | m := &jsonpb.Marshaler{} 43 | str, err := m.MarshalToString(result) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | 48 | fmt.Println(str) 49 | } 50 | 51 | // Config is the configuration for a LogCache Gateway. 52 | type Config struct { 53 | LogCacheAddr string `env:"LOG_CACHE_ADDR, required"` 54 | TLS TLS 55 | } 56 | 57 | type TLS struct { 58 | CAPath string `env:"CA_PATH, required"` 59 | CertPath string `env:"CERT_PATH, required"` 60 | KeyPath string `env:"KEY_PATH, required"` 61 | } 62 | 63 | func (t TLS) Credentials(cn string) credentials.TransportCredentials { 64 | creds, err := NewTLSCredentials(t.CAPath, t.CertPath, t.KeyPath, cn) 65 | if err != nil { 66 | log.Fatalf("failed to load TLS config: %s", err) 67 | } 68 | 69 | return creds 70 | } 71 | 72 | func NewTLSCredentials( 73 | caPath string, 74 | certPath string, 75 | keyPath string, 76 | cn string, 77 | ) (credentials.TransportCredentials, error) { 78 | cfg, err := NewTLSConfig(caPath, certPath, keyPath, cn) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | return credentials.NewTLS(cfg), nil 84 | } 85 | 86 | func NewTLSConfig(caPath, certPath, keyPath, cn string) (*tls.Config, error) { 87 | cert, err := tls.LoadX509KeyPair(certPath, keyPath) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | tlsConfig := &tls.Config{ 93 | ServerName: cn, 94 | Certificates: []tls.Certificate{cert}, 95 | InsecureSkipVerify: false, 96 | } 97 | 98 | caCertBytes, err := ioutil.ReadFile(caPath) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | caCertPool := x509.NewCertPool() 104 | if ok := caCertPool.AppendCertsFromPEM(caCertBytes); !ok { 105 | return nil, errors.New("cannot parse ca cert") 106 | } 107 | 108 | tlsConfig.RootCAs = caCertPool 109 | 110 | return tlsConfig, nil 111 | } 112 | 113 | func LoadConfig() (*Config, error) { 114 | c := Config{} 115 | 116 | if err := envstruct.Load(&c); err != nil { 117 | return nil, err 118 | } 119 | 120 | return &c, nil 121 | } 122 | -------------------------------------------------------------------------------- /pkg/client/examples/sliding_window/.gitignore: -------------------------------------------------------------------------------- 1 | sliding_window 2 | -------------------------------------------------------------------------------- /pkg/client/examples/sliding_window/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "time" 9 | 10 | envstruct "code.cloudfoundry.org/go-envstruct" 11 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 12 | "code.cloudfoundry.org/log-cache/pkg/client" 13 | ) 14 | 15 | func main() { 16 | cfg := loadConfig() 17 | 18 | httpClient := newHTTPClient(cfg) 19 | 20 | logcache_client := client.NewClient(cfg.Addr, client.WithHTTPClient(httpClient)) 21 | 22 | visitor := func(es []*loggregator_v2.Envelope) bool { 23 | fmt.Println("*********************Start Window********************") 24 | defer fmt.Println("**********************End Window*********************") 25 | for _, e := range es { 26 | if cfg.PrintTimestamps { 27 | fmt.Printf("%d\n", time.Unix(0, e.GetTimestamp()).Unix()) 28 | continue 29 | } 30 | 31 | fmt.Printf("%+v\n", e) 32 | } 33 | return true 34 | } 35 | 36 | walker := client.BuildWalker(cfg.SourceID, logcache_client.Read) 37 | client.Window( 38 | context.Background(), 39 | visitor, 40 | walker, 41 | client.WithWindowWidth(cfg.WindowWidth), 42 | client.WithWindowInterval(cfg.WindowInterval), 43 | client.WithWindowStartTime(time.Unix(0, cfg.StartTime)), 44 | ) 45 | } 46 | 47 | type config struct { 48 | Addr string `env:"ADDR, required"` 49 | AuthToken string `env:"AUTH_TOKEN, required"` 50 | SourceID string `env:"SOURCE_ID, required"` 51 | WindowInterval time.Duration `env:"WINDOW_INTERVAL"` 52 | WindowWidth time.Duration `env:"WINDOW_WIDTH"` 53 | StartTime int64 `env:"START_TIME"` 54 | PrintTimestamps bool `env:"PRINT_TIMESTAMP"` 55 | } 56 | 57 | func loadConfig() config { 58 | c := config{ 59 | WindowWidth: time.Hour, 60 | WindowInterval: time.Minute, 61 | } 62 | 63 | if err := envstruct.Load(&c); err != nil { 64 | log.Fatal(err) 65 | } 66 | 67 | if c.StartTime == 0 { 68 | c.StartTime = time.Now().Add(-c.WindowWidth).UnixNano() 69 | } 70 | 71 | return c 72 | } 73 | 74 | type HTTPClient struct { 75 | cfg config 76 | client *http.Client 77 | } 78 | 79 | func newHTTPClient(c config) *HTTPClient { 80 | return &HTTPClient{cfg: c, client: http.DefaultClient} 81 | } 82 | 83 | func (h *HTTPClient) Do(req *http.Request) (*http.Response, error) { 84 | req.Header.Set("Authorization", h.cfg.AuthToken) 85 | return h.client.Do(req) 86 | } 87 | -------------------------------------------------------------------------------- /pkg/client/examples/walker/.gitignore: -------------------------------------------------------------------------------- 1 | walker 2 | -------------------------------------------------------------------------------- /pkg/client/examples/walker/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "os" 9 | "time" 10 | 11 | envstruct "code.cloudfoundry.org/go-envstruct" 12 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 13 | "code.cloudfoundry.org/log-cache/pkg/client" 14 | ) 15 | 16 | func main() { 17 | log := log.New(os.Stderr, "", 0) 18 | cfg := loadConfig() 19 | 20 | httpClient := newHTTPClient(cfg) 21 | 22 | logcache_client := client.NewClient(cfg.Addr, client.WithHTTPClient(httpClient)) 23 | 24 | visitor := func(es []*loggregator_v2.Envelope) bool { 25 | for _, e := range es { 26 | fmt.Printf("%+v\n", e) 27 | } 28 | return true 29 | } 30 | 31 | now := time.Now() 32 | 33 | opts := []client.WalkOption{ 34 | client.WithWalkBackoff(client.NewAlwaysRetryBackoff(time.Second)), 35 | client.WithWalkLogger(log), 36 | } 37 | 38 | if cfg.Duration != 0 { 39 | log.Printf("Have duration (%v). Walking finite window...", cfg.Duration) 40 | opts = append(opts, 41 | client.WithWalkStartTime(now.Add(-cfg.Duration)), 42 | client.WithWalkEndTime(now), 43 | ) 44 | } 45 | 46 | client.Walk( 47 | context.Background(), 48 | cfg.SourceID, 49 | visitor, 50 | logcache_client.Read, 51 | opts..., 52 | ) 53 | } 54 | 55 | type config struct { 56 | Addr string `env:"ADDR, required"` 57 | AuthToken string `env:"AUTH_TOKEN, required"` 58 | SourceID string `env:"SOURCE_ID, required"` 59 | Duration time.Duration `env:"DURATION"` 60 | } 61 | 62 | func loadConfig() config { 63 | c := config{} 64 | 65 | if err := envstruct.Load(&c); err != nil { 66 | log.Fatal(err) 67 | } 68 | 69 | return c 70 | } 71 | 72 | type HTTPClient struct { 73 | cfg config 74 | client *http.Client 75 | } 76 | 77 | func newHTTPClient(c config) *HTTPClient { 78 | return &HTTPClient{cfg: c, client: http.DefaultClient} 79 | } 80 | 81 | func (h *HTTPClient) Do(req *http.Request) (*http.Response, error) { 82 | req.Header.Set("Authorization", h.cfg.AuthToken) 83 | return h.client.Do(req) 84 | } 85 | -------------------------------------------------------------------------------- /pkg/client/oauth2_http_client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | // Oauth2HTTPClient sets the "Authorization" header of any outgoing request. 14 | // It gets a JWT from the configured Oauth2 server. It only gets a new JWT 15 | // when a request comes back with a 401. 16 | type Oauth2HTTPClient struct { 17 | c HTTPClient 18 | oauth2Addr string 19 | client string 20 | clientSecret string 21 | 22 | username string 23 | userPassword string 24 | 25 | mu sync.Mutex 26 | token string 27 | } 28 | 29 | // NewOauth2HTTPClient creates a new Oauth2HTTPClient. 30 | func NewOauth2HTTPClient(oauth2Addr, client, clientSecret string, opts ...Oauth2Option) *Oauth2HTTPClient { 31 | c := &Oauth2HTTPClient{ 32 | oauth2Addr: oauth2Addr, 33 | client: client, 34 | clientSecret: clientSecret, 35 | 36 | c: &http.Client{ 37 | Timeout: 5 * time.Second, 38 | }, 39 | } 40 | 41 | for _, o := range opts { 42 | o.configure(c) 43 | } 44 | 45 | return c 46 | } 47 | 48 | // Oauth2Option configures the Oauth2HTTPClient. 49 | type Oauth2Option interface { 50 | configure(c *Oauth2HTTPClient) 51 | } 52 | 53 | // WithOauth2HTTPClient sets the HTTPClient for the Oauth2HTTPClient. It 54 | // defaults to the same default as Client. 55 | func WithOauth2HTTPClient(client HTTPClient) Oauth2Option { 56 | return oauth2HTTPClientOptionFunc(func(c *Oauth2HTTPClient) { 57 | c.c = client 58 | }) 59 | } 60 | 61 | // WithOauth2HTTPUser sets the username and password for user authentication. 62 | func WithOauth2HTTPUser(username, password string) Oauth2Option { 63 | return oauth2HTTPClientOptionFunc(func(c *Oauth2HTTPClient) { 64 | c.username = username 65 | c.userPassword = password 66 | }) 67 | } 68 | 69 | // oauth2HTTPClientOptionFunc enables a function to be a 70 | // Oauth2Option. 71 | type oauth2HTTPClientOptionFunc func(c *Oauth2HTTPClient) 72 | 73 | // configure implements Oauth2Option. 74 | func (f oauth2HTTPClientOptionFunc) configure(c *Oauth2HTTPClient) { 75 | f(c) 76 | } 77 | 78 | // Do implements HTTPClient. It adds the Authorization header to the request 79 | // (unless the header already exists). If the token is expired, it will reach 80 | // out the Oauth2 server and get a new one. The given error CAN be from the 81 | // request to the Oauth2 server. 82 | // 83 | // Do modifies the given Request. It is invalid to use the same Request 84 | // instance on multiple go-routines. 85 | func (c *Oauth2HTTPClient) Do(req *http.Request) (*http.Response, error) { 86 | if _, ok := req.Header["Authorization"]; ok { 87 | // Authorization Header is pre-populated, so just do the request. 88 | return c.c.Do(req) 89 | } 90 | 91 | token, err := c.getToken() 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | req.Header.Set("Authorization", token) 97 | 98 | resp, err := c.c.Do(req) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { 104 | c.token = "" 105 | return resp, nil 106 | } 107 | 108 | return resp, nil 109 | } 110 | 111 | func (c *Oauth2HTTPClient) getToken() (string, error) { 112 | if c.username == "" { 113 | return c.getClientToken() 114 | } 115 | return c.getUserToken() 116 | } 117 | 118 | func (c *Oauth2HTTPClient) getClientToken() (string, error) { 119 | c.mu.Lock() 120 | defer c.mu.Unlock() 121 | 122 | if c.token != "" { 123 | return c.token, nil 124 | } 125 | 126 | v := make(url.Values) 127 | v.Set("client_id", c.client) 128 | v.Set("grant_type", "client_credentials") 129 | 130 | req, err := http.NewRequest( 131 | "POST", 132 | c.oauth2Addr, 133 | strings.NewReader(v.Encode()), 134 | ) 135 | if err != nil { 136 | return "", err 137 | } 138 | req.URL.Path = "/oauth/token" 139 | 140 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 141 | 142 | req.URL.User = url.UserPassword(c.client, c.clientSecret) 143 | 144 | return c.doTokenRequest(req) 145 | } 146 | 147 | func (c *Oauth2HTTPClient) getUserToken() (string, error) { 148 | c.mu.Lock() 149 | defer c.mu.Unlock() 150 | 151 | if c.token != "" { 152 | return c.token, nil 153 | } 154 | 155 | v := make(url.Values) 156 | v.Set("client_id", c.client) 157 | v.Set("client_secret", c.clientSecret) 158 | v.Set("grant_type", "password") 159 | v.Set("username", c.username) 160 | v.Set("password", c.userPassword) 161 | 162 | req, err := http.NewRequest( 163 | "POST", 164 | c.oauth2Addr, 165 | strings.NewReader(v.Encode()), 166 | ) 167 | if err != nil { 168 | return "", err 169 | } 170 | req.URL.Path = "/oauth/token" 171 | 172 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") 173 | 174 | return c.doTokenRequest(req) 175 | } 176 | 177 | func (c *Oauth2HTTPClient) doTokenRequest(req *http.Request) (string, error) { 178 | resp, err := c.c.Do(req) 179 | if err != nil { 180 | return "", err 181 | } 182 | 183 | defer resp.Body.Close() 184 | 185 | if resp.StatusCode != http.StatusOK { 186 | return "", fmt.Errorf("unexpected status code from Oauth2 server %d", resp.StatusCode) 187 | } 188 | 189 | token := struct { 190 | TokenType string `json:"token_type"` 191 | AccessToken string `json:"access_token"` 192 | }{} 193 | 194 | if err := json.NewDecoder(resp.Body).Decode(&token); err != nil { 195 | return "", fmt.Errorf("failed to unmarshal response from Oauth2 server: %s", err) 196 | } 197 | 198 | c.token = fmt.Sprintf("%s %s", token.TokenType, token.AccessToken) 199 | return c.token, nil 200 | } 201 | -------------------------------------------------------------------------------- /pkg/client/window.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "log" 7 | "time" 8 | 9 | "code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2" 10 | ) 11 | 12 | // Window crawls a reader incrementally to give the Visitor a batch of 13 | // envelopes. Each start time is incremented by the set increment duration if 14 | // that window produced data or not. This is useful when looking for trends 15 | // over time. 16 | func Window(ctx context.Context, v Visitor, w Walker, opts ...WindowOption) { 17 | c := windowConfig{ 18 | log: log.New(ioutil.Discard, "", 0), 19 | width: time.Hour, 20 | interval: time.Minute, 21 | } 22 | 23 | for _, o := range opts { 24 | o.configure(&c) 25 | } 26 | 27 | if c.start.IsZero() { 28 | c.start = time.Now().Add(-c.width) 29 | } 30 | 31 | ticker := time.NewTicker(c.interval) 32 | defer ticker.Stop() 33 | for range ticker.C { 34 | walkCtx, cancel := context.WithTimeout(ctx, c.interval) 35 | 36 | es := w(walkCtx, c.start, c.start.Add(c.width)) 37 | cancel() 38 | if !v(es) { 39 | return 40 | } 41 | c.start = c.start.Add(c.interval) 42 | } 43 | } 44 | 45 | // Walker walks a reader. It makes several calls to get all the data between 46 | // a boundary of time. 47 | type Walker func( 48 | ctx context.Context, 49 | start time.Time, 50 | end time.Time, 51 | ) []*loggregator_v2.Envelope 52 | 53 | // BuildWalker captures the sourceID and reader to be used with a Walker. 54 | func BuildWalker(sourceID string, r Reader) Walker { 55 | return func(ctx context.Context, start, end time.Time) []*loggregator_v2.Envelope { 56 | var results []*loggregator_v2.Envelope 57 | Walk(ctx, sourceID, func(e []*loggregator_v2.Envelope) bool { 58 | results = append(results, e...) 59 | return true 60 | }, r, 61 | WithWalkStartTime(start), 62 | WithWalkEndTime(end), 63 | ) 64 | 65 | return results 66 | } 67 | } 68 | 69 | // WindowOption configures the Window algorithm. 70 | type WindowOption interface { 71 | configure(*windowConfig) 72 | } 73 | 74 | // WithWindowLogger is used to set the logger for the Walk. It defaults to 75 | // not logging. 76 | func WithWindowLogger(l *log.Logger) WindowOption { 77 | return windowOptionFunc(func(c *windowConfig) { 78 | c.log = l 79 | }) 80 | } 81 | 82 | // WithWindowStartTime sets the start time of the query. It defaults to 83 | // Now-Width. 84 | func WithWindowStartTime(t time.Time) WindowOption { 85 | return windowOptionFunc(func(c *windowConfig) { 86 | c.start = t 87 | }) 88 | } 89 | 90 | // WithWindowWidth sets the width (end-start=width) of the query. It defaults 91 | // to an hour. 92 | func WithWindowWidth(w time.Duration) WindowOption { 93 | return windowOptionFunc(func(c *windowConfig) { 94 | c.width = w 95 | }) 96 | } 97 | 98 | // WithWindowInterval sets the duration to advance the start and end of the 99 | // query. It defaults to a minute. 100 | func WithWindowInterval(i time.Duration) WindowOption { 101 | return windowOptionFunc(func(c *windowConfig) { 102 | c.interval = i 103 | }) 104 | } 105 | 106 | type windowConfig struct { 107 | log *log.Logger 108 | start time.Time 109 | width time.Duration 110 | interval time.Duration 111 | } 112 | 113 | // windowOptionFunc enables functions to implement WindowOption 114 | type windowOptionFunc func(c *windowConfig) 115 | 116 | // configure implements WindowOption. 117 | func (f windowOptionFunc) configure(c *windowConfig) { 118 | f(c) 119 | } 120 | -------------------------------------------------------------------------------- /pkg/marshaler/promql_suite_test.go: -------------------------------------------------------------------------------- 1 | package marshaler_test 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | 7 | "testing" 8 | ) 9 | 10 | func TestPromql(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Marshaler Suite") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/rpc/logcache_v1/doc.go: -------------------------------------------------------------------------------- 1 | package logcache_v1 2 | 3 | //go:generate ./generate.sh 4 | -------------------------------------------------------------------------------- /pkg/rpc/logcache_v1/egress.pb.gw.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. 2 | // source: egress.proto 3 | 4 | /* 5 | Package logcache_v1 is a reverse proxy. 6 | 7 | It translates gRPC into RESTful JSON APIs. 8 | */ 9 | package logcache_v1 10 | 11 | import ( 12 | "io" 13 | "net/http" 14 | 15 | "github.com/golang/protobuf/proto" 16 | "github.com/grpc-ecosystem/grpc-gateway/runtime" 17 | "github.com/grpc-ecosystem/grpc-gateway/utilities" 18 | "golang.org/x/net/context" 19 | "google.golang.org/grpc" 20 | "google.golang.org/grpc/codes" 21 | "google.golang.org/grpc/grpclog" 22 | "google.golang.org/grpc/status" 23 | ) 24 | 25 | var _ codes.Code 26 | var _ io.Reader 27 | var _ status.Status 28 | var _ = runtime.String 29 | var _ = utilities.NewDoubleArray 30 | 31 | var ( 32 | filter_Egress_Read_0 = &utilities.DoubleArray{Encoding: map[string]int{"source_id": 0}, Base: []int{1, 1, 0}, Check: []int{0, 1, 2}} 33 | ) 34 | 35 | func request_Egress_Read_0(ctx context.Context, marshaler runtime.Marshaler, client EgressClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { 36 | var protoReq ReadRequest 37 | var metadata runtime.ServerMetadata 38 | 39 | var ( 40 | val string 41 | ok bool 42 | err error 43 | _ = err 44 | ) 45 | 46 | val, ok = pathParams["source_id"] 47 | if !ok { 48 | return nil, metadata, status.Errorf(codes.InvalidArgument, "missing parameter %s", "source_id") 49 | } 50 | 51 | protoReq.SourceId, err = runtime.String(val) 52 | 53 | if err != nil { 54 | return nil, metadata, status.Errorf(codes.InvalidArgument, "type mismatch, parameter: %s, error: %v", "source_id", err) 55 | } 56 | 57 | if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_Egress_Read_0); err != nil { 58 | return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) 59 | } 60 | 61 | msg, err := client.Read(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) 62 | return msg, metadata, err 63 | 64 | } 65 | 66 | var ( 67 | filter_Egress_Meta_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} 68 | ) 69 | 70 | func request_Egress_Meta_0(ctx context.Context, marshaler runtime.Marshaler, client EgressClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { 71 | var protoReq MetaRequest 72 | var metadata runtime.ServerMetadata 73 | 74 | if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_Egress_Meta_0); err != nil { 75 | return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) 76 | } 77 | 78 | msg, err := client.Meta(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) 79 | return msg, metadata, err 80 | 81 | } 82 | 83 | // RegisterEgressHandlerFromEndpoint is same as RegisterEgressHandler but 84 | // automatically dials to "endpoint" and closes the connection when "ctx" gets done. 85 | func RegisterEgressHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { 86 | conn, err := grpc.Dial(endpoint, opts...) 87 | if err != nil { 88 | return err 89 | } 90 | defer func() { 91 | if err != nil { 92 | if cerr := conn.Close(); cerr != nil { 93 | grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr) 94 | } 95 | return 96 | } 97 | go func() { 98 | <-ctx.Done() 99 | if cerr := conn.Close(); cerr != nil { 100 | grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr) 101 | } 102 | }() 103 | }() 104 | 105 | return RegisterEgressHandler(ctx, mux, conn) 106 | } 107 | 108 | // RegisterEgressHandler registers the http handlers for service Egress to "mux". 109 | // The handlers forward requests to the grpc endpoint over "conn". 110 | func RegisterEgressHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { 111 | return RegisterEgressHandlerClient(ctx, mux, NewEgressClient(conn)) 112 | } 113 | 114 | // RegisterEgressHandlerClient registers the http handlers for service Egress 115 | // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "EgressClient". 116 | // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "EgressClient" 117 | // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in 118 | // "EgressClient" to call the correct interceptors. 119 | func RegisterEgressHandlerClient(ctx context.Context, mux *runtime.ServeMux, client EgressClient) error { 120 | 121 | mux.Handle("GET", pattern_Egress_Read_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { 122 | ctx, cancel := context.WithCancel(req.Context()) 123 | defer cancel() 124 | if cn, ok := w.(http.CloseNotifier); ok { 125 | go func(done <-chan struct{}, closed <-chan bool) { 126 | select { 127 | case <-done: 128 | case <-closed: 129 | cancel() 130 | } 131 | }(ctx.Done(), cn.CloseNotify()) 132 | } 133 | inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) 134 | rctx, err := runtime.AnnotateContext(ctx, mux, req) 135 | if err != nil { 136 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 137 | return 138 | } 139 | resp, md, err := request_Egress_Read_0(rctx, inboundMarshaler, client, req, pathParams) 140 | ctx = runtime.NewServerMetadataContext(ctx, md) 141 | if err != nil { 142 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 143 | return 144 | } 145 | 146 | forward_Egress_Read_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) 147 | 148 | }) 149 | 150 | mux.Handle("GET", pattern_Egress_Meta_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { 151 | ctx, cancel := context.WithCancel(req.Context()) 152 | defer cancel() 153 | if cn, ok := w.(http.CloseNotifier); ok { 154 | go func(done <-chan struct{}, closed <-chan bool) { 155 | select { 156 | case <-done: 157 | case <-closed: 158 | cancel() 159 | } 160 | }(ctx.Done(), cn.CloseNotify()) 161 | } 162 | inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) 163 | rctx, err := runtime.AnnotateContext(ctx, mux, req) 164 | if err != nil { 165 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 166 | return 167 | } 168 | resp, md, err := request_Egress_Meta_0(rctx, inboundMarshaler, client, req, pathParams) 169 | ctx = runtime.NewServerMetadataContext(ctx, md) 170 | if err != nil { 171 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 172 | return 173 | } 174 | 175 | forward_Egress_Meta_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) 176 | 177 | }) 178 | 179 | return nil 180 | } 181 | 182 | var ( 183 | pattern_Egress_Read_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2, 3, 0, 4, 1, 5, 3}, []string{"api", "v1", "read", "source_id"}, "")) 184 | 185 | pattern_Egress_Meta_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "meta"}, "")) 186 | ) 187 | 188 | var ( 189 | forward_Egress_Read_0 = runtime.ForwardResponseMessage 190 | 191 | forward_Egress_Meta_0 = runtime.ForwardResponseMessage 192 | ) 193 | -------------------------------------------------------------------------------- /pkg/rpc/logcache_v1/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | dir_resolve() 4 | { 5 | cd "$1" 2>/dev/null || return $? # cd to desired directory; if fail, quell any error messages but return exit status 6 | echo "`pwd -P`" # output full, link-resolved path 7 | } 8 | 9 | set -e 10 | 11 | TARGET=`dirname $0` 12 | TARGET=`dir_resolve $TARGET` 13 | cd $TARGET 14 | 15 | go get github.com/golang/protobuf/{proto,protoc-gen-go} 16 | go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway 17 | 18 | 19 | tmp_dir=$(mktemp -d) 20 | mkdir -p $tmp_dir/log-cache 21 | 22 | cp $GOPATH/src/code.cloudfoundry.org/log-cache/api/v1/*proto $tmp_dir/log-cache 23 | 24 | protoc \ 25 | $tmp_dir/log-cache/*.proto \ 26 | --go_out=plugins=grpc,Mv2/envelope.proto=code.cloudfoundry.org/go-loggregator/rpc/loggregator_v2:. \ 27 | --proto_path=$tmp_dir/log-cache \ 28 | --grpc-gateway_out=logtostderr=true:. \ 29 | -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \ 30 | -I=/usr/local/include \ 31 | -I=$tmp_dir/log-cache \ 32 | -I=$GOPATH/src/code.cloudfoundry.org/loggregator-api/. 33 | 34 | rm -r $tmp_dir 35 | -------------------------------------------------------------------------------- /pkg/rpc/logcache_v1/promql.pb.gw.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-grpc-gateway. DO NOT EDIT. 2 | // source: promql.proto 3 | 4 | /* 5 | Package logcache_v1 is a reverse proxy. 6 | 7 | It translates gRPC into RESTful JSON APIs. 8 | */ 9 | package logcache_v1 10 | 11 | import ( 12 | "io" 13 | "net/http" 14 | 15 | "github.com/golang/protobuf/proto" 16 | "github.com/grpc-ecosystem/grpc-gateway/runtime" 17 | "github.com/grpc-ecosystem/grpc-gateway/utilities" 18 | "golang.org/x/net/context" 19 | "google.golang.org/grpc" 20 | "google.golang.org/grpc/codes" 21 | "google.golang.org/grpc/grpclog" 22 | "google.golang.org/grpc/status" 23 | ) 24 | 25 | var _ codes.Code 26 | var _ io.Reader 27 | var _ status.Status 28 | var _ = runtime.String 29 | var _ = utilities.NewDoubleArray 30 | 31 | var ( 32 | filter_PromQLQuerier_InstantQuery_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} 33 | ) 34 | 35 | func request_PromQLQuerier_InstantQuery_0(ctx context.Context, marshaler runtime.Marshaler, client PromQLQuerierClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { 36 | var protoReq PromQL_InstantQueryRequest 37 | var metadata runtime.ServerMetadata 38 | 39 | if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_PromQLQuerier_InstantQuery_0); err != nil { 40 | return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) 41 | } 42 | 43 | msg, err := client.InstantQuery(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) 44 | return msg, metadata, err 45 | 46 | } 47 | 48 | var ( 49 | filter_PromQLQuerier_RangeQuery_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} 50 | ) 51 | 52 | func request_PromQLQuerier_RangeQuery_0(ctx context.Context, marshaler runtime.Marshaler, client PromQLQuerierClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { 53 | var protoReq PromQL_RangeQueryRequest 54 | var metadata runtime.ServerMetadata 55 | 56 | if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_PromQLQuerier_RangeQuery_0); err != nil { 57 | return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) 58 | } 59 | 60 | msg, err := client.RangeQuery(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) 61 | return msg, metadata, err 62 | 63 | } 64 | 65 | // RegisterPromQLQuerierHandlerFromEndpoint is same as RegisterPromQLQuerierHandler but 66 | // automatically dials to "endpoint" and closes the connection when "ctx" gets done. 67 | func RegisterPromQLQuerierHandlerFromEndpoint(ctx context.Context, mux *runtime.ServeMux, endpoint string, opts []grpc.DialOption) (err error) { 68 | conn, err := grpc.Dial(endpoint, opts...) 69 | if err != nil { 70 | return err 71 | } 72 | defer func() { 73 | if err != nil { 74 | if cerr := conn.Close(); cerr != nil { 75 | grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr) 76 | } 77 | return 78 | } 79 | go func() { 80 | <-ctx.Done() 81 | if cerr := conn.Close(); cerr != nil { 82 | grpclog.Infof("Failed to close conn to %s: %v", endpoint, cerr) 83 | } 84 | }() 85 | }() 86 | 87 | return RegisterPromQLQuerierHandler(ctx, mux, conn) 88 | } 89 | 90 | // RegisterPromQLQuerierHandler registers the http handlers for service PromQLQuerier to "mux". 91 | // The handlers forward requests to the grpc endpoint over "conn". 92 | func RegisterPromQLQuerierHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc.ClientConn) error { 93 | return RegisterPromQLQuerierHandlerClient(ctx, mux, NewPromQLQuerierClient(conn)) 94 | } 95 | 96 | // RegisterPromQLQuerierHandlerClient registers the http handlers for service PromQLQuerier 97 | // to "mux". The handlers forward requests to the grpc endpoint over the given implementation of "PromQLQuerierClient". 98 | // Note: the gRPC framework executes interceptors within the gRPC handler. If the passed in "PromQLQuerierClient" 99 | // doesn't go through the normal gRPC flow (creating a gRPC client etc.) then it will be up to the passed in 100 | // "PromQLQuerierClient" to call the correct interceptors. 101 | func RegisterPromQLQuerierHandlerClient(ctx context.Context, mux *runtime.ServeMux, client PromQLQuerierClient) error { 102 | 103 | mux.Handle("GET", pattern_PromQLQuerier_InstantQuery_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { 104 | ctx, cancel := context.WithCancel(req.Context()) 105 | defer cancel() 106 | if cn, ok := w.(http.CloseNotifier); ok { 107 | go func(done <-chan struct{}, closed <-chan bool) { 108 | select { 109 | case <-done: 110 | case <-closed: 111 | cancel() 112 | } 113 | }(ctx.Done(), cn.CloseNotify()) 114 | } 115 | inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) 116 | rctx, err := runtime.AnnotateContext(ctx, mux, req) 117 | if err != nil { 118 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 119 | return 120 | } 121 | resp, md, err := request_PromQLQuerier_InstantQuery_0(rctx, inboundMarshaler, client, req, pathParams) 122 | ctx = runtime.NewServerMetadataContext(ctx, md) 123 | if err != nil { 124 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 125 | return 126 | } 127 | 128 | forward_PromQLQuerier_InstantQuery_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) 129 | 130 | }) 131 | 132 | mux.Handle("GET", pattern_PromQLQuerier_RangeQuery_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { 133 | ctx, cancel := context.WithCancel(req.Context()) 134 | defer cancel() 135 | if cn, ok := w.(http.CloseNotifier); ok { 136 | go func(done <-chan struct{}, closed <-chan bool) { 137 | select { 138 | case <-done: 139 | case <-closed: 140 | cancel() 141 | } 142 | }(ctx.Done(), cn.CloseNotify()) 143 | } 144 | inboundMarshaler, outboundMarshaler := runtime.MarshalerForRequest(mux, req) 145 | rctx, err := runtime.AnnotateContext(ctx, mux, req) 146 | if err != nil { 147 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 148 | return 149 | } 150 | resp, md, err := request_PromQLQuerier_RangeQuery_0(rctx, inboundMarshaler, client, req, pathParams) 151 | ctx = runtime.NewServerMetadataContext(ctx, md) 152 | if err != nil { 153 | runtime.HTTPError(ctx, mux, outboundMarshaler, w, req, err) 154 | return 155 | } 156 | 157 | forward_PromQLQuerier_RangeQuery_0(ctx, mux, outboundMarshaler, w, req, resp, mux.GetForwardResponseOptions()...) 158 | 159 | }) 160 | 161 | return nil 162 | } 163 | 164 | var ( 165 | pattern_PromQLQuerier_InstantQuery_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "query"}, "")) 166 | 167 | pattern_PromQLQuerier_RangeQuery_0 = runtime.MustPattern(runtime.NewPattern(1, []int{2, 0, 2, 1, 2, 2}, []string{"api", "v1", "query_range"}, "")) 168 | ) 169 | 170 | var ( 171 | forward_PromQLQuerier_InstantQuery_0 = runtime.ForwardResponseMessage 172 | 173 | forward_PromQLQuerier_RangeQuery_0 = runtime.ForwardResponseMessage 174 | ) 175 | --------------------------------------------------------------------------------