├── README.md ├── broker-logs.go ├── broker-metrics.go ├── log.go ├── metrics.go └── trace.go /README.md: -------------------------------------------------------------------------------- 1 | # go-micro-middleware 2 | 3 | ## Metrics middleware 4 | 5 | ``` 6 | // Setting the statsd endpoint with env vars in the main function 7 | m := datadog.NewMetrics( 8 | metrics.Namespace("micro"), 9 | metrics.WithFields(metrics.Fields{ 10 | "service": "greeter", 11 | }), 12 | metrics.Collectors("dd-agent:8125"), 13 | ) 14 | defer m.Close() 15 | 16 | // Create broker 17 | b := rabbitmq.NewBroker( 18 | broker.Addrs("amqp://guest:guest@rabbit:5672"), 19 | ) 20 | if err := b.Init(); err != nil { 21 | log.Fatalf("Unexpected init error: %v", err) 22 | } 23 | if err := b.Connect(); err != nil { 24 | log.Fatalf("Unexpected connect error: %v", err) 25 | } 26 | // Wrap the broker in logging and metric middleware 27 | b = middleware.LogBrokerWrapper( 28 | middleware.MetricBrokerWrapper(b, m, time.Millisecond), 29 | ) 30 | 31 | // Setup the service with metrics wrappers for handlers and subscribers 32 | service := micro.NewService( 33 | micro.Name("greeter"), 34 | micro.Broker(b), 35 | micro.Server( 36 | server.NewServer( 37 | server.Name("greeter"), 38 | server.WrapHandler(middleware.MetricHandlerWrapper(m, time.Millisecond)), 39 | server.WrapSubscriber(middleware.MetricSubscriberWrapper(m, time.Millisecond)), 40 | ), 41 | ), 42 | ) 43 | ``` 44 | 45 | ## Logging middleware 46 | 47 | ``` 48 | service := micro.NewService( 49 | micro.Name("greeter"), 50 | micro.Server( 51 | server.NewServer( 52 | server.Name("greeter"), 53 | server.WrapHandler(middleware.LogHandlerWrapper), 54 | server.WrapSubscriber(middleware.LogSubscriberWrapper), 55 | ), 56 | ), 57 | ) 58 | ``` 59 | 60 | 61 | ## Trace middleware 62 | 63 | ``` 64 | service := micro.NewService( 65 | micro.Name("greeter"), 66 | micro.Client(client.NewClient( 67 | client.Wrap(middleware.TraceWrap), 68 | client.Wrap(middleware.LogWrap), 69 | )), 70 | ) 71 | ``` 72 | 73 | ## Datadog + Kubernetes 74 | 75 | If you are running the Datadog agents on Kubernetes in the same namespace as your micro services. You can create a kubernetes service template for the dd-agent and push metrics via the statsd port with all your kubernetes metrics. 76 | 77 | Get your dd-agent.yaml daemon set configuration from here, https://app.datadoghq.com/account/settings#agent/kubernetes 78 | 79 | You will need to configure a kube service so your go-micro services can connect to the dd-agent pods. A tradeoff of using a service is you may not connect to the same node. Host ports maybe a better option. 80 | 81 | ``` 82 | --- 83 | apiVersion: v1 84 | kind: Service 85 | metadata: 86 | name: dd-agent-service 87 | labels: 88 | app: dd-agent 89 | spec: 90 | ports: 91 | - port: 8125 92 | targetPort: 8125 93 | protocol: UDP 94 | selector: 95 | app: dd-agent 96 | ``` 97 | -------------------------------------------------------------------------------- /broker-logs.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "golang.org/x/net/context" 6 | 7 | "github.com/micro/go-micro/broker" 8 | "github.com/micro/go-micro/metadata" 9 | "github.com/micro/go-micro/server" 10 | ) 11 | 12 | type brokerLogWrapper struct { 13 | broker.Broker 14 | } 15 | 16 | func (w *brokerLogWrapper) Publish(t string, b *broker.Message, opts ...broker.PublishOption) error { 17 | log.Info("Published message") 18 | // do wrapper thing 19 | err := w.Broker.Publish(t, b, opts...) 20 | 21 | return err 22 | } 23 | 24 | func (w *brokerLogWrapper) Subscribe(t string, h broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { 25 | // wrapped handler 26 | wh := func(p broker.Publication) error { 27 | log.WithFields(log.Fields{ 28 | "topic": p.Topic(), 29 | }).Info("Received message") 30 | 31 | err := h(p) 32 | 33 | return err 34 | } 35 | 36 | return w.Broker.Subscribe(t, wh, opts...) 37 | } 38 | 39 | func LogBrokerWrapper(b broker.Broker) broker.Broker { 40 | return &brokerLogWrapper{b} 41 | } 42 | 43 | // LogSubscriberWrapper implements the server.HandlerWrapper interface 44 | func LogSubscriberWrapper(fn server.SubscriberFunc) server.SubscriberFunc { 45 | return func(ctx context.Context, msg server.Publication) error { 46 | md, _ := metadata.FromContext(ctx) 47 | log.WithFields(log.Fields{ 48 | "ctx": md, 49 | "topic": msg.Topic(), 50 | "content-type": msg.ContentType(), 51 | "event": msg.Message(), 52 | }).Infof("Received message") 53 | 54 | err := fn(ctx, msg) 55 | 56 | return err 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /broker-metrics.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "golang.org/x/net/context" 5 | "time" 6 | 7 | "github.com/micro/go-micro/broker" 8 | "github.com/micro/go-micro/server" 9 | "github.com/micro/go-os/metrics" 10 | ) 11 | 12 | var ( 13 | MetricPublish = "service.publish" 14 | MetricSubscribe = "service.subscribe" 15 | ) 16 | 17 | type brokerMetricWrapper struct { 18 | broker.Broker 19 | stats *stats 20 | } 21 | 22 | func (w *brokerMetricWrapper) Publish(t string, b *broker.Message, opts ...broker.PublishOption) error { 23 | // Begin the timer 24 | begin := time.Now() 25 | // do wrapper thing 26 | err := w.Broker.Publish(t, b, opts...) 27 | // status success or error 28 | tags := map[string]string{"status": "error"} 29 | if err == nil { 30 | tags["status"] = "success" 31 | } 32 | // Record 33 | w.stats.histogram(MetricPublish).WithFields(tags).Record(w.stats.durationToUnit(time.Since(begin))) 34 | 35 | return err 36 | } 37 | 38 | func (w *brokerMetricWrapper) Subscribe(t string, h broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { 39 | // wrapped handler 40 | wh := func(p broker.Publication) error { 41 | // Begin the timer 42 | begin := time.Now() 43 | // do wrapper thing 44 | err := h(p) 45 | // status success or error 46 | // successful request, record and return 47 | tags := map[string]string{"status": "error"} 48 | if err == nil { 49 | tags["status"] = "success" 50 | } 51 | // Record 52 | w.stats.histogram(MetricSubscribe).WithFields(tags).Record(w.stats.durationToUnit(time.Since(begin))) 53 | 54 | return err 55 | } 56 | 57 | return w.Broker.Subscribe(t, wh, opts...) 58 | } 59 | 60 | func MetricBrokerWrapper(b broker.Broker, m metrics.Metrics, u time.Duration) broker.Broker { 61 | stats := newStats(m, u) 62 | 63 | return &brokerMetricWrapper{b, stats} 64 | } 65 | 66 | // MetricSubscriberWrapper implements the server.SubscriberWrapper interface 67 | func MetricSubscriberWrapper(m metrics.Metrics, u time.Duration) server.SubscriberWrapper { 68 | return func(fn server.SubscriberFunc) server.SubscriberFunc { 69 | 70 | stats := newStats(m, u) 71 | 72 | return func(ctx context.Context, msg server.Publication) error { 73 | // Begin the timer 74 | begin := time.Now() 75 | // Run additional middleware + subscriber function 76 | err := fn(ctx, msg) 77 | // Record success or error 78 | tags := map[string]string{"status": "error"} 79 | if err == nil { 80 | tags["status"] = "success" 81 | } 82 | 83 | stats.histogram(MetricSubscribe).WithFields(tags).Record(stats.durationToUnit(time.Since(begin))) 84 | 85 | return err 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "time" 5 | 6 | log "github.com/sirupsen/logrus" 7 | 8 | "golang.org/x/net/context" 9 | 10 | "github.com/micro/go-micro/client" 11 | "github.com/micro/go-micro/metadata" 12 | "github.com/micro/go-micro/server" 13 | ) 14 | 15 | type logWrapper struct { 16 | client.Client 17 | } 18 | 19 | func (l *logWrapper) Call(ctx context.Context, req client.Request, rsp interface{}, opts ...client.CallOption) error { 20 | md, _ := metadata.FromContext(ctx) 21 | begin := time.Now() 22 | logMsg := log.WithFields(log.Fields{ 23 | "ctx": md, 24 | "service": req.Service(), 25 | "method": req.Method(), 26 | }) 27 | 28 | logMsg.Info("Calling service") 29 | 30 | err := l.Client.Call(ctx, req, rsp) 31 | 32 | if err != nil { 33 | logMsg = logMsg.WithFields(log.Fields{ 34 | "error": err, 35 | }) 36 | } 37 | // Add the duration in ms to the log message, rounding to the nearest int64 38 | logMsg.WithFields(log.Fields{ 39 | "duration": int64(float64(time.Since(begin))/float64(time.Millisecond) + 0.5), 40 | }).Info("Called service") 41 | 42 | return err 43 | } 44 | 45 | // LogClientWrapper implements client.Wrapper as logWrapper interface 46 | func LogClientWrapper(c client.Client) client.Client { 47 | return &logWrapper{c} 48 | } 49 | 50 | // LogHandlerWrapper implements the server.HandlerWrapper interface 51 | func LogHandlerWrapper(fn server.HandlerFunc) server.HandlerFunc { 52 | return func(ctx context.Context, req server.Request, rsp interface{}) error { 53 | md, _ := metadata.FromContext(ctx) 54 | log.WithFields(log.Fields{ 55 | "ctx": md, 56 | "method": req.Method(), 57 | }).Infof("Serving request") 58 | 59 | err := fn(ctx, req, rsp) 60 | 61 | return err 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /metrics.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "golang.org/x/net/context" 5 | "strconv" 6 | "strings" 7 | "time" 8 | 9 | "github.com/micro/go-micro/errors" 10 | "github.com/micro/go-micro/server" 11 | "github.com/micro/go-os/metrics" 12 | ) 13 | 14 | var ( 15 | MetricRequest = "service.request" 16 | ) 17 | 18 | type stats struct { 19 | // Metrics interface 20 | // see https://github.com/micro/go-plugins/tree/master/metrics 21 | metrics metrics.Metrics 22 | // unit of time to be recorded 23 | unit time.Duration 24 | // track multiple histograms by name 25 | histograms map[string]metrics.Histogram 26 | } 27 | 28 | func newHistogram(m metrics.Metrics, s string) metrics.Histogram { 29 | return m.Histogram(s) 30 | } 31 | 32 | func newStats(m metrics.Metrics, u time.Duration) *stats { 33 | return &stats{ 34 | metrics: m, 35 | unit: u, 36 | histograms: make(map[string]metrics.Histogram), 37 | } 38 | } 39 | 40 | func (s *stats) histogram(n string) metrics.Histogram { 41 | histogram, ok := s.histograms[n] 42 | if !ok { 43 | histogram = newHistogram(s.metrics, n) 44 | s.histograms[n] = histogram 45 | } 46 | 47 | return histogram 48 | } 49 | 50 | func (s *stats) durationToUnit(d time.Duration) int64 { 51 | return d.Nanoseconds() / int64(s.unit) 52 | } 53 | 54 | func codeFromString(s string) int64 { 55 | search := "\"code\":" 56 | i := strings.Index(s, search) 57 | if i == -1 { 58 | // search string not found in input string 59 | return 500 60 | } 61 | // parse the code 62 | code := s[i+len(search) : i+len(search)+3] 63 | // convert to int64 64 | i64, err := strconv.ParseInt(code, 10, 32) 65 | if err == nil { 66 | return i64 67 | } 68 | // default status code is 500 69 | return 500 70 | } 71 | 72 | func (s *stats) Record(req server.Request, d time.Duration, err error) { 73 | // Get the service stats 74 | // service := s.endpoint(DefaultSumOfAllRequestsName) 75 | // Get the endpoint stats 76 | // endpoint := s.endpoint(req.Method()) 77 | // convert the duration into the time unit 78 | dUnit := s.durationToUnit(d) 79 | // successful request, record and return 80 | tags := map[string]string{"status": "error"} 81 | 82 | if err == nil { 83 | tags["status"] = "success" 84 | s.histogram(MetricRequest).WithFields(tags).Record(dUnit) 85 | return 86 | } 87 | 88 | // parse the error 89 | perr, ok := err.(*errors.Error) 90 | if !ok { 91 | // Check the error message for a response code, error can be client.serverError 92 | perr = &errors.Error{Id: req.Service(), Code: int32(codeFromString(err.Error())), Detail: err.Error()} 93 | } 94 | // filter error code into bad requests and errors 95 | switch { 96 | case perr.Code == 408: 97 | tags["status"] = "dropped" 98 | case 400 <= perr.Code && perr.Code <= 499: 99 | tags["status"] = "bad" 100 | } 101 | 102 | s.histogram(MetricRequest).WithFields(tags).Record(dUnit) 103 | } 104 | 105 | // MetricHandlerWrapper implements the server.HandlerWrapper interface 106 | func MetricHandlerWrapper(m metrics.Metrics, u time.Duration) server.HandlerWrapper { 107 | return func(fn server.HandlerFunc) server.HandlerFunc { 108 | 109 | stats := newStats(m, u) 110 | 111 | return func(ctx context.Context, req server.Request, rsp interface{}) error { 112 | // Begin the timer 113 | begin := time.Now() 114 | // Run additional middleware + handler function 115 | err := fn(ctx, req, rsp) 116 | // Request is almost done, record metrics 117 | stats.Record(req, time.Since(begin), err) 118 | 119 | return err 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /trace.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | uuid "github.com/satori/go.uuid" 5 | 6 | "golang.org/x/net/context" 7 | 8 | "github.com/micro/go-micro/client" 9 | "github.com/micro/go-micro/metadata" 10 | "github.com/micro/go-micro/server" 11 | ) 12 | 13 | // trace wrapper attaches a unique trace ID - timestamp 14 | type traceWrapper struct { 15 | client.Client 16 | } 17 | 18 | func (t *traceWrapper) Call(ctx context.Context, req client.Request, rsp interface{}, opts ...client.CallOption) error { 19 | // Add trace id to context if it doesn't exist 20 | ctx = addTraceId(ctx) 21 | // continue call stack 22 | return t.Client.Call(ctx, req, rsp) 23 | } 24 | 25 | // Implements the server.HandlerWrapper 26 | func TraceHandlerWrapper(fn server.HandlerFunc) server.HandlerFunc { 27 | return func(ctx context.Context, req server.Request, rsp interface{}) error { 28 | // Add trace id to context if it doesn't exist 29 | ctx = addTraceId(ctx) 30 | // continue call stack 31 | return fn(ctx, req, rsp) 32 | } 33 | } 34 | 35 | func addTraceId(ctx context.Context) context.Context { 36 | // get metadata from context or create new 37 | md, ok := metadata.FromContext(ctx) 38 | if !ok { 39 | md = metadata.Metadata{} 40 | } 41 | 42 | if _, ok := md["X-Trace-Id"]; !ok { 43 | // Set new trace id for this call 44 | md["X-Trace-Id"] = uuid.Must(uuid.NewV4()).String() 45 | // create new context with trace id 46 | ctx = metadata.NewContext(ctx, md) 47 | } 48 | 49 | return ctx 50 | } 51 | 52 | // Implements client.Wrapper as traceWrapper 53 | func TraceClientWrapper(c client.Client) client.Client { 54 | return &traceWrapper{c} 55 | } 56 | --------------------------------------------------------------------------------