Welcome! Click some links and then view the traces for each HTTP request by following the link at the bottom of the page. 124 |
I just made 3 API calls. Check the trace below to see them!
`) 144 | case "/endpoint-A": 145 | time.Sleep(250 * time.Millisecond) 146 | io.WriteString(w, "performed an operation!") 147 | return 148 | case "/endpoint-B": 149 | time.Sleep(75 * time.Millisecond) 150 | io.WriteString(w, "performed another operation!") 151 | return 152 | case "/endpoint-C": 153 | time.Sleep(300 * time.Millisecond) 154 | io.WriteString(w, "performed yet another operation!") 155 | return 156 | } 157 | 158 | spanURL := a.appdashURL.ResolveReference(&url.URL{Path: fmt.Sprintf("/traces/%v", span.Trace)}) 159 | io.WriteString(w, fmt.Sprintf(`Three API requests have been made!
`) 130 | fmt.Fprintf(w, ``) 131 | } 132 | 133 | // Endpoint is an example API endpoint. In a real application, the backend of 134 | // your service would be contacting several external and internal API endpoints 135 | // which may be the bottleneck of your application. 136 | // 137 | // For example purposes we just sleep for 200ms before responding to simulate a 138 | // slow API endpoint as the bottleneck of your application. 139 | func Endpoint(w http.ResponseWriter, r *http.Request) { 140 | // Extract the trace from the headers and join it with a new child span. 141 | carrier := opentracing.HTTPHeadersCarrier(r.Header) 142 | spanCtx, err := opentracing.GlobalTracer().Extract(opentracing.HTTPHeaders, carrier) 143 | if err != nil { 144 | return 145 | } 146 | span := opentracing.StartSpan(r.URL.Path, opentracing.ChildOf(spanCtx)) 147 | defer span.Finish() 148 | 149 | span.SetTag("Request.Host", r.Host) 150 | span.SetTag("Request.Method", r.Method) 151 | addHeaderTags(span, r.Header) 152 | 153 | time.Sleep(200 * time.Millisecond) 154 | fmt.Fprintf(w, "Slept for 200ms!") 155 | } 156 | 157 | const headerTagPrefix = "Request.Header." 158 | 159 | // addHeaderTags adds header key:value pairs to a span as a tag with the prefix 160 | // "Request.Header.*" 161 | func addHeaderTags(span opentracing.Span, h http.Header) { 162 | for k, v := range h { 163 | span.SetTag(headerTagPrefix+k, strings.Join(v, ", ")) 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /examples/cmd/webapp/main.go: -------------------------------------------------------------------------------- 1 | // webapp: a standalone example Negroni / Gorilla based webapp. 2 | // 3 | // This example demonstrates basic usage of Appdash in a Negroni / Gorilla 4 | // based web application. The entire application is ran locally (i.e. on the 5 | // same server) -- even the Appdash web UI. 6 | package main 7 | 8 | import ( 9 | "fmt" 10 | "log" 11 | "net/http" 12 | "net/url" 13 | "time" 14 | 15 | "sourcegraph.com/sourcegraph/appdash" 16 | "sourcegraph.com/sourcegraph/appdash/httptrace" 17 | "sourcegraph.com/sourcegraph/appdash/traceapp" 18 | 19 | "github.com/gorilla/mux" 20 | "github.com/urfave/negroni" 21 | ) 22 | 23 | // We want to create HTTP clients recording to this collector inside our Home 24 | // handler below, so we use a global variable (for simplicity sake) to store 25 | // the collector in use. 26 | var collector appdash.Collector 27 | 28 | func main() { 29 | // Create a recent in-memory store, evicting data after 20s. 30 | // 31 | // The store defines where information about traces (i.e. spans and 32 | // annotations) will be stored during the lifetime of the application. This 33 | // application uses a MemoryStore store wrapped by a RecentStore with an 34 | // eviction time of 20s (i.e. all data after 20s is deleted from memory). 35 | memStore := appdash.NewMemoryStore() 36 | store := &appdash.RecentStore{ 37 | MinEvictAge: 20 * time.Second, 38 | DeleteStore: memStore, 39 | } 40 | 41 | // Start the Appdash web UI on port 8700. 42 | // 43 | // This is the actual Appdash web UI -- usable as a Go package itself, We 44 | // embed it directly into our application such that visiting the web server 45 | // on HTTP port 8700 will bring us to the web UI, displaying information 46 | // about this specific web-server (another alternative would be to connect 47 | // to a centralized Appdash collection server). 48 | url, err := url.Parse("http://localhost:8700") 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | tapp, err := traceapp.New(nil, url) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | tapp.Store = store 57 | tapp.Queryer = memStore 58 | log.Println("Appdash web UI running on HTTP :8700") 59 | go func() { 60 | log.Fatal(http.ListenAndServe(":8700", tapp)) 61 | }() 62 | 63 | // We will use a local collector (as we are running the Appdash web UI 64 | // embedded within our app). 65 | // 66 | // A collector is responsible for collecting the information about traces 67 | // (i.e. spans and annotations) and placing them into a store. In this app 68 | // we use a local collector (we could also use a remote collector, sending 69 | // the information to a remote Appdash collection server). 70 | collector = appdash.NewLocalCollector(store) 71 | 72 | // Create the appdash/httptrace middleware. 73 | // 74 | // Here we initialize the appdash/httptrace middleware. It is a Negroni 75 | // compliant HTTP middleware that will generate HTTP events for Appdash to 76 | // display. We could also instruct Appdash with events manually, if we 77 | // wanted to. 78 | tracemw := httptrace.Middleware(collector, &httptrace.MiddlewareConfig{ 79 | RouteName: func(r *http.Request) string { return r.URL.Path }, 80 | }) 81 | 82 | // Setup our router (for information, see the gorilla/mux docs): 83 | router := mux.NewRouter() 84 | router.HandleFunc("/", Home) 85 | router.HandleFunc("/endpoint", Endpoint) 86 | 87 | // Setup Negroni for our app (for information, see the negroni docs): 88 | n := negroni.Classic() 89 | n.Use(negroni.HandlerFunc(tracemw)) // Register appdash's HTTP middleware. 90 | n.UseHandler(router) 91 | n.Run(":8699") 92 | } 93 | 94 | // Home is the homepage handler for our app. 95 | func Home(w http.ResponseWriter, r *http.Request) { 96 | // Grab the span from the request. We do this so that we can grab the 97 | // span.Trace ID and link directly to the trace on the web-page itself! 98 | span := httptrace.SpanID(r) 99 | 100 | // We're going to make some API requests, so we create a HTTP client using 101 | // a appdash/httptrace transport here. The transport will inform Appdash of 102 | // the HTTP events occurring. 103 | httpClient := &http.Client{ 104 | Transport: &httptrace.Transport{ 105 | Recorder: appdash.NewRecorder(span, collector), 106 | SetName: true, 107 | }, 108 | } 109 | 110 | // Make three API requests using our HTTP client. 111 | for i := 0; i < 3; i++ { 112 | resp, err := httpClient.Get("http://localhost:8699/endpoint") 113 | if err != nil { 114 | log.Println("/endpoint:", err) 115 | continue 116 | } 117 | resp.Body.Close() 118 | } 119 | 120 | // Render the page. 121 | fmt.Fprintf(w, `Three API requests have been made!
`) 122 | fmt.Fprintf(w, ``, span.Trace, span.Trace) 123 | } 124 | 125 | // Endpoint is an example API endpoint. In a real application, the backend of 126 | // your service would be contacting several external and internal API endpoints 127 | // which may be the bottleneck of your application. 128 | // 129 | // For example purposes we just sleep for 200ms before responding to simulate a 130 | // slow API endpoint as the bottleneck of your application. 131 | func Endpoint(w http.ResponseWriter, r *http.Request) { 132 | time.Sleep(200 * time.Millisecond) 133 | fmt.Fprintf(w, "Slept for 200ms!") 134 | } 135 | -------------------------------------------------------------------------------- /httptrace/client.go: -------------------------------------------------------------------------------- 1 | package httptrace 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | "sync" 7 | "time" 8 | 9 | "sourcegraph.com/sourcegraph/appdash" 10 | ) 11 | 12 | var ( 13 | // RedactedHeaders is a slice of header names whose values should be 14 | // entirely redacted from logs. 15 | RedactedHeaders = []string{"Authorization"} 16 | ) 17 | 18 | func init() { appdash.RegisterEvent(ClientEvent{}) } 19 | 20 | // NewClientEvent returns an event which records various aspects of an 21 | // HTTP request. The returned value is incomplete, and should have 22 | // the response status, size, and the ClientSend/ClientRecv times set 23 | // before being logged. 24 | func NewClientEvent(r *http.Request) *ClientEvent { 25 | return &ClientEvent{Request: requestInfo(r)} 26 | } 27 | 28 | // RequestInfo describes an HTTP request. 29 | type RequestInfo struct { 30 | Method string 31 | URI string 32 | Proto string 33 | Headers map[string]string 34 | Host string 35 | RemoteAddr string 36 | ContentLength int64 37 | } 38 | 39 | func requestInfo(r *http.Request) RequestInfo { 40 | return RequestInfo{ 41 | Method: r.Method, 42 | URI: r.URL.RequestURI(), 43 | Proto: r.Proto, 44 | Headers: redactHeaders(r.Header, r.Trailer), 45 | Host: r.Host, 46 | RemoteAddr: r.RemoteAddr, 47 | ContentLength: r.ContentLength, 48 | } 49 | } 50 | 51 | // ClientEvent records an HTTP client request event. 52 | type ClientEvent struct { 53 | Request RequestInfo `trace:"Client.Request"` 54 | Response ResponseInfo `trace:"Client.Response"` 55 | ClientSend time.Time `trace:"Client.Send"` 56 | ClientRecv time.Time `trace:"Client.Recv"` 57 | } 58 | 59 | // Schema returns the constant "HTTPClient". 60 | func (ClientEvent) Schema() string { return "HTTPClient" } 61 | 62 | // Important implements the appdash ImportantEvent. 63 | func (ClientEvent) Important() []string { 64 | return []string{ 65 | "Client.Request.Headers.If-Modified-Since", 66 | "Client.Request.Headers.If-None-Match", 67 | "Client.Response.StatusCode", 68 | } 69 | } 70 | 71 | // Start implements the appdash TimespanEvent interface. 72 | func (e ClientEvent) Start() time.Time { return e.ClientSend } 73 | 74 | // End implements the appdash TimespanEvent interface. 75 | func (e ClientEvent) End() time.Time { return e.ClientRecv } 76 | 77 | var ( 78 | redacted = []string{"REDACTED"} 79 | ) 80 | 81 | func redactHeaders(header, trailer http.Header) map[string]string { 82 | h := make(http.Header, len(header)+len(trailer)) 83 | for k, v := range header { 84 | if isRedacted(k) { 85 | h[k] = redacted 86 | } else { 87 | h[k] = v 88 | } 89 | } 90 | for k, v := range trailer { 91 | if isRedacted(k) { 92 | h[k] = redacted 93 | } else { 94 | h[k] = append(h[k], v...) 95 | } 96 | } 97 | m := make(map[string]string, len(h)) 98 | for k, v := range h { 99 | m[http.CanonicalHeaderKey(k)] = strings.Join(v, ",") 100 | } 101 | return m 102 | } 103 | 104 | func isRedacted(name string) bool { 105 | for _, v := range RedactedHeaders { 106 | if strings.EqualFold(name, v) { 107 | return true 108 | } 109 | } 110 | return false 111 | } 112 | 113 | // Transport is an HTTP transport that adds appdash span ID headers 114 | // to requests so that downstream operations are associated with the 115 | // same trace. 116 | type Transport struct { 117 | // Recorder is the current span's recorder. A new child Recorder 118 | // (with a new child SpanID) is created for each HTTP roundtrip. 119 | *appdash.Recorder 120 | 121 | // Transport is the underlying HTTP transport to use when making 122 | // requests. It will default to http.DefaultTransport if nil. 123 | Transport http.RoundTripper 124 | 125 | SetName bool 126 | 127 | // requests keeps clone request 128 | reqMu sync.RWMutex 129 | requests map[*http.Request]*http.Request 130 | } 131 | 132 | // RoundTrip implements the RoundTripper interface. 133 | func (t *Transport) RoundTrip(original *http.Request) (*http.Response, error) { 134 | // To set extra querystring params, we must make a copy of the Request so 135 | // that we don't modify the Request we were given. This is required by the 136 | // specification of http.RoundTripper. 137 | req := cloneRequest(original) 138 | t.setCloneRequest(original, req) 139 | defer t.setCloneRequest(original, nil) 140 | 141 | child := t.Recorder.Child() 142 | if t.SetName { 143 | child.Name("Request " + req.URL.Host) 144 | } 145 | 146 | // New child span is created and set as HTTP header instead of using `child` 147 | // in order to have a single span recording operation per httptrace event 148 | // (HTTPClient or HTTPServer). 149 | span := appdash.NewSpanID(t.Recorder.SpanID) 150 | 151 | SetSpanIDHeader(req.Header, span) 152 | 153 | e := NewClientEvent(req) 154 | e.ClientSend = time.Now() 155 | 156 | // Make the HTTP request. 157 | transport := t.getTransport() 158 | resp, err := transport.RoundTrip(req) 159 | 160 | e.ClientRecv = time.Now() 161 | if err == nil { 162 | e.Response = responseInfo(resp) 163 | } else { 164 | e.Response.StatusCode = -1 165 | } 166 | child.Event(e) 167 | child.Finish() 168 | return resp, err 169 | } 170 | 171 | // cloneRequest returns a clone of the provided *http.Request. The clone is a 172 | // shallow copy of the struct and its Header map. 173 | func cloneRequest(r *http.Request) *http.Request { 174 | // shallow copy of the struct 175 | r2 := new(http.Request) 176 | *r2 = *r 177 | // deep copy of the Header 178 | r2.Header = make(http.Header) 179 | for k, s := range r.Header { 180 | r2.Header[k] = s 181 | } 182 | return r2 183 | } 184 | 185 | // setCloneRequest keeps track of the cloned based on the original request so 186 | // that it can be canceled in the future in CancelRequest. 187 | func (t *Transport) setCloneRequest(original *http.Request, clone *http.Request) { 188 | t.reqMu.Lock() 189 | defer t.reqMu.Unlock() 190 | if t.requests == nil { 191 | t.requests = make(map[*http.Request]*http.Request) 192 | } 193 | 194 | if clone != nil { 195 | t.requests[original] = clone 196 | } else { 197 | delete(t.requests, original) 198 | } 199 | } 200 | 201 | // getTransport returns custom transport or DefaultTransport 202 | func (t *Transport) getTransport() http.RoundTripper { 203 | if t.Transport != nil { 204 | return t.Transport 205 | } 206 | return http.DefaultTransport 207 | } 208 | 209 | // CancelRequest cancels an in-flight request by closing its connection. 210 | func (t *Transport) CancelRequest(req *http.Request) { 211 | type canceler interface { 212 | CancelRequest(*http.Request) 213 | } 214 | 215 | t.reqMu.RLock() 216 | newReq, ok := t.requests[req] 217 | t.reqMu.RUnlock() 218 | if !ok { 219 | return 220 | } 221 | 222 | transport := t.getTransport() 223 | if ts, ok := transport.(canceler); ok { 224 | ts.CancelRequest(newReq) 225 | } 226 | } 227 | -------------------------------------------------------------------------------- /httptrace/client_test.go: -------------------------------------------------------------------------------- 1 | package httptrace 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "reflect" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "sourcegraph.com/sourcegraph/appdash" 12 | ) 13 | 14 | var _ appdash.Event = ClientEvent{} 15 | 16 | func TestNewClientEvent(t *testing.T) { 17 | r := &http.Request{ 18 | Host: "example.com", 19 | Method: "GET", 20 | URL: &url.URL{Path: "/foo"}, 21 | Proto: "HTTP/1.1", 22 | RemoteAddr: "127.0.0.1", 23 | ContentLength: 0, 24 | Header: http.Header{ 25 | "Authorization": []string{"Basic seeecret"}, 26 | "Accept": []string{"application/json"}, 27 | }, 28 | Trailer: http.Header{ 29 | "Authorization": []string{"Basic seeecret"}, 30 | "Connection": []string{"close"}, 31 | }, 32 | } 33 | e := NewClientEvent(r) 34 | e.Response.StatusCode = 200 35 | if e.Schema() != "HTTPClient" { 36 | t.Errorf("unexpected schema: %v", e.Schema()) 37 | } 38 | anns, err := appdash.MarshalEvent(e) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | expected := map[string]string{ 43 | "_schema:HTTPClient": "", 44 | "Client.Request.Headers.Connection": "close", 45 | "Client.Request.Headers.Accept": "application/json", 46 | "Client.Request.Headers.Authorization": "REDACTED", 47 | "Client.Request.Proto": "HTTP/1.1", 48 | "Client.Request.RemoteAddr": "127.0.0.1", 49 | "Client.Request.Host": "example.com", 50 | "Client.Request.ContentLength": "0", 51 | "Client.Request.Method": "GET", 52 | "Client.Request.URI": "/foo", 53 | "Client.Response.StatusCode": "200", 54 | "Client.Response.ContentLength": "0", 55 | "Client.Send": "0001-01-01T00:00:00Z", 56 | "Client.Recv": "0001-01-01T00:00:00Z", 57 | } 58 | if !reflect.DeepEqual(anns.StringMap(), expected) { 59 | t.Errorf("got %#v, want %#v", anns.StringMap(), expected) 60 | } 61 | } 62 | 63 | func TestTransport(t *testing.T) { 64 | ms := appdash.NewMemoryStore() 65 | rec := appdash.NewRecorder(appdash.SpanID{1, 2, 3}, appdash.NewLocalCollector(ms)) 66 | 67 | req, _ := http.NewRequest("GET", "http://example.com/foo", nil) 68 | req.Header.Set("X-Req-Header", "a") 69 | mt := &mockTransport{ 70 | resp: &http.Response{ 71 | StatusCode: 200, 72 | ContentLength: 123, 73 | Header: http.Header{"X-Resp-Header": []string{"b"}}, 74 | }, 75 | } 76 | transport := &Transport{ 77 | Recorder: rec, 78 | Transport: mt, 79 | } 80 | 81 | _, err := transport.RoundTrip(req) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | 86 | spanID, err := appdash.ParseSpanID(mt.req.Header.Get("Span-ID")) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | if want := (appdash.SpanID{1, spanID.Span, 2}); *spanID != want { 91 | t.Errorf("got Span-ID in header %+v, want %+v", *spanID, want) 92 | } 93 | 94 | trace, err := ms.Trace(1) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | 99 | var e ClientEvent 100 | if err := appdash.UnmarshalEvent(trace.Span.Annotations, &e); err != nil { 101 | t.Fatal(err) 102 | } 103 | 104 | wantEvent := ClientEvent{ 105 | Request: RequestInfo{ 106 | Method: "GET", 107 | Proto: "HTTP/1.1", 108 | URI: "/foo", 109 | Host: "example.com", 110 | Headers: map[string]string{"X-Req-Header": "a"}, 111 | }, 112 | Response: ResponseInfo{ 113 | StatusCode: 200, 114 | ContentLength: 123, 115 | Headers: map[string]string{"X-Resp-Header": "b"}, 116 | }, 117 | } 118 | delete(e.Request.Headers, "Span-Id") 119 | e.ClientSend = time.Time{} 120 | e.ClientRecv = time.Time{} 121 | if !reflect.DeepEqual(e, wantEvent) { 122 | t.Errorf("got ClientEvent %+v, want %+v", e, wantEvent) 123 | } 124 | } 125 | 126 | func TestCancelRequest(t *testing.T) { 127 | ms := appdash.NewMemoryStore() 128 | rec := appdash.NewRecorder(appdash.SpanID{1, 2, 3}, appdash.NewLocalCollector(ms)) 129 | req, _ := http.NewRequest("GET", "http://example.com/foo", nil) 130 | transport := &Transport{ 131 | Recorder: rec, 132 | } 133 | client := &http.Client{ 134 | Timeout: 1 * time.Millisecond, 135 | Transport: transport, 136 | } 137 | 138 | resp, err := client.Do(req) 139 | 140 | expected := "Get http://example.com/foo: net/http: request canceled while waiting for connection" 141 | if err == nil || !strings.HasPrefix(err.Error(), expected) { 142 | t.Errorf("got %#v, want %s", err, expected) 143 | } 144 | if resp != nil { 145 | t.Errorf("got http.Response %#v, want nil", resp) 146 | } 147 | } 148 | 149 | type mockTransport struct { 150 | req *http.Request 151 | resp *http.Response 152 | } 153 | 154 | func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { 155 | t.req = req 156 | return t.resp, nil 157 | } 158 | -------------------------------------------------------------------------------- /httptrace/doc.go: -------------------------------------------------------------------------------- 1 | // Package httptrace implements support for tracing HTTP applications. 2 | // 3 | // This package exposes a HTTP middleware usable for generating traces for 4 | // measuring the performance and debugging distributed HTTP applications using 5 | // appdash. 6 | // 7 | // The middleware is Negroni-compliant, and can thus be used with Negroni 8 | // easily or with a pure net/http (i.e. stdlib-only) application with ease. 9 | // 10 | // Trace Collection Server 11 | // 12 | // Trace collection occurs anywhere (on this HTTP server, remotely on another, 13 | // etc). It is independent from this package. One approach is to run a local 14 | // collection server (on the HTTP server itself) that keeps the last 20s of 15 | // appdash events in-memory, like so: 16 | // 17 | // // Create a recent in-memory store, evicting data after 20s. 18 | // store := &appdash.RecentStore{ 19 | // MinEvictAge: 20 * time.Second, 20 | // DeleteStore: appdash.NewMemoryStore(), 21 | // } 22 | // 23 | // // Listen on port 7701. 24 | // ln, err := net.Listen("tcp", ":7701") 25 | // if err != nil { 26 | // // handle error 27 | // } 28 | // 29 | // // Create an appdash server, listen and serve in a separate goroutine. 30 | // cs := appdash.NewServer(ln, appdash.NewLocalCollector(store)) 31 | // go cs.Start() 32 | // 33 | // Note that the above server exposes the traces in plain-text (i.e. insecurely) 34 | // over the given port. Allowing access to that port outside your network allows 35 | // others to potentially see API keys and other information about HTTP requests 36 | // going through your network. 37 | // 38 | // If you intend to make appdash available outside your network, use a secure 39 | // appdash server instead (see the appdash package for details). 40 | // 41 | // Server Init 42 | // 43 | // Whether you plan to use Negroni, or just net/http, you'll first need to make 44 | // a collector. For example, by connecting to the appdash server that we made 45 | // earlier: 46 | // 47 | // // Connect to a remote collection server. 48 | // collector := appdash.NewRemoteCollector(":7701") 49 | // 50 | // And a basic middleware: 51 | // 52 | // // Create a httptrace middleware. 53 | // tracemw := httptrace.Middleware(collector, &httptrace.MiddlewareConfig{}) 54 | // 55 | // With Negroni 56 | // 57 | // Negroni is a idiomatic web middleware package for Go, and the middleware 58 | // exposed by this package is fully compliant with it's requirements -- which 59 | // makes using it a breeze: 60 | // 61 | // // Create app handler: 62 | // mux := http.NewServeMux() 63 | // mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { 64 | // fmt.Fprintf(w, "Hello world!") 65 | // }) 66 | // 67 | // // Setup Negroni for our app: 68 | // n := negroni.Classic() 69 | // n.Use(negroni.HandlerFunc(tracemw)) // Register appdash's HTTP middleware. 70 | // n.UseHandler(mux) 71 | // n.Run(":3000") 72 | // 73 | // With The http Package 74 | // 75 | // The HTTP middleware can also be used without Negroni, although slightly more 76 | // verbose. Say for example that you have a net/http handler for your app: 77 | // 78 | // func appHandler(w http.ResponseWriter, r *http.Request) { 79 | // fmt.Fprintf(w, "Hello World!") 80 | // } 81 | // 82 | // Simply create a middleware and pass each HTTP request through it, continuing 83 | // with your application handler: 84 | // 85 | // // Let all requests pass through the middleware, and then go on to let our 86 | // // app handler serve the request. 87 | // http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 88 | // tracemw(w, r, appHandler) 89 | // }) 90 | // 91 | // Other details such as outbound client requests, displaying the trace ID in 92 | // the webpage e.g. to let users give you their trace ID for troubleshooting, 93 | // and much more are covered in the example application provided at 94 | // cmd/appdash/example_app.go. 95 | package httptrace 96 | -------------------------------------------------------------------------------- /httptrace/header.go: -------------------------------------------------------------------------------- 1 | package httptrace 2 | 3 | import ( 4 | "net/http" 5 | 6 | "sourcegraph.com/sourcegraph/appdash" 7 | ) 8 | 9 | const ( 10 | // HeaderSpanID is the name of the HTTP header by which the trace 11 | // and span IDs are passed along. 12 | HeaderSpanID = "Span-ID" 13 | 14 | // HeaderParentSpanID is the name of the HTTP header by which the 15 | // parent trace and span IDs are passed along. It should only be 16 | // set by clients that are incapable of creating their own span 17 | // IDs (e.g., JavaScript API clients in a web page, which can 18 | // easily pass along an existing parent span ID but not create a 19 | // new child span ID). 20 | HeaderParentSpanID = "Parent-Span-ID" 21 | ) 22 | 23 | // SetSpanIDHeader sets the Span-ID header. 24 | func SetSpanIDHeader(h http.Header, e appdash.SpanID) { 25 | h.Set(HeaderSpanID, e.String()) 26 | } 27 | 28 | // GetSpanID returns the SpanID for the current request, based on the 29 | // values in the HTTP headers. If a Span-ID header is provided, it is 30 | // parsed; if a Parent-Span-ID header is provided, a new child span is 31 | // created and it is returned; otherwise a new root SpanID is created. 32 | func GetSpanID(h http.Header) (*appdash.SpanID, error) { 33 | spanID, _, err := getSpanID(h) 34 | return spanID, err 35 | } 36 | 37 | func getSpanID(h http.Header) (spanID *appdash.SpanID, fromHeader string, err error) { 38 | // Check for Span-ID. 39 | fromHeader = HeaderSpanID 40 | spanID, err = getSpanIDHeader(h, HeaderSpanID) 41 | if err != nil { 42 | return nil, fromHeader, err 43 | } 44 | 45 | // Check for Parent-Span-ID. 46 | if spanID == nil { 47 | fromHeader = HeaderParentSpanID 48 | spanID, err = getSpanIDHeader(h, HeaderParentSpanID) 49 | if err != nil { 50 | return nil, fromHeader, err 51 | } 52 | if spanID != nil { 53 | newSpanID := appdash.NewSpanID(*spanID) 54 | spanID = &newSpanID 55 | } 56 | } 57 | 58 | // Create a new root span ID. 59 | if spanID == nil { 60 | fromHeader = "" 61 | newSpanID := appdash.NewRootSpanID() 62 | spanID = &newSpanID 63 | } 64 | return spanID, fromHeader, nil 65 | } 66 | 67 | // getSpanIDHeader returns the SpanID in the header (specified by 68 | // key), nil if no such header was provided, or an error if the value 69 | // was unparseable. 70 | func getSpanIDHeader(h http.Header, key string) (*appdash.SpanID, error) { 71 | s := h.Get(key) 72 | if s == "" { 73 | return nil, nil 74 | } 75 | return appdash.ParseSpanID(s) 76 | } 77 | -------------------------------------------------------------------------------- /httptrace/header_test.go: -------------------------------------------------------------------------------- 1 | package httptrace 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "sourcegraph.com/sourcegraph/appdash" 8 | ) 9 | 10 | func TestSetSpanIDHeader(t *testing.T) { 11 | h := make(http.Header) 12 | SetSpanIDHeader(h, appdash.SpanID{ 13 | Trace: 100, 14 | Span: 150, 15 | }) 16 | actual := h.Get("Span-ID") 17 | expected := "0000000000000064/0000000000000096" 18 | if actual != expected { 19 | t.Errorf("got %#v, want %#v", actual, expected) 20 | } 21 | } 22 | 23 | func TestGetSpanID_hasSpanID(t *testing.T) { 24 | h := make(http.Header) 25 | h.Add("Span-ID", "0000000000000064/0000000000000096") 26 | id, err := GetSpanID(h) 27 | if err != nil { 28 | t.Fatalf("unexpected error: %v", err) 29 | } 30 | if id.Trace != 100 || id.Span != 150 { 31 | t.Errorf("unexpected span ID: %+v", id) 32 | } 33 | } 34 | 35 | func TestGetSpanID_hasParentSpanID(t *testing.T) { 36 | h := make(http.Header) 37 | h.Add("Parent-Span-ID", "0000000000000064/0000000000000096") 38 | id, err := GetSpanID(h) 39 | if err != nil { 40 | t.Fatalf("unexpected error: %v", err) 41 | } 42 | if id.Trace != 100 || id.Parent != 150 { 43 | t.Errorf("unexpected span ID: %+v", id) 44 | } 45 | if id.Span == 150 { 46 | t.Errorf("unexpected span ID: %+v", id) 47 | } 48 | } 49 | 50 | func TestGetSpanID_hasNoSpanID(t *testing.T) { 51 | h := make(http.Header) 52 | id, err := GetSpanID(h) 53 | if err != nil { 54 | t.Fatalf("unexpected error: %v", err) 55 | } 56 | if id == nil { 57 | t.Fatal("got nil ID, expected a new root span ID") 58 | } 59 | if id.Trace == 0 || id.Span == 0 { 60 | t.Errorf("unexpected span ID: %+v", id) 61 | } 62 | if id.Parent != 0 { 63 | t.Errorf("unexpected span ID nonzero parent: %+v", id) 64 | } 65 | } 66 | 67 | func TestGetSpanID_hasSpanIDAndParentSpanID(t *testing.T) { 68 | // This should never happen, but just make sure we don't fail (or 69 | // accidentally change the behavior). 70 | h := make(http.Header) 71 | h.Add("Span-ID", "0000000000000064/0000000000000096") 72 | h.Add("Parent-Span-ID", "0000000000000032/0000000000000048") 73 | id, err := GetSpanID(h) 74 | if err != nil { 75 | t.Fatalf("unexpected error: %v", err) 76 | } 77 | if id.Trace != 100 || id.Span != 150 { 78 | t.Errorf("unexpected span ID: %+v", id) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /httptrace/server.go: -------------------------------------------------------------------------------- 1 | package httptrace 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "net/http" 7 | "time" 8 | 9 | "sourcegraph.com/sourcegraph/appdash" 10 | ) 11 | 12 | func init() { appdash.RegisterEvent(ServerEvent{}) } 13 | 14 | // NewServerEvent returns an event which records various aspects of an 15 | // HTTP response. It takes an HTTP request, not response, as input 16 | // because the information it records is derived from the request, and 17 | // HTTP handlers don't have access to the response struct (only 18 | // http.ResponseWriter, which requires wrapping or buffering to 19 | // introspect). 20 | // 21 | // The returned value is incomplete and should have its Response and 22 | // ServerRecv/ServerSend values set before being logged. 23 | func NewServerEvent(r *http.Request) *ServerEvent { 24 | return &ServerEvent{Request: requestInfo(r)} 25 | } 26 | 27 | // ResponseInfo describes an HTTP response. 28 | type ResponseInfo struct { 29 | Headers map[string]string 30 | ContentLength int64 31 | StatusCode int 32 | } 33 | 34 | func responseInfo(r *http.Response) ResponseInfo { 35 | return ResponseInfo{ 36 | Headers: redactHeaders(r.Header, r.Trailer), 37 | ContentLength: r.ContentLength, 38 | StatusCode: r.StatusCode, 39 | } 40 | } 41 | 42 | // ServerEvent records an HTTP server request handling event. 43 | type ServerEvent struct { 44 | Request RequestInfo `trace:"Server.Request"` 45 | Response ResponseInfo `trace:"Server.Response"` 46 | Route string `trace:"Server.Route"` 47 | User string `trace:"Server.User"` 48 | ServerRecv time.Time `trace:"Server.Recv"` 49 | ServerSend time.Time `trace:"Server.Send"` 50 | } 51 | 52 | // Schema returns the constant "HTTPServer". 53 | func (ServerEvent) Schema() string { return "HTTPServer" } 54 | 55 | // Important implements the appdash ImportantEvent. 56 | func (ServerEvent) Important() []string { 57 | return []string{"Server.Response.StatusCode"} 58 | } 59 | 60 | // Start implements the appdash TimespanEvent interface. 61 | func (e ServerEvent) Start() time.Time { return e.ServerRecv } 62 | 63 | // End implements the appdash TimespanEvent interface. 64 | func (e ServerEvent) End() time.Time { return e.ServerSend } 65 | 66 | // Middleware creates a new http.Handler middleware 67 | // (negroni-compliant) that records incoming HTTP requests to the 68 | // collector c as "HTTPServer"-schema events. 69 | func Middleware(c appdash.Collector, conf *MiddlewareConfig) func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 70 | return func(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) { 71 | spanID, spanFromHeader, err := getSpanID(r.Header) 72 | if err != nil { 73 | log.Printf("Warning: invalid %s header: %s. (Continuing with request handling.)", spanFromHeader, err) 74 | } 75 | usingProvidedSpanID := (spanFromHeader == HeaderSpanID) 76 | 77 | if conf.SetContextSpan != nil { 78 | conf.SetContextSpan(r, *spanID) 79 | } else { 80 | ctx := context.WithValue(r.Context(), contextKeySpanID, *spanID) 81 | r = r.WithContext(ctx) 82 | } 83 | 84 | e := NewServerEvent(r) 85 | e.ServerRecv = time.Now() 86 | 87 | rr := &responseInfoRecorder{ResponseWriter: rw} 88 | next(rr, r) 89 | SetSpanIDHeader(rr.Header(), *spanID) 90 | 91 | if !usingProvidedSpanID { 92 | e.Request = requestInfo(r) 93 | } 94 | if conf.RouteName != nil { 95 | e.Route = conf.RouteName(r) 96 | } 97 | if conf.CurrentUser != nil { 98 | e.User = conf.CurrentUser(r) 99 | } 100 | e.Response = responseInfo(rr.partialResponse()) 101 | e.ServerSend = time.Now() 102 | 103 | rec := appdash.NewRecorder(*spanID, c) 104 | if e.Route != "" { 105 | rec.Name("Serve " + e.Route) 106 | } else { 107 | rec.Name("Serve " + r.URL.Host + r.URL.Path) 108 | } 109 | rec.Event(e) 110 | rec.Finish() 111 | } 112 | } 113 | 114 | // MiddlewareConfig configures the HTTP tracing middleware. 115 | type MiddlewareConfig struct { 116 | // RouteName, if non-nil, is called to get the current route's 117 | // name. This name is used as the span's name. 118 | RouteName func(*http.Request) string 119 | 120 | // CurrentUser, if non-nil, is called to get the current user ID 121 | // (which may be a login or a numeric ID). 122 | CurrentUser func(*http.Request) string 123 | 124 | // SetContextSpan is deprecated. If non-nil, is called to set the span 125 | // (which is either taken from the client request header or created anew) 126 | // in the HTTP request context, so it may be used by other parts of the 127 | // handling process. 128 | SetContextSpan func(*http.Request, appdash.SpanID) 129 | } 130 | 131 | type contextKey string 132 | 133 | var contextKeySpanID = contextKey("spanID") 134 | 135 | // SpanID returns the SpanID set for r by httptrace middleware. It requires 136 | // that MiddlewareConfig.SetContextSpan is nil. If not, it panics. 137 | func SpanID(r *http.Request) appdash.SpanID { 138 | return r.Context().Value(contextKeySpanID).(appdash.SpanID) 139 | } 140 | 141 | // SpanIDFromContext returns the SpanID set for the request context ctx by 142 | // httptrace middleware. It requires that MiddlewareConfig.SetContextSpan is 143 | // nil. If not, ok is false. 144 | func SpanIDFromContext(ctx context.Context) (spanID appdash.SpanID, ok bool) { 145 | spanID, ok = ctx.Value(contextKeySpanID).(appdash.SpanID) 146 | return 147 | } 148 | 149 | // responseInfoRecorder is an http.ResponseWriter that records a 150 | // response's HTTP status code and body length and forwards all 151 | // operations onto an underlying http.ResponseWriter, without 152 | // buffering the response body. 153 | type responseInfoRecorder struct { 154 | statusCode int // HTTP response status code 155 | ContentLength int64 // number of bytes written using the Write method 156 | 157 | http.ResponseWriter // underlying ResponseWriter to pass-thru to 158 | } 159 | 160 | // Write always succeeds and writes to r.Body, if not nil. 161 | func (r *responseInfoRecorder) Write(b []byte) (int, error) { 162 | r.ContentLength += int64(len(b)) 163 | if r.statusCode == 0 { 164 | r.statusCode = http.StatusOK 165 | } 166 | return r.ResponseWriter.Write(b) 167 | } 168 | 169 | func (r *responseInfoRecorder) StatusCode() int { 170 | if r.statusCode == 0 { 171 | return http.StatusOK 172 | } 173 | return r.statusCode 174 | } 175 | 176 | // WriteHeader sets r.Code. 177 | func (r *responseInfoRecorder) WriteHeader(code int) { 178 | r.statusCode = code 179 | r.ResponseWriter.WriteHeader(code) 180 | } 181 | 182 | // partialResponse constructs a partial response object based on the 183 | // information it is able to determine about the response. 184 | func (r *responseInfoRecorder) partialResponse() *http.Response { 185 | return &http.Response{ 186 | StatusCode: r.StatusCode(), 187 | ContentLength: r.ContentLength, 188 | Header: r.Header(), 189 | } 190 | } 191 | 192 | // Flush implements the http.Flusher interface and sends any buffered 193 | // data to the client, if the underlying http.ResponseWriter itself 194 | // implements http.Flusher. 195 | func (r *responseInfoRecorder) Flush() { 196 | if f, ok := r.ResponseWriter.(http.Flusher); ok { 197 | f.Flush() 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /id.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "crypto/aes" 5 | "crypto/cipher" 6 | "crypto/rand" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | "strconv" 11 | "sync" 12 | "unsafe" 13 | ) 14 | 15 | // An ID is a unique, uniformly distributed 64-bit ID. 16 | type ID uint64 17 | 18 | // String returns the ID as a hex string. 19 | func (id ID) String() string { 20 | return fmt.Sprintf("%016x", uint64(id)) 21 | } 22 | 23 | // MarshalJSON encodes the ID as a hex string. 24 | func (id ID) MarshalJSON() ([]byte, error) { 25 | return json.Marshal(id.String()) 26 | } 27 | 28 | // UnmarshalJSON decodes the given data as either a hexadecimal string or JSON 29 | // integer. 30 | func (id *ID) UnmarshalJSON(data []byte) error { 31 | i, err := parseJSONString(data) 32 | if err == nil { 33 | *id = i 34 | return nil 35 | } 36 | i, err = parseJSONInt(data) 37 | if err == nil { 38 | *id = i 39 | return nil 40 | } 41 | return fmt.Errorf("%s is not a valid ID", data) 42 | } 43 | 44 | // ParseID parses the given string as a hexadecimal string. 45 | func ParseID(s string) (ID, error) { 46 | i, err := strconv.ParseUint(s, 16, 64) 47 | if err != nil { 48 | return 0, err 49 | } 50 | return ID(i), nil 51 | } 52 | 53 | // generateID returns a randomly-generated 64-bit ID. This function is 54 | // thread-safe. IDs are produced by consuming an AES-CTR-128 keystream in 55 | // 64-bit chunks. The AES key is randomly generated on initialization, as is the 56 | // counter's initial state. On machines with AES-NI support, ID generation takes 57 | // ~30ns and generates no garbage. 58 | func generateID() ID { 59 | m.Lock() 60 | if n == aes.BlockSize { 61 | c.Encrypt(b, ctr) 62 | for i := aes.BlockSize - 1; i >= 0; i-- { // increment ctr 63 | ctr[i]++ 64 | if ctr[i] != 0 { 65 | break 66 | } 67 | } 68 | n = 0 69 | } 70 | id := *(*ID)(unsafe.Pointer(&b[n])) // zero-copy b/c we're arch-neutral 71 | n += idSize 72 | m.Unlock() 73 | return id 74 | } 75 | 76 | const ( 77 | idSize = aes.BlockSize / 2 // 64 bits 78 | keySize = aes.BlockSize // 128 bits 79 | ) 80 | 81 | var ( 82 | ctr []byte 83 | n int 84 | b []byte 85 | c cipher.Block 86 | m sync.Mutex 87 | ) 88 | 89 | func init() { 90 | buf := make([]byte, keySize+aes.BlockSize) 91 | _, err := io.ReadFull(rand.Reader, buf) 92 | if err != nil { 93 | panic(err) // /dev/urandom had better work 94 | } 95 | c, err = aes.NewCipher(buf[:keySize]) 96 | if err != nil { 97 | panic(err) // AES had better work 98 | } 99 | n = aes.BlockSize 100 | ctr = buf[keySize:] 101 | b = make([]byte, aes.BlockSize) 102 | } 103 | 104 | func parseJSONString(data []byte) (ID, error) { 105 | var s string 106 | if err := json.Unmarshal(data, &s); err != nil { 107 | return 0, err 108 | } 109 | i, err := ParseID(s) 110 | if err != nil { 111 | return 0, err 112 | } 113 | return i, nil 114 | } 115 | 116 | func parseJSONInt(data []byte) (ID, error) { 117 | var i uint64 118 | if err := json.Unmarshal(data, &i); err != nil { 119 | return 0, err 120 | } 121 | return ID(i), nil 122 | } 123 | -------------------------------------------------------------------------------- /id_test.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestIDMarshalJSON(t *testing.T) { 11 | id := ID(10018820) 12 | buf := bytes.NewBuffer(nil) 13 | json.NewEncoder(buf).Encode(id) 14 | want := `"000000000098e004"` 15 | got := strings.TrimSpace(buf.String()) 16 | if got != want { 17 | t.Errorf("got %v, want %v", got, want) 18 | } 19 | } 20 | 21 | func TestIDUnmarshalJSONHexString(t *testing.T) { 22 | j := []byte(`"000000000098e004"`) 23 | var got ID 24 | if err := json.Unmarshal(j, &got); err != nil { 25 | t.Fatal(err) 26 | } 27 | want := ID(10018820) 28 | if got != want { 29 | t.Errorf("got %v, want %v", got, want) 30 | } 31 | } 32 | 33 | func TestIDUnmarshalJSONInt(t *testing.T) { 34 | j := []byte(`10018820`) 35 | var got ID 36 | if err := json.Unmarshal(j, &got); err != nil { 37 | t.Fatal(err) 38 | } 39 | want := ID(10018820) 40 | if got != want { 41 | t.Errorf("got %v, want %v", got, want) 42 | } 43 | } 44 | 45 | func TestIDUnmarshalJSONNonInt(t *testing.T) { 46 | j := []byte(`[]`) 47 | var got ID 48 | err := json.Unmarshal(j, &got) 49 | if err == nil { 50 | t.Fatalf("unexpectedly unmarshalled %v", got) 51 | } 52 | } 53 | 54 | func TestIDUnmarshalJSONNonHexString(t *testing.T) { 55 | j := []byte(`"woo"`) 56 | var got ID 57 | err := json.Unmarshal(j, &got) 58 | if err == nil { 59 | t.Fatalf("unexpectedly unmarshalled %v", got) 60 | } 61 | } 62 | 63 | func TestIDGeneration(t *testing.T) { 64 | n := 10000 65 | ids := make(map[ID]bool, n) 66 | for i := 0; i < n; i++ { 67 | id := generateID() 68 | if ids[id] { 69 | t.Errorf("duplicate ID: %v", id) 70 | } 71 | ids[id] = true 72 | } 73 | } 74 | 75 | func TestParseID(t *testing.T) { 76 | want := ID(10018181901) 77 | got, err := ParseID(want.String()) 78 | if err != nil { 79 | t.Error(err) 80 | } 81 | if got != want { 82 | t.Errorf("got %v, want %v", got, want) 83 | } 84 | } 85 | 86 | func TestParseIDError(t *testing.T) { 87 | id, err := ParseID("woo") 88 | if err == nil { 89 | t.Errorf("unexpectedly parsed value: %v", id) 90 | } 91 | } 92 | 93 | func BenchmarkIDGeneration(b *testing.B) { 94 | for i := 0; i < b.N; i++ { 95 | generateID() 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /internal/wire/collector.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gogo. 2 | // source: collector.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package wire is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | collector.proto 10 | 11 | It has these top-level messages: 12 | CollectPacket 13 | */ 14 | package wire 15 | 16 | import proto "github.com/gogo/protobuf/proto" 17 | import fmt "fmt" 18 | import math "math" 19 | 20 | // Reference imports to suppress errors if they are not otherwise used. 21 | var _ = proto.Marshal 22 | var _ = fmt.Errorf 23 | var _ = math.Inf 24 | 25 | // CollectPacket is the message sent to a remote collector server by one of 26 | // it's clients. 27 | type CollectPacket struct { 28 | Spanid *CollectPacket_SpanID `protobuf:"group,1,req,name=SpanID" json:"spanid,omitempty"` 29 | Annotation []*CollectPacket_Annotation `protobuf:"group,5,rep,name=Annotation" json:"annotation,omitempty"` 30 | XXX_unrecognized []byte `json:"-"` 31 | } 32 | 33 | func (m *CollectPacket) Reset() { *m = CollectPacket{} } 34 | func (m *CollectPacket) String() string { return proto.CompactTextString(m) } 35 | func (*CollectPacket) ProtoMessage() {} 36 | 37 | func (m *CollectPacket) GetSpanid() *CollectPacket_SpanID { 38 | if m != nil { 39 | return m.Spanid 40 | } 41 | return nil 42 | } 43 | 44 | func (m *CollectPacket) GetAnnotation() []*CollectPacket_Annotation { 45 | if m != nil { 46 | return m.Annotation 47 | } 48 | return nil 49 | } 50 | 51 | // SpanID is the group of information which can uniquely identify the exact 52 | // span being collected. 53 | type CollectPacket_SpanID struct { 54 | // trace is the root ID of the tree that contains all of the spans 55 | // related to this one. 56 | Trace *uint64 `protobuf:"fixed64,2,req,name=trace" json:"trace,omitempty"` 57 | // span is an ID that probabilistically uniquely identifies this span. 58 | Span *uint64 `protobuf:"fixed64,3,req,name=span" json:"span,omitempty"` 59 | // parent is the ID of the parent span, if any. 60 | Parent *uint64 `protobuf:"fixed64,4,opt,name=parent" json:"parent,omitempty"` 61 | XXX_unrecognized []byte `json:"-"` 62 | } 63 | 64 | func (m *CollectPacket_SpanID) Reset() { *m = CollectPacket_SpanID{} } 65 | func (m *CollectPacket_SpanID) String() string { return proto.CompactTextString(m) } 66 | func (*CollectPacket_SpanID) ProtoMessage() {} 67 | 68 | func (m *CollectPacket_SpanID) GetTrace() uint64 { 69 | if m != nil && m.Trace != nil { 70 | return *m.Trace 71 | } 72 | return 0 73 | } 74 | 75 | func (m *CollectPacket_SpanID) GetSpan() uint64 { 76 | if m != nil && m.Span != nil { 77 | return *m.Span 78 | } 79 | return 0 80 | } 81 | 82 | func (m *CollectPacket_SpanID) GetParent() uint64 { 83 | if m != nil && m.Parent != nil { 84 | return *m.Parent 85 | } 86 | return 0 87 | } 88 | 89 | // Annotation is any number of annotations for the span to be collected. 90 | type CollectPacket_Annotation struct { 91 | // key is the annotation's key. 92 | Key *string `protobuf:"bytes,6,req,name=key" json:"key,omitempty"` 93 | // value is the annotation's value, which may be either human or 94 | // machine readable, depending on the schema of the event that 95 | // generated it. 96 | Value []byte `protobuf:"bytes,7,opt,name=value" json:"value,omitempty"` 97 | XXX_unrecognized []byte `json:"-"` 98 | } 99 | 100 | func (m *CollectPacket_Annotation) Reset() { *m = CollectPacket_Annotation{} } 101 | func (m *CollectPacket_Annotation) String() string { return proto.CompactTextString(m) } 102 | func (*CollectPacket_Annotation) ProtoMessage() {} 103 | 104 | func (m *CollectPacket_Annotation) GetKey() string { 105 | if m != nil && m.Key != nil { 106 | return *m.Key 107 | } 108 | return "" 109 | } 110 | 111 | func (m *CollectPacket_Annotation) GetValue() []byte { 112 | if m != nil { 113 | return m.Value 114 | } 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /internal/wire/collector.proto: -------------------------------------------------------------------------------- 1 | package wire; 2 | 3 | // CollectPacket is the message sent to a remote collector server by one of 4 | // it's clients. 5 | message CollectPacket { 6 | // SpanID is the group of information which can uniquely identify the exact 7 | // span being collected. 8 | required group SpanID = 1 { 9 | // trace is the root ID of the tree that contains all of the spans 10 | // related to this one. 11 | required fixed64 trace = 2; 12 | 13 | // span is an ID that probabilistically uniquely identifies this span. 14 | required fixed64 span = 3; 15 | 16 | // parent is the ID of the parent span, if any. 17 | optional fixed64 parent = 4; 18 | } 19 | 20 | // Annotation is any number of annotations for the span to be collected. 21 | repeated group Annotation = 5 { 22 | // key is the annotation's key. 23 | required string key = 6; 24 | 25 | // value is the annotation's value, which may be either human or 26 | // machine readable, depending on the schema of the event that 27 | // generated it. 28 | optional bytes value = 7; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /internal/wire/gen.go: -------------------------------------------------------------------------------- 1 | package wire 2 | 3 | //go:generate protoc --proto_path=$GOPATH/src/github.com/gogo/protobuf/protobuf/google/protobuf:. --gogo_out=. collector.proto 4 | -------------------------------------------------------------------------------- /memstats.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | // memStats collects and prints some formatted *runtime.MemStats fields. 9 | type memStats struct { 10 | // Number of collections occurring. 11 | Collections int 12 | 13 | indent int 14 | } 15 | 16 | // fmtBytes returns b (in bytes) as a nice human readable string. 17 | func (m *memStats) fmtBytes(b uint64) string { 18 | var ( 19 | kb uint64 = 1024 20 | mb uint64 = kb * 1024 21 | gb uint64 = mb * 1024 22 | ) 23 | if b < kb { 24 | return fmt.Sprintf("%dB", b) 25 | } 26 | if b < mb { 27 | return fmt.Sprintf("%dKB", b/kb) 28 | } 29 | if b < gb { 30 | return fmt.Sprintf("%dMB", b/mb) 31 | } 32 | return fmt.Sprintf("%dGB", b/gb) 33 | } 34 | 35 | // repeat returns s repeated N times consecutively. 36 | func (m *memStats) repeat(s string, n int) string { 37 | var v string 38 | for i := 0; i < n; i++ { 39 | v += s 40 | } 41 | return v 42 | } 43 | 44 | // logf invokes fmt.Printf but with m.indent spaces prefixed. 45 | func (m *memStats) logf(format string, args ...interface{}) { 46 | fmt.Printf("%s%s", m.repeat(" ", m.indent), fmt.Sprintf(format, args...)) 47 | } 48 | 49 | // logColumns logs the given rows as formatted (properly indented) columns. 50 | func (m *memStats) logColumns(rows ...[]interface{}) { 51 | columnWidths := make([]int, len(rows[0])) 52 | for column := range rows[0] { 53 | for row := 0; row < len(rows); row++ { 54 | w := len(fmt.Sprintf("%v", rows[row][column])) 55 | if w > columnWidths[column] { 56 | columnWidths[column] = w 57 | } 58 | } 59 | } 60 | 61 | for _, row := range rows { 62 | m.logf("- ") 63 | for c, column := range row { 64 | w := len(fmt.Sprintf("%v", column)) 65 | fmt.Printf("%v %s", column, m.repeat(" ", columnWidths[c]-w)) 66 | } 67 | fmt.Printf("\n") 68 | } 69 | } 70 | 71 | // Log should be called with a human-readable segment name (i.e. the segment of 72 | // code whose memory perf is being tested). 73 | func (m *memStats) Log(segment string) { 74 | var s runtime.MemStats 75 | runtime.ReadMemStats(&s) 76 | m.logf("\n\n[%s] %d-collections:\n", segment, m.Collections) 77 | m.indent += 2 78 | 79 | m.logf("General statistics\n") 80 | m.indent += 2 81 | m.logColumns( 82 | []interface{}{"Alloc", m.fmtBytes(s.Alloc), "(allocated and still in use)"}, 83 | []interface{}{"TotalAlloc", m.fmtBytes(s.TotalAlloc), "(allocated (even if freed))"}, 84 | []interface{}{"Sys", m.fmtBytes(s.Sys), "(obtained from system (sum of XxxSys below))"}, 85 | []interface{}{"Lookups", s.Lookups, "(number of pointer lookups)"}, 86 | []interface{}{"Mallocs", s.Mallocs, "(number of mallocs)"}, 87 | []interface{}{"Frees", s.Frees, "(number of frees)"}, 88 | ) 89 | m.indent -= 2 90 | fmt.Printf("\n") 91 | 92 | m.logf("Main allocation heap statistics\n") 93 | m.indent += 2 94 | m.logColumns( 95 | []interface{}{"HeapAlloc", m.fmtBytes(s.HeapAlloc), "(allocated and still in use)"}, 96 | []interface{}{"HeapSys", m.fmtBytes(s.HeapSys), "(obtained from system)"}, 97 | []interface{}{"HeapIdle", m.fmtBytes(s.HeapIdle), "(in idle spans)"}, 98 | []interface{}{"HeapInuse", m.fmtBytes(s.HeapInuse), "(in non-idle span)"}, 99 | []interface{}{"HeapReleased", m.fmtBytes(s.HeapReleased), "(released to the OS)"}, 100 | []interface{}{"HeapObjects", s.HeapObjects, "(total number of allocated objects)"}, 101 | ) 102 | m.indent -= 2 103 | fmt.Printf("\n") 104 | 105 | m.logf("Low-level fixed-size structure allocator statistics\n") 106 | m.indent += 2 107 | m.logColumns( 108 | []interface{}{"StackInuse", m.fmtBytes(s.StackInuse), "(used by stack allocator right now)"}, 109 | []interface{}{"StackSys", m.fmtBytes(s.StackSys), "(obtained from system)"}, 110 | []interface{}{"MSpanInuse", m.fmtBytes(s.MSpanInuse), "(mspan structures / in use now)"}, 111 | []interface{}{"MSpanSys", m.fmtBytes(s.MSpanSys), "(obtained from system)"}, 112 | []interface{}{"MCacheInuse", m.fmtBytes(s.MCacheInuse), "(in use now))"}, 113 | []interface{}{"MCacheSys", m.fmtBytes(s.MCacheSys), "(mcache structures / obtained from system)"}, 114 | []interface{}{"BuckHashSys", m.fmtBytes(s.BuckHashSys), "(profiling bucket hash table / obtained from system)"}, 115 | []interface{}{"GCSys", m.fmtBytes(s.GCSys), "(GC metadata / obtained form system)"}, 116 | []interface{}{"OtherSys", m.fmtBytes(s.OtherSys), "(other system allocations)"}, 117 | ) 118 | fmt.Printf("\n") 119 | m.indent -= 2 120 | 121 | // TODO(slimsag): remaining GC fields may be useful later: 122 | /* 123 | // Garbage collector statistics. 124 | NextGC uint64 // next collection will happen when HeapAlloc ≥ this amount 125 | LastGC uint64 // end time of last collection (nanoseconds since 1970) 126 | PauseTotalNs uint64 127 | PauseNs [256]uint64 // circular buffer of recent GC pause durations, most recent at [(NumGC+255)%256] 128 | PauseEnd [256]uint64 // circular buffer of recent GC pause end times 129 | NumGC uint32 130 | EnableGC bool 131 | DebugGC bool 132 | 133 | // Per-size allocation statistics. 134 | // 61 is NumSizeClasses in the C code. 135 | BySize [61]struct { 136 | Size uint32 137 | Mallocs uint64 138 | Frees uint64 139 | } 140 | */ 141 | } 142 | -------------------------------------------------------------------------------- /multi.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | // multiStore is like a normal store except all operations occur on the multiple 4 | // underlying stores. 5 | type multiStore struct { 6 | // stores is the underlying set of stores that operations take place on. 7 | stores []Store 8 | } 9 | 10 | // Collect implements the Collector interface by invoking Collect on each 11 | // underlying store, returning the first error that occurs. 12 | func (ms *multiStore) Collect(id SpanID, anns ...Annotation) error { 13 | for _, s := range ms.stores { 14 | if err := s.Collect(id, anns...); err != nil { 15 | return err 16 | } 17 | } 18 | return nil 19 | } 20 | 21 | // Trace implements the Store interface by returning the first trace found by 22 | // asking each underlying store for it in consecutive order. 23 | func (ms *multiStore) Trace(t ID) (*Trace, error) { 24 | for _, s := range ms.stores { 25 | trace, err := s.Trace(t) 26 | if err == ErrTraceNotFound { 27 | continue 28 | } else if err != nil { 29 | return nil, err 30 | } 31 | return trace, nil 32 | } 33 | return nil, ErrTraceNotFound 34 | } 35 | 36 | // MultiStore returns a Store whose operations occur on the multiple given 37 | // stores. 38 | func MultiStore(s ...Store) Store { 39 | return &multiStore{ 40 | stores: s, 41 | } 42 | } 43 | 44 | // multiStore is like a normal queryer except it queries from multiple 45 | // underlying stores. 46 | type multiQueryer struct { 47 | // queryers is the underlying set of queryers that operations take place on. 48 | queryers []Queryer 49 | } 50 | 51 | // Traces implements the Queryer interface by returning the union of all 52 | // underlying stores. 53 | // 54 | // It panics if any underlying store does not implement the appdash Queryer 55 | // interface. 56 | func (mq *multiQueryer) Traces(opts TracesOpts) ([]*Trace, error) { 57 | var ( 58 | union = make(map[ID]struct{}) 59 | all []*Trace 60 | ) 61 | for _, q := range mq.queryers { 62 | traces, err := q.Traces(TracesOpts{}) 63 | if err != nil { 64 | return nil, err 65 | } 66 | for _, t := range traces { 67 | if _, ok := union[t.ID.Trace]; !ok { 68 | union[t.ID.Trace] = struct{}{} 69 | all = append(all, t) 70 | } 71 | } 72 | } 73 | return all, nil 74 | } 75 | 76 | // MultiQueryer returns a Queryer whose Traces method returns a union of all 77 | // traces across each queryer. 78 | func MultiQueryer(q ...Queryer) Queryer { 79 | return &multiQueryer{ 80 | queryers: q, 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /opentracing/json.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2016 Uber Technologies, Inc. 2 | // Copyright (c) 2016 Bas van Beek 3 | 4 | // Permission is hereby granted, free of charge, to any person obtaining a copy 5 | // of this software and associated documentation files (the "Software"), to deal 6 | // in the Software without restriction, including without limitation the rights 7 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | // copies of the Software, and to permit persons to whom the Software is 9 | // furnished to do so, subject to the following conditions: 10 | // 11 | // The above copyright notice and this permission notice shall be included in 12 | // all copies or substantial portions of the Software. 13 | // 14 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | // THE SOFTWARE. 21 | 22 | package opentracing 23 | 24 | import ( 25 | "encoding/json" 26 | "fmt" 27 | 28 | "github.com/opentracing/opentracing-go/log" 29 | ) 30 | 31 | type fieldsAsMap map[string]string 32 | 33 | // materializeWithJSON converts log Fields into JSON string 34 | func materializeWithJSON(logFields []log.Field) ([]byte, error) { 35 | fields := fieldsAsMap(make(map[string]string, len(logFields))) 36 | for _, field := range logFields { 37 | field.Marshal(fields) 38 | } 39 | // if we only have an event log Field we do not create a json serialization of 40 | // the key-value pairs contained within the log Fields, but simply return the 41 | // payload of the event log Field. 42 | if len(fields) == 1 { 43 | if event, ok := fields["event"]; ok { 44 | return []byte(event), nil 45 | } 46 | } 47 | return json.Marshal(fields) 48 | } 49 | 50 | func (ml fieldsAsMap) EmitString(key, value string) { 51 | ml[key] = value 52 | } 53 | 54 | func (ml fieldsAsMap) EmitBool(key string, value bool) { 55 | ml[key] = fmt.Sprintf("%t", value) 56 | } 57 | 58 | func (ml fieldsAsMap) EmitInt(key string, value int) { 59 | ml[key] = fmt.Sprintf("%d", value) 60 | } 61 | 62 | func (ml fieldsAsMap) EmitInt32(key string, value int32) { 63 | ml[key] = fmt.Sprintf("%d", value) 64 | } 65 | 66 | func (ml fieldsAsMap) EmitInt64(key string, value int64) { 67 | ml[key] = fmt.Sprintf("%d", value) 68 | } 69 | 70 | func (ml fieldsAsMap) EmitUint32(key string, value uint32) { 71 | ml[key] = fmt.Sprintf("%d", value) 72 | } 73 | 74 | func (ml fieldsAsMap) EmitUint64(key string, value uint64) { 75 | ml[key] = fmt.Sprintf("%d", value) 76 | } 77 | 78 | func (ml fieldsAsMap) EmitFloat32(key string, value float32) { 79 | ml[key] = fmt.Sprintf("%f", value) 80 | } 81 | 82 | func (ml fieldsAsMap) EmitFloat64(key string, value float64) { 83 | ml[key] = fmt.Sprintf("%f", value) 84 | } 85 | 86 | func (ml fieldsAsMap) EmitObject(key string, value interface{}) { 87 | ml[key] = fmt.Sprintf("%+v", value) 88 | } 89 | 90 | func (ml fieldsAsMap) EmitLazyLogger(value log.LazyLogger) { 91 | value(ml) 92 | } 93 | -------------------------------------------------------------------------------- /opentracing/recorder.go: -------------------------------------------------------------------------------- 1 | // Package opentracing provides an Appdash implementation of the OpenTracing 2 | // API. 3 | // 4 | // The OpenTracing specification allows for Span Tags to have an arbitrary 5 | // value. The way the Appdash.Recorder handles this is by converting the 6 | // tag value into a string using the default format for its type. Arbitrary 7 | // structs have their field name included. 8 | // 9 | // The Appdash implementation also does not record Log payloads. 10 | package opentracing 11 | 12 | import ( 13 | "fmt" 14 | "log" 15 | "sync" 16 | 17 | basictracer "github.com/opentracing/basictracer-go" 18 | "sourcegraph.com/sourcegraph/appdash" 19 | ) 20 | 21 | // Recorder implements the basictracer.Recorder interface. 22 | type Recorder struct { 23 | collector appdash.Collector 24 | logOnce sync.Once 25 | verbose bool 26 | Log *log.Logger 27 | } 28 | 29 | // NewRecorder forwards basictracer.RawSpans to an appdash.Collector. 30 | func NewRecorder(collector appdash.Collector, opts Options) *Recorder { 31 | if opts.Logger == nil { 32 | opts.Logger = newLogger() 33 | } 34 | return &Recorder{ 35 | collector: collector, 36 | verbose: opts.Verbose, 37 | Log: opts.Logger, 38 | } 39 | } 40 | 41 | // RecordSpan converts a RawSpan into the Appdash representation of a span 42 | // and records it to the underlying collector. 43 | func (r *Recorder) RecordSpan(sp basictracer.RawSpan) { 44 | if !sp.Context.Sampled { 45 | return 46 | } 47 | 48 | spanID := appdash.SpanID{ 49 | Span: appdash.ID(uint64(sp.Context.SpanID)), 50 | Trace: appdash.ID(uint64(sp.Context.TraceID)), 51 | Parent: appdash.ID(uint64(sp.ParentSpanID)), 52 | } 53 | 54 | r.collectEvent(spanID, appdash.SpanName(sp.Operation)) 55 | 56 | // Record all of the logs. 57 | for _, log := range sp.Logs { 58 | if logs, err := materializeWithJSON(log.Fields); err != nil { 59 | r.logError(spanID, err) 60 | } else { 61 | r.collectEvent(spanID, appdash.LogWithTimestamp(string(logs), log.Timestamp)) 62 | } 63 | } 64 | 65 | for key, value := range sp.Tags { 66 | val := []byte(fmt.Sprintf("%+v", value)) 67 | r.collectAnnotation(spanID, appdash.Annotation{Key: key, Value: val}) 68 | } 69 | 70 | for key, val := range sp.Context.Baggage { 71 | r.collectAnnotation(spanID, appdash.Annotation{Key: key, Value: []byte(val)}) 72 | } 73 | 74 | // Add the duration to the start time to get an approximate end time. 75 | approxEndTime := sp.Start.Add(sp.Duration) 76 | r.collectEvent(spanID, appdash.Timespan{S: sp.Start, E: approxEndTime}) 77 | } 78 | 79 | // collectEvent marshals and collects the Event. 80 | func (r *Recorder) collectEvent(spanID appdash.SpanID, e appdash.Event) { 81 | ans, err := appdash.MarshalEvent(e) 82 | if err != nil { 83 | r.logError(spanID, err) 84 | return 85 | } 86 | r.collectAnnotation(spanID, ans...) 87 | } 88 | 89 | func (r *Recorder) collectAnnotation(spanID appdash.SpanID, ans ...appdash.Annotation) { 90 | err := r.collector.Collect(spanID, ans...) 91 | if err != nil { 92 | r.logError(spanID, err) 93 | } 94 | } 95 | 96 | // logError converts an error into a log event and collects it. 97 | // If for whatever reason the error can't be collected, it is logged to the 98 | // Recorder's logger if it is non-nil. 99 | func (r *Recorder) logError(spanID appdash.SpanID, err error) { 100 | ans, _ := appdash.MarshalEvent(appdash.Log(err.Error())) 101 | 102 | // At this point, something is definitely wrong. 103 | if err := r.collector.Collect(spanID, ans...); err != nil { 104 | if r.verbose { 105 | r.Log.Printf("Appdash Recorder collect error: %v\n", err) 106 | } else { 107 | r.logOnce.Do(func() { r.Log.Printf("Appdash Recorder collect error: %v\n", err) }) 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /opentracing/recorder_test.go: -------------------------------------------------------------------------------- 1 | package opentracing 2 | 3 | import ( 4 | "reflect" 5 | "sort" 6 | "testing" 7 | "time" 8 | 9 | basictracer "github.com/opentracing/basictracer-go" 10 | 11 | "sourcegraph.com/sourcegraph/appdash" 12 | "sourcegraph.com/sourcegraph/appdash/internal/wire" 13 | ) 14 | 15 | func TestOpentracingRecorder(t *testing.T) { 16 | var packets []*wire.CollectPacket 17 | mc := collectorFunc(func(span appdash.SpanID, anns ...appdash.Annotation) error { 18 | packets = append(packets, newCollectPacket(span, anns)) 19 | return nil 20 | }) 21 | 22 | baggageKey := "somelongval" 23 | baggageVal := "val" 24 | opName := "testOperation" 25 | 26 | r := NewRecorder(mc, Options{}) 27 | raw := basictracer.RawSpan{ 28 | Context: basictracer.SpanContext{ 29 | TraceID: 1, 30 | SpanID: 2, 31 | Sampled: true, 32 | Baggage: map[string]string{ 33 | baggageKey: baggageVal, 34 | }, 35 | }, 36 | ParentSpanID: 3, 37 | Tags: map[string]interface{}{ 38 | "tag": 1, 39 | }, 40 | Operation: opName, 41 | Duration: time.Duration(1), 42 | } 43 | 44 | unsampledRaw := basictracer.RawSpan{ 45 | Context: basictracer.SpanContext{ 46 | TraceID: 1, 47 | SpanID: 2, 48 | Sampled: false, 49 | }, 50 | ParentSpanID: 3, 51 | } 52 | 53 | r.RecordSpan(raw) 54 | r.RecordSpan(unsampledRaw) 55 | 56 | tsAnnotations := marshalEvent(appdash.Timespan{raw.Start, raw.Start.Add(raw.Duration)}) 57 | want := []*wire.CollectPacket{ 58 | newCollectPacket(appdash.SpanID{1, 2, 3}, appdash.Annotations{{"tag", []byte("1")}}), 59 | newCollectPacket(appdash.SpanID{1, 2, 3}, appdash.Annotations{{baggageKey, []byte(baggageVal)}}), 60 | newCollectPacket(appdash.SpanID{1, 2, 3}, marshalEvent(appdash.SpanName(opName))), 61 | newCollectPacket(appdash.SpanID{1, 2, 3}, tsAnnotations), 62 | } 63 | 64 | sort.Sort(byTraceID(packets)) 65 | sort.Sort(byTraceID(want)) 66 | if !reflect.DeepEqual(packets, want) { 67 | t.Errorf("Got packets %v, want %v", packets, want) 68 | } 69 | } 70 | 71 | // newCollectPacket returns an initialized *wire.CollectPacket given a span and 72 | // set of annotations. 73 | func newCollectPacket(s appdash.SpanID, as appdash.Annotations) *wire.CollectPacket { 74 | swire := &wire.CollectPacket_SpanID{ 75 | Trace: (*uint64)(&s.Trace), 76 | Span: (*uint64)(&s.Span), 77 | Parent: (*uint64)(&s.Parent), 78 | } 79 | w := []*wire.CollectPacket_Annotation{} 80 | 81 | for _, a := range as { 82 | // Important: Make a copy of a that we can retain a pointer to that 83 | // doesn't change after each iteration. Otherwise all wire annotations 84 | // would have the same key. 85 | cpy := a 86 | w = append(w, &wire.CollectPacket_Annotation{ 87 | Key: &cpy.Key, 88 | Value: cpy.Value, 89 | }) 90 | } 91 | return &wire.CollectPacket{ 92 | Spanid: swire, 93 | Annotation: w, 94 | } 95 | } 96 | 97 | type collectorFunc func(appdash.SpanID, ...appdash.Annotation) error 98 | 99 | // Collect implements the Collector interface by calling the function itself. 100 | func (c collectorFunc) Collect(id appdash.SpanID, as ...appdash.Annotation) error { 101 | return c(id, as...) 102 | } 103 | 104 | func marshalEvent(e appdash.Event) appdash.Annotations { 105 | ans, _ := appdash.MarshalEvent(e) 106 | return ans 107 | } 108 | 109 | type byTraceID []*wire.CollectPacket 110 | 111 | func (bt byTraceID) Len() int { return len(bt) } 112 | func (bt byTraceID) Swap(i, j int) { bt[i], bt[j] = bt[j], bt[i] } 113 | func (bt byTraceID) Less(i, j int) bool { 114 | if *bt[i].Spanid.Trace < *bt[j].Spanid.Trace { 115 | return true 116 | } 117 | 118 | // If the packet has more than one annotation, sort those annotation by name. 119 | if len(bt[i].Annotation) > 1 { 120 | sort.Sort(byAnnotation(bt[i].Annotation)) 121 | } 122 | if len(bt[j].Annotation) > 1 { 123 | sort.Sort(byAnnotation(bt[i].Annotation)) 124 | } 125 | return *bt[i].Annotation[0].Key < *bt[j].Annotation[0].Key 126 | } 127 | 128 | type byAnnotation []*wire.CollectPacket_Annotation 129 | 130 | func (bt byAnnotation) Len() int { return len(bt) } 131 | func (bt byAnnotation) Swap(i, j int) { bt[i], bt[j] = bt[j], bt[i] } 132 | func (bt byAnnotation) Less(i, j int) bool { return *bt[i].Key < *bt[j].Key } 133 | -------------------------------------------------------------------------------- /opentracing/tracer.go: -------------------------------------------------------------------------------- 1 | package opentracing 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | basictracer "github.com/opentracing/basictracer-go" 8 | opentracing "github.com/opentracing/opentracing-go" 9 | "sourcegraph.com/sourcegraph/appdash" 10 | ) 11 | 12 | var _ opentracing.Tracer = NewTracer(nil) // Compile time check. 13 | 14 | // Options defines options for a Tracer. 15 | type Options struct { 16 | // ShouldSample is a function that allows deterministic sampling of a trace 17 | // using the randomly generated Trace ID. The decision is made when a new Trace 18 | // is created and is propagated to all of the trace's spans. For example, 19 | // 20 | // func(traceID int64) { return traceID % 128 == 0 } 21 | // 22 | // samples 1 in every 128 traces, approximately. 23 | ShouldSample func(traceID uint64) bool 24 | 25 | // Verbose determines whether errors are logged to stdout only once or all 26 | // the time. By default, Verbose is false so only the first error is logged 27 | // and the rest are silenced. 28 | Verbose bool 29 | 30 | // Logger is used to log critical errors that can't be collected by the 31 | // Appdash Collector. 32 | Logger *log.Logger 33 | } 34 | 35 | func newLogger() *log.Logger { 36 | return log.New(os.Stderr, "opentracing: ", log.LstdFlags) 37 | } 38 | 39 | // DefaultOptions creates an Option with a sampling function that always return 40 | // true and a logger that logs errors to stderr. 41 | func DefaultOptions() Options { 42 | return Options{ 43 | ShouldSample: func(_ uint64) bool { return true }, 44 | Logger: newLogger(), 45 | } 46 | } 47 | 48 | // NewTracer creates a new opentracing.Tracer implementation that reports 49 | // spans to an Appdash collector. 50 | // 51 | // The Tracer created by NewTracer reports all spans by default. If you want to 52 | // sample 1 in every N spans, see NewTracerWithOptions. Spans are written to 53 | // the underlying collector when Finish() is called on the span. It is 54 | // possible to buffer and write span on a time interval using appdash.ChunkedCollector. 55 | // 56 | // For example: 57 | // 58 | // collector := appdash.NewLocalCollector(myAppdashStore) 59 | // chunkedCollector := appdash.NewChunkedCollector(collector) 60 | // 61 | // tracer := NewTracer(chunkedCollector) 62 | // 63 | // If writing traces to a remote Appdash collector, an appdash.RemoteCollector would 64 | // be needed, for example: 65 | // 66 | // collector := appdash.NewRemoteCollector("localhost:8700") 67 | // tracer := NewTracer(collector) 68 | // 69 | // will record all spans to a collector server on localhost:8700. 70 | func NewTracer(c appdash.Collector) opentracing.Tracer { 71 | return NewTracerWithOptions(c, DefaultOptions()) 72 | } 73 | 74 | // NewTracerWithOptions creates a new opentracing.Tracer that records spans to 75 | // the given appdash.Collector. 76 | func NewTracerWithOptions(c appdash.Collector, options Options) opentracing.Tracer { 77 | opts := basictracer.DefaultOptions() 78 | opts.ShouldSample = options.ShouldSample 79 | opts.Recorder = NewRecorder(c, options) 80 | return basictracer.NewWithOptions(opts) 81 | } 82 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | # AppDash: Python 2 | 3 | Appdash-python enables Python-based applications to send performance and debug information to a remote Go-based Appdash collection server. 4 | 5 | The `appdash` module provides everything that is needed to get started, as well as two collectors (an asynchronous one based on the Twisted networking library, and a synchronous one Python's socket module). 6 | 7 | ## Integration 8 | 9 | At this point, there is no integration with common Python web-frameworks -- although adding support for such should be trivial. 10 | 11 | Generally speaking if the web framework has some form of integration with the Twisted networking library, using the Twisted collector (`appdash.twcollector`) is a better choice as it is asynchronous and Twisted is better at scheduling tasks. 12 | 13 | Using Twisted is not always possible, for example Django applications that are hosted by Apache (and thus, Twisted cannot be long-running). To suit these cases, a synchronous remote collector is provided (`appdash.sockcollector`) which operates using Python's standard socket module. 14 | 15 | For quick'n'dirty integration, the synchronous collector is probably more straight-forward as well (i.e. if you are not accustomed to working with Twisted's asynchronous programming model). 16 | 17 | ## Prerequisites 18 | 19 | To install appdash for python you'll first need to install a few things through the standard `easy_install` and `pip` python package managers: 20 | 21 | ``` 22 | # (Ubuntu/Linux) Install easy_install and pip: 23 | sudo apt-get install python-setuptools python-pip 24 | 25 | # Install Google's protobuf: 26 | easy_install protobuf 27 | 28 | # Install Twisted networking (optional, only needed for Twisted integration): 29 | easy_install twisted 30 | 31 | # Install strict-rfc3339 32 | pip install strict-rfc3339 33 | ``` 34 | 35 | Depending on where Python is installed and/or what permissions the directory has, you may need to run the above `easy_install` and `pip` commands as root. 36 | 37 | ## Installation 38 | 39 | To install Appdash into your Python path (i.e. so you can import it into your own code), simply change directory to `sourcegraph.com/sourcewgraph/appdash/python` and run the traditional `setup.py` script: 40 | 41 | ``` 42 | # Install Python-appdash: 43 | cd $GOPATH/src/sourcegraph.com/sourcegraph/appdash/python 44 | python setup.py install 45 | ``` 46 | 47 | Again, depending on where Python is installed and/or what permissions the directory has, you may need to run the above `setup.py` script as root. 48 | 49 | To test that installation went well, in any directory _except that one_, you can launch an interactive Python interpreter and simply `import appdash`. 50 | 51 | ## Twisted Example 52 | 53 | If all is well with your setup, you should be able to change directory to `sourcegraph.com/sourcewgraph/appdash/python` and run the Twisted example: 54 | 55 | ``` 56 | # Run appdash server in separate terminal: 57 | appdash serve 58 | 59 | # Run the example script: 60 | cd $GOPATH/src/sourcegraph.com/sourcegraph/appdash/python 61 | ./example_twisted.py 62 | ``` 63 | 64 | ## Socket Example 65 | 66 | If you prefer not to use Twisted, you can utilize the standard socket collector (`appdash.sockcollector`) provided. You can run the example: 67 | 68 | ``` 69 | # Run appdash server in separate terminal: 70 | appdash serve 71 | 72 | # Run the example script: 73 | cd $GOPATH/src/sourcegraph.com/sourcegraph/appdash/python 74 | ./example_socket.py 75 | ``` 76 | -------------------------------------------------------------------------------- /python/appdash/__init__.py: -------------------------------------------------------------------------------- 1 | from spanid import * 2 | from event import * 3 | from recorder import * 4 | -------------------------------------------------------------------------------- /python/appdash/collector_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: collector.proto 3 | 4 | from google.protobuf import descriptor as _descriptor 5 | from google.protobuf import message as _message 6 | from google.protobuf import reflection as _reflection 7 | from google.protobuf import descriptor_pb2 8 | # @@protoc_insertion_point(imports) 9 | 10 | 11 | 12 | 13 | DESCRIPTOR = _descriptor.FileDescriptor( 14 | name='collector.proto', 15 | package='wire', 16 | serialized_pb='\n\x0f\x63ollector.proto\x12\x04wire\"\xd0\x01\n\rCollectPacket\x12*\n\x06spanid\x18\x01 \x02(\n2\x1a.wire.CollectPacket.SpanID\x12\x32\n\nannotation\x18\x05 \x03(\n2\x1e.wire.CollectPacket.Annotation\x1a\x35\n\x06SpanID\x12\r\n\x05trace\x18\x02 \x02(\x06\x12\x0c\n\x04span\x18\x03 \x02(\x06\x12\x0e\n\x06parent\x18\x04 \x01(\x06\x1a(\n\nAnnotation\x12\x0b\n\x03key\x18\x06 \x02(\t\x12\r\n\x05value\x18\x07 \x01(\x0c') 17 | 18 | 19 | 20 | 21 | _COLLECTPACKET_SPANID = _descriptor.Descriptor( 22 | name='SpanID', 23 | full_name='wire.CollectPacket.SpanID', 24 | filename=None, 25 | file=DESCRIPTOR, 26 | containing_type=None, 27 | fields=[ 28 | _descriptor.FieldDescriptor( 29 | name='trace', full_name='wire.CollectPacket.SpanID.trace', index=0, 30 | number=2, type=6, cpp_type=4, label=2, 31 | has_default_value=False, default_value=0, 32 | message_type=None, enum_type=None, containing_type=None, 33 | is_extension=False, extension_scope=None, 34 | options=None), 35 | _descriptor.FieldDescriptor( 36 | name='span', full_name='wire.CollectPacket.SpanID.span', index=1, 37 | number=3, type=6, cpp_type=4, label=2, 38 | has_default_value=False, default_value=0, 39 | message_type=None, enum_type=None, containing_type=None, 40 | is_extension=False, extension_scope=None, 41 | options=None), 42 | _descriptor.FieldDescriptor( 43 | name='parent', full_name='wire.CollectPacket.SpanID.parent', index=2, 44 | number=4, type=6, cpp_type=4, label=1, 45 | has_default_value=False, default_value=0, 46 | message_type=None, enum_type=None, containing_type=None, 47 | is_extension=False, extension_scope=None, 48 | options=None), 49 | ], 50 | extensions=[ 51 | ], 52 | nested_types=[], 53 | enum_types=[ 54 | ], 55 | options=None, 56 | is_extendable=False, 57 | extension_ranges=[], 58 | serialized_start=139, 59 | serialized_end=192, 60 | ) 61 | 62 | _COLLECTPACKET_ANNOTATION = _descriptor.Descriptor( 63 | name='Annotation', 64 | full_name='wire.CollectPacket.Annotation', 65 | filename=None, 66 | file=DESCRIPTOR, 67 | containing_type=None, 68 | fields=[ 69 | _descriptor.FieldDescriptor( 70 | name='key', full_name='wire.CollectPacket.Annotation.key', index=0, 71 | number=6, type=9, cpp_type=9, label=2, 72 | has_default_value=False, default_value=unicode("", "utf-8"), 73 | message_type=None, enum_type=None, containing_type=None, 74 | is_extension=False, extension_scope=None, 75 | options=None), 76 | _descriptor.FieldDescriptor( 77 | name='value', full_name='wire.CollectPacket.Annotation.value', index=1, 78 | number=7, type=12, cpp_type=9, label=1, 79 | has_default_value=False, default_value="", 80 | message_type=None, enum_type=None, containing_type=None, 81 | is_extension=False, extension_scope=None, 82 | options=None), 83 | ], 84 | extensions=[ 85 | ], 86 | nested_types=[], 87 | enum_types=[ 88 | ], 89 | options=None, 90 | is_extendable=False, 91 | extension_ranges=[], 92 | serialized_start=194, 93 | serialized_end=234, 94 | ) 95 | 96 | _COLLECTPACKET = _descriptor.Descriptor( 97 | name='CollectPacket', 98 | full_name='wire.CollectPacket', 99 | filename=None, 100 | file=DESCRIPTOR, 101 | containing_type=None, 102 | fields=[ 103 | _descriptor.FieldDescriptor( 104 | name='spanid', full_name='wire.CollectPacket.spanid', index=0, 105 | number=1, type=10, cpp_type=10, label=2, 106 | has_default_value=False, default_value=None, 107 | message_type=None, enum_type=None, containing_type=None, 108 | is_extension=False, extension_scope=None, 109 | options=None), 110 | _descriptor.FieldDescriptor( 111 | name='annotation', full_name='wire.CollectPacket.annotation', index=1, 112 | number=5, type=10, cpp_type=10, label=3, 113 | has_default_value=False, default_value=[], 114 | message_type=None, enum_type=None, containing_type=None, 115 | is_extension=False, extension_scope=None, 116 | options=None), 117 | ], 118 | extensions=[ 119 | ], 120 | nested_types=[_COLLECTPACKET_SPANID, _COLLECTPACKET_ANNOTATION, ], 121 | enum_types=[ 122 | ], 123 | options=None, 124 | is_extendable=False, 125 | extension_ranges=[], 126 | serialized_start=26, 127 | serialized_end=234, 128 | ) 129 | 130 | _COLLECTPACKET_SPANID.containing_type = _COLLECTPACKET; 131 | _COLLECTPACKET_ANNOTATION.containing_type = _COLLECTPACKET; 132 | _COLLECTPACKET.fields_by_name['spanid'].message_type = _COLLECTPACKET_SPANID 133 | _COLLECTPACKET.fields_by_name['annotation'].message_type = _COLLECTPACKET_ANNOTATION 134 | DESCRIPTOR.message_types_by_name['CollectPacket'] = _COLLECTPACKET 135 | 136 | class CollectPacket(_message.Message): 137 | __metaclass__ = _reflection.GeneratedProtocolMessageType 138 | 139 | class SpanID(_message.Message): 140 | __metaclass__ = _reflection.GeneratedProtocolMessageType 141 | DESCRIPTOR = _COLLECTPACKET_SPANID 142 | 143 | # @@protoc_insertion_point(class_scope:wire.CollectPacket.SpanID) 144 | 145 | class Annotation(_message.Message): 146 | __metaclass__ = _reflection.GeneratedProtocolMessageType 147 | DESCRIPTOR = _COLLECTPACKET_ANNOTATION 148 | 149 | # @@protoc_insertion_point(class_scope:wire.CollectPacket.Annotation) 150 | DESCRIPTOR = _COLLECTPACKET 151 | 152 | # @@protoc_insertion_point(class_scope:wire.CollectPacket) 153 | 154 | 155 | # @@protoc_insertion_point(module_scope) 156 | -------------------------------------------------------------------------------- /python/appdash/encode.py: -------------------------------------------------------------------------------- 1 | import collector_pb2 as wire 2 | import varint 3 | 4 | # _msg encodes the protobuf message and returns it as a string. The 5 | # serialized protobuf message is preceded by a varint-encoded length of the 6 | # message, which allows for streaming of multiple messages (i.e. a delimited 7 | # protobuf message). 8 | def _msg(msg): 9 | data = msg.SerializeToString() 10 | return varint.encode(len(data)) + data 11 | 12 | # _collect collects the annotations for the spanID by returning a 13 | # protobuf CollectPacket's which can be directly encoded via a call to encodeMsg. 14 | def _collect(spanID, *annotations): 15 | # Create the protobuf message. 16 | p = wire.CollectPacket() 17 | 18 | # Copy over the IDs. 19 | p.spanid.trace = spanID.trace 20 | p.spanid.span = spanID.span 21 | p.spanid.parent = spanID.parent 22 | 23 | # Add each annotation to the message. 24 | for a in annotations: 25 | # Add a new annotation to the packet, copying over the key/value pair. 26 | ap = p.annotation.add() 27 | ap.key = a.key 28 | ap.value = a.value 29 | 30 | # Return the protobuf message. 31 | return p 32 | -------------------------------------------------------------------------------- /python/appdash/recorder.py: -------------------------------------------------------------------------------- 1 | import event 2 | import itertools 3 | from spanid import SpanID, Annotation 4 | from basictracer import BasicTracer, SpanRecorder 5 | 6 | def create_new_tracer(collector, sampler=None): 7 | """create_new_tracer creates a new appdash opentracing tracer using an Appdash collector. 8 | """ 9 | return BasicTracer(recorder=AppdashRecorder(collector), sampler=sampler) 10 | 11 | class AppdashRecorder(SpanRecorder): 12 | """AppdashRecorder collects and records spans to a remote Appdash collector. 13 | """ 14 | def __init__(self, collector): 15 | self._collector = collector 16 | 17 | def record_span(self, span): 18 | if not span.context.sampled: 19 | return 20 | span_id = SpanID() 21 | span_id.trace = span.context.trace_id 22 | span_id.span = span.context.span_id 23 | if span.parent_id is not None: 24 | span_id.parent = span.parent_id 25 | 26 | self._collector.collect(span_id, 27 | *event.MarshalEvent(event.SpanNameEvent(span.operation_name))) 28 | 29 | approx_endtime = span.start_time + span.duration 30 | self._collector.collect(span_id, 31 | *event.MarshalEvent(event.TimespanEvent(span.start_time, approx_endtime))) 32 | 33 | if span.tags is not None: 34 | for key in span.tags: 35 | self._collector.collect(span_id, Annotation(key, span.tags[key])) 36 | 37 | if span.context.baggage is not None: 38 | for key in span.context.baggage: 39 | self._collector.collect(span_id, Annotation(key, span.contex.baggage[key])) 40 | 41 | -------------------------------------------------------------------------------- /python/appdash/sockcollector.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import encode 3 | 4 | # RemoteCollector is a remote collector that operates over a standard, and 5 | # synchronous, Python socket. 6 | # 7 | # rc = RemoteCollector(sock=None, debug=True) 8 | # 9 | # try: 10 | # rc.connect(host="localhost", port=7701) 11 | # except Exception as e: 12 | # print "Failed to connect:", e 13 | # 14 | # try: 15 | # rc.collect(spanID, annotationOne, annotationTwo) 16 | # except Exception as e: 17 | # print "Failed to collect:", e 18 | # 19 | # rc.close() 20 | # 21 | # A custom socket for the sock parameter to RemoteCollector's allows one to 22 | # specify e.g. a TLS socket. 23 | class RemoteCollector: 24 | # sock is literally the socket that is used to communicate with the remote 25 | # collector. 26 | sock = None 27 | 28 | _debug = False 29 | 30 | def __init__(self, sock=None, debug=False): 31 | self.sock = sock 32 | self._debug = debug 33 | 34 | def _log(self, *args): 35 | if self._debug: 36 | print "appdash: %s" % (" ".join(args)) 37 | 38 | # connect connects the underlying socket to the given address, waiting at 39 | # max for the given timeout before raising an exception. 40 | def connect(self, host="localhost", port=7701, timeout=10): 41 | # Use the given socket, or create a new one. 42 | if self.sock is None: 43 | self.sock = socket.create_connection((host, port), timeout=timeout) 44 | else: 45 | self.sock.connect() 46 | 47 | # collect collects annotations for the given spanID. 48 | # 49 | # The annotations are sent to the remote server immediately, and this 50 | # function does not return until all have been sent out or an exception has 51 | # occured (e.g. disconnection). 52 | def collect(self, spanID, *annotations): 53 | self._log("collecting", str(len(annotations)), "annotations for", str(spanID)) 54 | packet = encode._collect(spanID, *annotations) 55 | buf = encode._msg(packet) 56 | 57 | totalSent = 0 58 | while totalSent < len(buf): 59 | sent = self.sock.send(buf[totalSent:]) 60 | if sent == 0: 61 | raise RuntimeError("socket connection broken") 62 | totalSent = totalSent + sent 63 | 64 | # close closes the underlying socket. 65 | def close(self): 66 | self.sock.close() 67 | self.sock = None 68 | 69 | -------------------------------------------------------------------------------- /python/appdash/spanid.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | # random.SystemRandom is effectively /dev/urandom with some helper utilities, 4 | # see the random docs for details. 5 | __sysrand = random.SystemRandom() 6 | 7 | # _generateID returns a randomly-generated 64-bit ID. It is produced using the 8 | # system's cryptographically-secure RNG (/dev/urandom). 9 | def _generateID(): 10 | return __sysrand.getrandbits(64) 11 | 12 | # A SpanID refers to a single span. 13 | # 14 | # If you pass root=True, you are generating a root span (which should only be 15 | # used to generate entries for spans caused exclusively by spans which are 16 | # outside of your system as a whole, for example, a root span for the first 17 | # time you see a user request). For example: 18 | # 19 | # trace = SpanID(root=True) 20 | # 21 | # Otherwise, if you pass parent=someParentSpanID, you are creating a new ID for 22 | # a span which is the child of the given parent ID. This should be used to 23 | # track causal relationships between spans. For example: 24 | # 25 | # span = SpanID(parent=someParentSpanID) 26 | # 27 | # Creation of a span with explicit ID's is also possible: 28 | # 29 | # span = SpanID() 30 | # span.trace = theTraceID 31 | # span.span = theSpanID 32 | # span.parent = theParentID 33 | # 34 | class SpanID: 35 | # trace (a 64-bit integer) is the root ID of the tree that contains all of 36 | # the spans related to this one. 37 | trace = 0 38 | 39 | # span (a 64-bit integer) is an ID that probabilistically uniquely 40 | # identifies this span. 41 | span = 0 42 | 43 | # parent (a 64-bit integer) is the ID of the parent span, if any. 44 | parent = 0 45 | 46 | def __init__(self, root=False, parent=None): 47 | if root: 48 | self.trace = _generateID() 49 | self.span = _generateID() 50 | elif parent: 51 | self.trace = parent.trace 52 | self.span = _generateID() 53 | self.parent = parent.span 54 | 55 | # __hexStr returns a hex string for the integer i. It is zero-padded 56 | # appropriately. 57 | def __hexStr(self, i): 58 | h = format(i, 'x') 59 | return h.zfill(len(h) + len(h) % 2) 60 | 61 | # __str__ returns the SpanID formatted as a string in the form of hex ID's 62 | # separated by slashes (trace/span/parent format). 63 | def __str__(self): 64 | if self.parent == 0: 65 | ids = (self.trace, self.span) 66 | else: 67 | ids = (self.trace, self.span, self.parent) 68 | return "/".join(self.__hexStr(x) for x in ids) 69 | 70 | # An Annotation is an arbitrary key-value property on a span. 71 | class Annotation: 72 | # key is the annotation's key. 73 | key = "" 74 | 75 | # value is the annotation's value, which may be either human or 76 | # machine readable, depending on the schema of the event that 77 | # generated it. 78 | value = "" 79 | 80 | def __init__(self, key, value): 81 | self.key = key 82 | self.value = value 83 | 84 | -------------------------------------------------------------------------------- /python/appdash/twcollector.py: -------------------------------------------------------------------------------- 1 | from twisted.internet.protocol import Protocol, ReconnectingClientFactory 2 | from twisted.internet import task 3 | from sys import stdout 4 | 5 | import encode 6 | 7 | class CollectorProtocol(Protocol): 8 | # CollectorProtocol is a Twisted implementation of appdash's protobuf-based 9 | # collector protocol. 10 | 11 | # writeMsg writes the delimited-protobuf message out to the protocol's 12 | # transport. See encodeMsg for details. 13 | def writeMsg(self, msg): 14 | self.transport.write(encode._msg(msg)) 15 | 16 | def connectionMade(self): 17 | self._factory._log('connected!') 18 | 19 | def connectionLost(self, reason): 20 | self._factory._log("disconnected.", reason.getErrorMessage()) 21 | 22 | def dataReceived(self, data): 23 | self._factory._log('got', len(data), 'bytes of unexpected data from server.') 24 | 25 | 26 | class RemoteCollectorFactory(ReconnectingClientFactory): 27 | # RemoteCollectorFactory is a Twisted factory for remote collectors, which 28 | # collect spans and their annotations, sending them to a remote Go appdash 29 | # server for collection. After collection they can be viewed in appdash's 30 | # web user interface. 31 | 32 | _reactor = None 33 | _debug = False 34 | _remote = None 35 | _pending = [] 36 | 37 | def __init__(self, reactor, debug=False): 38 | self._reactor = reactor 39 | self._debug = debug 40 | 41 | def _log(self, *args): 42 | if self._debug: 43 | print "appdash: %s" % (" ".join(args)) 44 | 45 | # collect collects annotations for the given spanID. 46 | # 47 | # The annotations will be flushed out at a later time, when a connection 48 | # to the remote server has been made. 49 | def collect(self, spanID, *annotations): 50 | self._log("collecting", str(len(annotations)), "annotations for", str(spanID)) 51 | 52 | # Append the collection packet to the pending queue. 53 | self._pending.append(encode._collect(spanID, *annotations)) 54 | 55 | # __flush is called internally after either a new collection has occured, or 56 | # after connection has been made with the remote server. It writes all the 57 | # pending messages out to the remote. 58 | def __flush(self): 59 | if len(self._pending) == 0: 60 | return 61 | 62 | self._log("flushing", str(len(self._pending)), "messages") 63 | for p in self._pending: 64 | self._remote.writeMsg(p) 65 | self._log("done.") 66 | self._pending = [] 67 | 68 | def __startFlushing(self): 69 | # Run the flush method every 1/2 second. 70 | l = task.LoopingCall(self.__flush) 71 | l.start(1/2) 72 | 73 | def startedConnecting(self, connector): 74 | self._log('connecting..') 75 | 76 | def buildProtocol(self, addr): 77 | # Reset delay to reconnection -- otherwise it's exponential (which is 78 | # not a good match for us). 79 | self.resetDelay() 80 | 81 | # Create the protocol. 82 | p = CollectorProtocol() 83 | p._factory = self 84 | self._remote = p 85 | self._reactor.callLater(1, self.__startFlushing) 86 | return p 87 | 88 | def clientConnectionFailed(self, connector, reason): 89 | self._log('connection failed:', reason.getErrorMessage()) 90 | ReconnectingClientFactory.clientConnectionFailed(self, connector, reason) 91 | 92 | -------------------------------------------------------------------------------- /python/appdash/varint.py: -------------------------------------------------------------------------------- 1 | # Hack: There is no official EncodeVarint function provided by protobuf. The 2 | # one we use here is technically private. If it is removed or becomes 3 | # troublesome to update -- we'll have to do it ourselves. 4 | # 5 | # See http://code.google.com/p/protobuf/issues/detail?id=226 6 | import google.protobuf.internal.encoder as enc 7 | from google.protobuf.internal.encoder import _EncodeVarint 8 | 9 | # encode encodes the unsigned varint i, and returns the encoded data as 10 | # a str. 11 | def encode(i): 12 | buf = [] 13 | # Note: The signed variant is named "_EncodeSignedVarint". 14 | _EncodeVarint(buf.append, i) 15 | return "".join(buf) 16 | 17 | -------------------------------------------------------------------------------- /python/example_opentracing_socket.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Imports 4 | import time 5 | import appdash 6 | 7 | # Appdash: Socket Collector 8 | from appdash.sockcollector import RemoteCollector 9 | 10 | # Create a remote appdash collector. 11 | collector = RemoteCollector(debug=True) 12 | collector.connect(host="localhost", port=7701) 13 | 14 | # Create a tracer 15 | tracer = appdash.create_new_tracer(collector) 16 | 17 | for i in range(0, 7): 18 | # Generate a few spans with some annotations. 19 | span = None 20 | # Name the span. 21 | if i == 0: 22 | span = tracer.start_span("Request") 23 | else: 24 | span = tracer.start_span("SQL Query") 25 | 26 | span.set_tag("query", "SELECT * FROM table_name;") 27 | span.set_tag("foo", "bar") 28 | 29 | if i % 2 == 0: 30 | span.log_event("Hello world!") 31 | 32 | child_span = tracer.start_span("child", child_of=span) 33 | child_span.finish() 34 | 35 | span.finish(finish_time=time.time()+2) 36 | 37 | # Close the collector's connection. 38 | collector.close() 39 | -------------------------------------------------------------------------------- /python/example_socket.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Imports 4 | import time 5 | import appdash 6 | 7 | # Appdash: Socket Collector 8 | from appdash.sockcollector import RemoteCollector 9 | 10 | # Create a remote appdash collector. 11 | collector = RemoteCollector(debug=True) 12 | collector.connect(host="localhost", port=7701) 13 | 14 | # Create a trace. 15 | trace = appdash.SpanID(root=True) 16 | 17 | # Generate a few spans with some annotations. 18 | span = trace 19 | for i in range(0, 7): 20 | # Name the span. 21 | if i == 0: 22 | collector.collect(span, *appdash.MarshalEvent(appdash.SpanNameEvent("Request"))) 23 | else: 24 | collector.collect(span, *appdash.MarshalEvent(appdash.SpanNameEvent("SQL Query"))) 25 | 26 | # Marshal some events into annotations and collect them. 27 | sendTime = time.time() 28 | recvTime = sendTime + 2 29 | collector.collect(span, *appdash.MarshalEvent(appdash.SQLEvent( 30 | "SELECT * FROM table_name;", 31 | sendTime, 32 | recv = recvTime, # optional: default is current time. 33 | tag = "foobar", # optional: user-specific tag, useful for e.g. filtering. 34 | ))) 35 | 36 | if i % 2 == 0: 37 | collector.collect(span, *appdash.MarshalEvent(appdash.LogEvent("Hello world!"))) 38 | 39 | # Create a new child span whose parent is the last span we created. 40 | span = appdash.SpanID(parent=span) 41 | 42 | # Close the collector's connection. 43 | collector.close() 44 | -------------------------------------------------------------------------------- /python/example_twisted.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Imports 4 | from twisted.internet import reactor 5 | import time 6 | import appdash 7 | 8 | # Appdash: Twisted Collector 9 | from appdash.twcollector import RemoteCollectorFactory 10 | 11 | # Create a remote appdash collector. 12 | collector = RemoteCollectorFactory(reactor, debug=True) 13 | 14 | # Create a trace. 15 | trace = appdash.SpanID(root=True) 16 | 17 | # Generate a few spans with some annotations. 18 | span = trace 19 | for i in range(0, 7): 20 | # Name the span. 21 | if i == 0: 22 | collector.collect(span, *appdash.MarshalEvent(appdash.SpanNameEvent("Request"))) 23 | else: 24 | collector.collect(span, *appdash.MarshalEvent(appdash.SpanNameEvent("SQL Query"))) 25 | 26 | # Marshal some events into annotations and collect them. 27 | sendTime = time.time() 28 | recvTime = sendTime + 2 29 | collector.collect(span, *appdash.MarshalEvent(appdash.SQLEvent( 30 | "SELECT * FROM table_name;", 31 | sendTime, 32 | recv = recvTime, # optional: default is current time. 33 | tag = "foobar", # optional: user-specific tag, useful for e.g. filtering. 34 | ))) 35 | 36 | if i % 2 == 0: 37 | collector.collect(span, *appdash.MarshalEvent(appdash.LogEvent("Hello world!"))) 38 | 39 | # Create a new child span whose parent is the last span we created. 40 | span = appdash.SpanID(parent=span) 41 | 42 | # Have Twisted perform the connection and run. 43 | reactor.connectTCP("", 7701, collector) 44 | reactor.run() 45 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | setup( 6 | name = 'Appdash', 7 | version = '1.0', 8 | description = 'Appdash Python Integration', 9 | author = 'Sourcegraph', 10 | author_email = 'hi@sourcegraph.com', 11 | url = 'https://sourcegraph.com/sourcegraph/appdash', 12 | packages = ['appdash'], 13 | install_requires = ['basictracer'], 14 | ) 15 | -------------------------------------------------------------------------------- /recorder.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var ( 12 | errMultipleFinishCalls = errors.New("multiple Recorder.Finish calls") 13 | ) 14 | 15 | // A Recorder is associated with a span and records annotations on the 16 | // span by sending them to a collector. 17 | type Recorder struct { 18 | // Logger, if non-nil, causes errors to be written to this logger directly 19 | // instead of being manually checked via the Error method. 20 | Logger *log.Logger 21 | 22 | SpanID // the span ID that annotations are about 23 | annotations []Annotation // SpanID's annotations to be collected 24 | finished bool // finished is whether Recorder.Finish was called 25 | 26 | collector Collector // the collector to send to 27 | 28 | errors []error // errors since the last call to Errors 29 | errorsMu sync.Mutex // protects errors 30 | } 31 | 32 | // NewRecorder creates a new recorder for the given span and 33 | // collector. If c is nil, NewRecorder panics. 34 | func NewRecorder(span SpanID, c Collector) *Recorder { 35 | if c == nil { 36 | panic("Collector is nil") 37 | } 38 | return &Recorder{ 39 | SpanID: span, 40 | collector: c, 41 | } 42 | } 43 | 44 | // Child creates a new Recorder with the same collector and a new 45 | // child SpanID whose parent is this recorder's SpanID. 46 | func (r *Recorder) Child() *Recorder { 47 | return NewRecorder(NewSpanID(r.SpanID), r.collector) 48 | } 49 | 50 | // Name sets the name of this span. 51 | func (r *Recorder) Name(name string) { 52 | r.Event(SpanNameEvent{name}) 53 | } 54 | 55 | // Msg records a Msg event (an event with a human-readable message) on 56 | // the span. 57 | func (r *Recorder) Msg(msg string) { 58 | r.Event(Msg(msg)) 59 | } 60 | 61 | // Log records a Log event (an event with the current timestamp and a 62 | // human-readable message) on the span. 63 | func (r *Recorder) Log(msg string) { 64 | r.Event(Log(msg)) 65 | } 66 | 67 | // LogWithTimestamp records a Log event with an explicit timestamp 68 | func (r *Recorder) LogWithTimestamp(msg string, timestamp time.Time) { 69 | r.Event(LogWithTimestamp(msg, timestamp)) 70 | } 71 | 72 | // Event records any event that implements the Event, TimespanEvent, or 73 | // TimestampedEvent interfaces. 74 | func (r *Recorder) Event(e Event) { 75 | as, err := MarshalEvent(e) 76 | if err != nil { 77 | r.error("Event", err) 78 | return 79 | } 80 | r.annotations = append(r.annotations, as...) 81 | } 82 | 83 | // Finish finishes recording and saves the recorded information to the 84 | // underlying collector. If Finish is not called, then no data will be written 85 | // to the underlying collector. 86 | // Finish must be called once, otherwise r.error is called, this constraint 87 | // ensures that collector is called once per Recorder, in order to avoid 88 | // for performance reasons extra operations(span look up & span's annotations update) 89 | // within the collector. 90 | func (r *Recorder) Finish() { 91 | if r.finished { 92 | r.error("Finish", errMultipleFinishCalls) 93 | return 94 | } 95 | r.finished = true 96 | r.Annotation(r.annotations...) 97 | } 98 | 99 | // Annotation records raw annotations on the span. 100 | func (r *Recorder) Annotation(as ...Annotation) { 101 | if err := r.failsafeAnnotation(as...); err != nil { 102 | r.error("Annotation", err) 103 | } 104 | } 105 | 106 | // Annotation records raw annotations on the span. 107 | func (r *Recorder) failsafeAnnotation(as ...Annotation) error { 108 | return r.collector.Collect(r.SpanID, as...) 109 | } 110 | 111 | // Errors returns all errors encountered by the Recorder since the 112 | // last call to Errors. After calling Errors, the Recorder's list of 113 | // errors is emptied. 114 | func (r *Recorder) Errors() []error { 115 | r.errorsMu.Lock() 116 | errs := r.errors 117 | r.errors = nil 118 | r.errorsMu.Unlock() 119 | return errs 120 | } 121 | 122 | func (r *Recorder) error(method string, err error) { 123 | logMsg := fmt.Sprintf("Recorder.%s error: %s", method, err) 124 | as, _ := MarshalEvent(Log(logMsg)) 125 | r.failsafeAnnotation(as...) 126 | 127 | // If we have a logger, we're not doing manual error checking but rather 128 | // just logging all errors. 129 | if r.Logger != nil { 130 | r.Logger.Println(logMsg) 131 | return 132 | } 133 | 134 | r.errorsMu.Lock() 135 | r.errors = append(r.errors, err) 136 | r.errorsMu.Unlock() 137 | } 138 | -------------------------------------------------------------------------------- /recorder_test.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "reflect" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestRecorder(t *testing.T) { 13 | id := SpanID{1, 2, 3} 14 | 15 | calledCollect := 0 16 | var anns Annotations 17 | c := collectorFunc(func(spanID SpanID, as ...Annotation) error { 18 | calledCollect++ 19 | if spanID != id { 20 | t.Errorf("Collect: got spanID arg %v, want %v", spanID, id) 21 | } 22 | anns = append(anns, as...) 23 | return nil 24 | }) 25 | 26 | r := NewRecorder(id, c) 27 | 28 | r.Msg("msg") 29 | r.Name("name") 30 | 31 | if calledCollect != 0 { 32 | t.Errorf("got calledCollect %d, want 0", calledCollect) 33 | } 34 | 35 | r.Finish() 36 | 37 | if diff := diffAnnotationsFromEvent(anns, Msg("msg")); len(diff) > 0 { 38 | t.Errorf("got diff annotations for Msg event:\n%s", strings.Join(diff, "\n")) 39 | } 40 | 41 | if diff := diffAnnotationsFromEvent(anns, SpanNameEvent{"name"}); len(diff) > 0 { 42 | t.Errorf("got diff annotations for SpanNameEvent:\n%s", strings.Join(diff, "\n")) 43 | } 44 | } 45 | 46 | func TestRecorder_Errors(t *testing.T) { 47 | collectErr := errors.New("Collect error") 48 | calledCollect := 0 49 | c := collectorFunc(func(spanID SpanID, as ...Annotation) error { 50 | calledCollect++ 51 | return collectErr 52 | }) 53 | 54 | r := NewRecorder(SpanID{}, c) 55 | 56 | r.Msg("msg") 57 | if calledCollect != 0 { 58 | t.Errorf("got calledCollect %d, want 0", calledCollect) 59 | } 60 | r.Finish() 61 | errs := r.Errors() 62 | if want := []error{collectErr}; !reflect.DeepEqual(errs, want) { 63 | t.Errorf("got errors %v, want %v", errs, want) 64 | } 65 | 66 | if errs := r.Errors(); len(errs) != 0 { 67 | t.Errorf("got len(errs) == %d, want 0 (after call to Errors)", len(errs)) 68 | } 69 | 70 | r.Finish() 71 | if errs := r.Errors(); len(errs) != 1 && errs[0] != errMultipleFinishCalls { 72 | t.Errorf("got %d errors, want 1", len(errs)) 73 | } 74 | } 75 | 76 | func diffAnnotationsFromEvent(anns Annotations, e Event) (diff []string) { 77 | eventAnns, err := MarshalEvent(e) 78 | if err != nil { 79 | panic(err) 80 | } 81 | 82 | matchesEventAnns := map[string]bool{} 83 | for _, ea := range eventAnns { 84 | for _, a := range anns { 85 | if ea.Key == a.Key && bytes.Equal(ea.Value, a.Value) { 86 | matchesEventAnns[ea.Key] = true 87 | } 88 | } 89 | } 90 | 91 | for _, ea := range eventAnns { 92 | if !matchesEventAnns[ea.Key] { 93 | diff = append(diff, fmt.Sprintf("key %s: %q != %q", ea.Key, ea.Value, anns.get(ea.Key))) 94 | } 95 | } 96 | return diff 97 | } 98 | -------------------------------------------------------------------------------- /span.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "strings" 9 | 10 | "sourcegraph.com/sourcegraph/appdash/internal/wire" 11 | ) 12 | 13 | // A SpanID refers to a single span. 14 | type SpanID struct { 15 | // Trace is the root ID of the tree that contains all of the spans 16 | // related to this one. 17 | Trace ID 18 | 19 | // Span is an ID that probabilistically uniquely identifies this 20 | // span. 21 | Span ID 22 | 23 | // Parent is the ID of the parent span, if any. 24 | Parent ID 25 | } 26 | 27 | var ( 28 | // ErrBadSpanID is returned when the span ID cannot be parsed. 29 | ErrBadSpanID = errors.New("bad span ID") 30 | ) 31 | 32 | // String returns the SpanID as a slash-separated, set of hex-encoded 33 | // parameters (root, ID, parent). If the SpanID has no parent, that value is 34 | // elided. 35 | func (id SpanID) String() string { 36 | if id.Parent == 0 { 37 | return fmt.Sprintf("%s%s%s", id.Trace, SpanIDDelimiter, id.Span) 38 | } 39 | return fmt.Sprintf( 40 | "%s%s%s%s%s", 41 | id.Trace, 42 | SpanIDDelimiter, 43 | id.Span, 44 | SpanIDDelimiter, 45 | id.Parent, 46 | ) 47 | } 48 | 49 | // Format formats according to a format specifier and returns the 50 | // resulting string. The receiver's string representation is the first 51 | // argument. 52 | func (id SpanID) Format(s string, args ...interface{}) string { 53 | args = append([]interface{}{id.String()}, args...) 54 | return fmt.Sprintf(s, args...) 55 | } 56 | 57 | // IsRoot returns whether id is the root ID of a trace. 58 | func (id SpanID) IsRoot() bool { 59 | return id.Parent == 0 60 | } 61 | 62 | // wire returns the span ID as it's protobuf definition. 63 | func (id SpanID) wire() *wire.CollectPacket_SpanID { 64 | return &wire.CollectPacket_SpanID{ 65 | Trace: (*uint64)(&id.Trace), 66 | Span: (*uint64)(&id.Span), 67 | Parent: (*uint64)(&id.Parent), 68 | } 69 | } 70 | 71 | // spanIDFromWire returns a SpanID from it's protobuf definition. 72 | func spanIDFromWire(w *wire.CollectPacket_SpanID) SpanID { 73 | return SpanID{ 74 | Trace: ID(*w.Trace), 75 | Span: ID(*w.Span), 76 | Parent: ID(*w.Parent), 77 | } 78 | } 79 | 80 | // NewRootSpanID generates a new span ID for a root span. This should 81 | // only be used to generate entries for spans caused exclusively by 82 | // spans which are outside of your system as a whole (e.g., a root 83 | // span for the first time you see a user request). 84 | func NewRootSpanID() SpanID { 85 | return SpanID{ 86 | Trace: generateID(), 87 | Span: generateID(), 88 | } 89 | } 90 | 91 | // NewSpanID returns a new ID for an span which is the child of the 92 | // given parent ID. This should be used to track causal relationships 93 | // between spans. 94 | func NewSpanID(parent SpanID) SpanID { 95 | return SpanID{ 96 | Trace: parent.Trace, 97 | Span: generateID(), 98 | Parent: parent.Span, 99 | } 100 | } 101 | 102 | const ( 103 | // SpanIDDelimiter is the delimiter used to concatenate an 104 | // SpanID's components. 105 | SpanIDDelimiter = "/" 106 | ) 107 | 108 | // ParseSpanID parses the given string as a slash-separated set of parameters. 109 | func ParseSpanID(s string) (*SpanID, error) { 110 | parts := strings.Split(s, SpanIDDelimiter) 111 | if len(parts) != 2 && len(parts) != 3 { 112 | return nil, ErrBadSpanID 113 | } 114 | root, err := ParseID(parts[0]) 115 | if err != nil { 116 | return nil, ErrBadSpanID 117 | } 118 | id, err := ParseID(parts[1]) 119 | if err != nil { 120 | return nil, ErrBadSpanID 121 | } 122 | var parent ID 123 | if len(parts) == 3 { 124 | i, err := ParseID(parts[2]) 125 | if err != nil { 126 | return nil, ErrBadSpanID 127 | } 128 | parent = i 129 | } 130 | return &SpanID{ 131 | Trace: root, 132 | Span: id, 133 | Parent: parent, 134 | }, nil 135 | } 136 | 137 | // Span is a span ID and its annotations. 138 | type Span struct { 139 | // ID probabilistically uniquely identifies this span. 140 | ID SpanID 141 | 142 | Annotations 143 | } 144 | 145 | // String returns the Span as a formatted string. 146 | func (s *Span) String() string { 147 | b, err := json.MarshalIndent(s, "", " ") 148 | if err != nil { 149 | panic(err) 150 | } 151 | return string(b) 152 | } 153 | 154 | // Name returns a span's name if it has a name annotation, and "" 155 | // otherwise. 156 | func (s *Span) Name() string { 157 | for _, ann := range s.Annotations { 158 | if ann.Key == "Name" { 159 | return string(ann.Value) 160 | } 161 | } 162 | return "" 163 | } 164 | 165 | // Annotations is a list of annotations (on a span). 166 | type Annotations []Annotation 167 | 168 | // An Annotation is an arbitrary key-value property on a span. 169 | type Annotation struct { 170 | // Key is the annotation's key. 171 | Key string 172 | 173 | // Value is the annotation's value, which may be either human or 174 | // machine readable, depending on the schema of the event that 175 | // generated it. 176 | Value []byte 177 | } 178 | 179 | // Important determines if this annotation's key is considered important to any 180 | // of the registered event types. 181 | func (a Annotation) Important() bool { 182 | for _, ev := range registeredEvents { 183 | i, ok := ev.(ImportantEvent) 184 | if !ok { 185 | continue 186 | } 187 | for _, k := range i.Important() { 188 | if a.Key == k { 189 | return true 190 | } 191 | } 192 | } 193 | return false 194 | } 195 | 196 | // String returns a formatted list of annotations. 197 | func (as Annotations) String() string { 198 | var buf bytes.Buffer 199 | for _, a := range as { 200 | fmt.Fprintf(&buf, "%s=%q\n", a.Key, a.Value) 201 | } 202 | return buf.String() 203 | } 204 | 205 | // schemas returns a list of schema types in the annotations. 206 | func (as Annotations) schemas() []string { 207 | var schemas []string 208 | for _, a := range as { 209 | if strings.HasPrefix(a.Key, SchemaPrefix) { 210 | schemas = append(schemas, a.Key[len(SchemaPrefix):]) 211 | } 212 | } 213 | return schemas 214 | } 215 | 216 | // get gets the value of the first annotation with the given key, or 217 | // nil if none exists. There may be multiple annotations with the key; 218 | // only the first's value is returned. 219 | func (as Annotations) get(key string) []byte { 220 | for _, a := range as { 221 | if a.Key == key { 222 | return a.Value 223 | } 224 | } 225 | return nil 226 | } 227 | 228 | // StringMap returns the annotations as a key-value map. Only one 229 | // annotation for a key appears in the map, and it is chosen 230 | // arbitrarily among the annotations with the same key. 231 | func (as Annotations) StringMap() map[string]string { 232 | m := make(map[string]string, len(as)) 233 | for _, a := range as { 234 | m[a.Key] = string(a.Value) 235 | } 236 | return m 237 | } 238 | 239 | // wire returns the set of annotations as their protobuf definitions. 240 | func (as Annotations) wire() (w []*wire.CollectPacket_Annotation) { 241 | for _, a := range as { 242 | // Important: Make a copy of a that we can retain a pointer to that 243 | // doesn't change after each iteration. Otherwise all wire annotations 244 | // would have the same key. 245 | cpy := a 246 | w = append(w, &wire.CollectPacket_Annotation{ 247 | Key: &cpy.Key, 248 | Value: cpy.Value, 249 | }) 250 | } 251 | return 252 | } 253 | 254 | // annotationsFromWire returns Annotations from their protobuf definitions. 255 | func annotationsFromWire(as []*wire.CollectPacket_Annotation) Annotations { 256 | var w Annotations 257 | for _, a := range as { 258 | w = append(w, Annotation{ 259 | Key: *a.Key, 260 | Value: a.Value, 261 | }) 262 | } 263 | return w 264 | } 265 | -------------------------------------------------------------------------------- /span_test.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func TestNewRootSpanID(t *testing.T) { 10 | id := NewRootSpanID() 11 | if id.Parent != 0 { 12 | t.Errorf("unexpected parent: %+v", id) 13 | } 14 | if id.Span == 0 { 15 | t.Errorf("zero Span: %+v", id) 16 | } 17 | if id.Trace == 0 { 18 | t.Errorf("zero root: %+v", id) 19 | } 20 | if id.Trace == id.Span { 21 | t.Errorf("duplicate IDs: %+v", id) 22 | } 23 | } 24 | 25 | func TestNewSpanID(t *testing.T) { 26 | root := NewRootSpanID() 27 | id := NewSpanID(root) 28 | if id.Parent != root.Span { 29 | t.Errorf("unexpected parent: %+v", id) 30 | } 31 | if id.Span == 0 { 32 | t.Errorf("zero Span: %+v", id) 33 | } 34 | if id.Trace != root.Trace { 35 | t.Errorf("mismatched root: %+v", id) 36 | } 37 | } 38 | 39 | func TestSpanIDString(t *testing.T) { 40 | id := SpanID{ 41 | Trace: 100, 42 | Span: 300, 43 | } 44 | got := id.String() 45 | want := "0000000000000064/000000000000012c" 46 | if got != want { 47 | t.Errorf("got %#v, want %#v", got, want) 48 | } 49 | } 50 | 51 | func TestSpanIDStringWithParent(t *testing.T) { 52 | id := SpanID{ 53 | Trace: 100, 54 | Parent: 200, 55 | Span: 300, 56 | } 57 | actual := id.String() 58 | expected := "0000000000000064/000000000000012c/00000000000000c8" 59 | if actual != expected { 60 | t.Errorf("Was %#v, but expected %#v", actual, expected) 61 | } 62 | } 63 | 64 | func TestSpanIDFormat(t *testing.T) { 65 | id := SpanID{ 66 | Trace: 100, 67 | Span: 300, 68 | } 69 | got := id.Format("/* %s */ %s", "SELECT 1") 70 | want := "/* 0000000000000064/000000000000012c */ SELECT 1" 71 | if got != want { 72 | t.Errorf("got %#v, want %#v", got, want) 73 | } 74 | } 75 | 76 | func ExampleSpanID_Format() { 77 | // Assume we're connected to a database. 78 | var ( 79 | event SpanID 80 | db *sql.DB 81 | userID int 82 | ) 83 | // This passes the root ID and the parent event ID to the database, which 84 | // allows us to correlate, for example, slow queries with the web requests 85 | // which caused them. 86 | query := event.Format(`/* %s/%s */ %s`, `SELECT email FROM users WHERE id = ?`) 87 | r := db.QueryRow(query, userID) 88 | if r == nil { 89 | panic("user not found") 90 | } 91 | var email string 92 | if err := r.Scan(&email); err != nil { 93 | panic("couldn't read email") 94 | } 95 | fmt.Printf("User's email: %s\n", email) 96 | } 97 | 98 | func TestParseSpanID(t *testing.T) { 99 | id, err := ParseSpanID("0000000000000064/000000000000012c") 100 | if err != nil { 101 | t.Fatal(err) 102 | } 103 | if id.Trace != 100 || id.Span != 300 { 104 | t.Errorf("unexpected ID: %+v", id) 105 | } 106 | } 107 | 108 | func TestParseSpanIDWithParent(t *testing.T) { 109 | id, err := ParseSpanID("0000000000000064/000000000000012c/0000000000000096") 110 | if err != nil { 111 | t.Fatalf("unexpected error: %v", err) 112 | } 113 | if id.Trace != 100 || id.Parent != 150 || id.Span != 300 { 114 | t.Errorf("unexpected event ID: %+v", id) 115 | } 116 | } 117 | 118 | func TestParseSpanIDMalformed(t *testing.T) { 119 | id, err := ParseSpanID(`0000000000000064000000000000012c`) 120 | if id != nil { 121 | t.Errorf("unexpected ID: %+v", id) 122 | } 123 | if err != ErrBadSpanID { 124 | t.Error(err) 125 | } 126 | } 127 | 128 | func TestParseSpanIDBadTrace(t *testing.T) { 129 | id, err := ParseSpanID("0000000000g000064/000000000000012c") 130 | if id != nil { 131 | t.Errorf("unexpected ID: %+v", id) 132 | } 133 | if err != ErrBadSpanID { 134 | t.Error(err) 135 | } 136 | } 137 | 138 | func TestParseSpanIDBadID(t *testing.T) { 139 | id, err := ParseSpanID("0000000000000064/0000000000g00012c") 140 | if id != nil { 141 | t.Errorf("unexpected ID: %+v", id) 142 | } 143 | if err != ErrBadSpanID { 144 | t.Error(err) 145 | } 146 | } 147 | 148 | func TestParseSpanIDBadParent(t *testing.T) { 149 | id, err := ParseSpanID("0000000000000064/000000000000012c/00000000000g0096") 150 | if id != nil { 151 | t.Errorf("unexpected event ID: %+v", id) 152 | } 153 | if err != ErrBadSpanID { 154 | t.Errorf("unexpected error: %v", err) 155 | } 156 | } 157 | 158 | func TestSpan_Name(t *testing.T) { 159 | namedSpan := &Span{Annotations: Annotations{{Key: "Name", Value: []byte("foo")}}} 160 | if want := "foo"; namedSpan.Name() != want { 161 | t.Errorf("got Name %q, want %q", namedSpan.Name(), want) 162 | } 163 | 164 | noNameSpan := &Span{} 165 | if want := ""; noNameSpan.Name() != want { 166 | t.Errorf("got Name %q, want %q", noNameSpan.Name(), want) 167 | } 168 | } 169 | 170 | type annotations Annotations 171 | 172 | func (a annotations) Len() int { return len(a) } 173 | func (a annotations) Less(i, j int) bool { return a[i].Key < a[j].Key } 174 | func (a annotations) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 175 | 176 | func BenchmarkNewRootSpanID(b *testing.B) { 177 | for i := 0; i < b.N; i++ { 178 | NewRootSpanID() 179 | } 180 | } 181 | 182 | func BenchmarkNewSpanID(b *testing.B) { 183 | root := NewRootSpanID() 184 | for i := 0; i < b.N; i++ { 185 | NewSpanID(root) 186 | } 187 | } 188 | 189 | func BenchmarkSpanIDString(b *testing.B) { 190 | id := SpanID{ 191 | Trace: 100, 192 | Parent: 200, 193 | Span: 300, 194 | } 195 | for i := 0; i < b.N; i++ { 196 | _ = id.String() 197 | } 198 | } 199 | 200 | func BenchmarkParseSpanID(b *testing.B) { 201 | for i := 0; i < b.N; i++ { 202 | _, err := ParseSpanID("0000000000000064/000000000000012c") 203 | if err != nil { 204 | b.Fatal(err) 205 | } 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /sqltrace/sql.go: -------------------------------------------------------------------------------- 1 | // Package sqltrace implements utility types for tracing SQL queries. 2 | package sqltrace 3 | 4 | import ( 5 | "time" 6 | 7 | "sourcegraph.com/sourcegraph/appdash" 8 | ) 9 | 10 | // SQLEvent is an SQL query event for use with appdash. It's primary function 11 | // is to measure the time between when the query is sent and later received. 12 | type SQLEvent struct { 13 | SQL string 14 | Tag string 15 | ClientSend time.Time 16 | ClientRecv time.Time 17 | } 18 | 19 | // Schema implements the appdash Event interface by returning this event's 20 | // constant schema string, "SQL". 21 | func (SQLEvent) Schema() string { return "SQL" } 22 | 23 | // Important implements the appdash ImportantEvent by returning the SQL and Tag 24 | // keys. 25 | func (SQLEvent) Important() []string { return []string{"SQL", "Tag"} } 26 | 27 | // Start implements the appdash TimespanEvent interface by returning the time 28 | // at which the SQL query was sent out. 29 | func (e SQLEvent) Start() time.Time { return e.ClientSend } 30 | 31 | // End implements the appdash TimespanEvent interface by returning the time at 32 | // which the SQL query returned / was received. 33 | func (e SQLEvent) End() time.Time { return e.ClientRecv } 34 | 35 | func init() { appdash.RegisterEvent(SQLEvent{}) } 36 | -------------------------------------------------------------------------------- /trace.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // A Trace is a tree of spans. 14 | type Trace struct { 15 | Span // Root span 16 | Sub []*Trace // Children 17 | } 18 | 19 | // String returns the Trace as a formatted string. 20 | func (t *Trace) String() string { 21 | b, err := json.MarshalIndent(t, "", " ") 22 | if err != nil { 23 | panic(err) 24 | } 25 | return string(b) 26 | } 27 | 28 | // FindSpan recursively searches for a span whose Span ID is spanID in 29 | // t and its descendants. If no such span is found, nil is returned. 30 | func (t *Trace) FindSpan(spanID ID) *Trace { 31 | if t.ID.Span == spanID { 32 | return t 33 | } 34 | for _, sub := range t.Sub { 35 | if s := sub.FindSpan(spanID); s != nil { 36 | return s 37 | } 38 | } 39 | return nil 40 | } 41 | 42 | // TreeString returns the Trace as a formatted string that visually 43 | // represents the trace's tree. 44 | func (t *Trace) TreeString() string { 45 | var buf bytes.Buffer 46 | t.treeString(&buf, 0) 47 | return buf.String() 48 | } 49 | 50 | func (t *Trace) TimespanEvent() (TimespanEvent, error) { 51 | var events []Event 52 | if err := UnmarshalEvents(t.Annotations, &events); err != nil { 53 | return timespanEvent{}, err 54 | } 55 | start, end, ok := findTraceTimes(events) 56 | if !ok { 57 | return timespanEvent{}, errors.New("time span event not found") 58 | } 59 | return timespanEvent{S: start, E: end}, nil 60 | } 61 | 62 | func (t *Trace) treeString(w io.Writer, depth int) { 63 | const indent1 = " " 64 | indent := strings.Repeat(indent1, depth) 65 | 66 | if depth == 0 { 67 | fmt.Fprintf(w, "+ Trace %x\n", uint64(t.Span.ID.Trace)) 68 | } else { 69 | if depth == 1 { 70 | fmt.Fprint(w, "|") 71 | } else { 72 | fmt.Fprint(w, "|", indent[len(indent1):]) 73 | } 74 | fmt.Fprintf(w, "%s+ Span %x", strings.Repeat("-", len(indent1)), uint64(t.Span.ID.Span)) 75 | if t.Span.ID.Parent != 0 { 76 | fmt.Fprintf(w, " (parent %x)", uint64(t.Span.ID.Parent)) 77 | } 78 | fmt.Fprintln(w) 79 | } 80 | for _, a := range t.Span.Annotations { 81 | if depth == 0 { 82 | fmt.Fprint(w, "| ") 83 | } else { 84 | fmt.Fprint(w, "|", indent[1:], " | ") 85 | } 86 | fmt.Fprintf(w, "%s = %s\n", a.Key, a.Value) 87 | } 88 | for _, sub := range t.Sub { 89 | sub.treeString(w, depth+1) 90 | } 91 | } 92 | 93 | // findTraceTimes finds the minimum and maximum timespan event times for the 94 | // given set of events, or returns ok == false if there are no such events. 95 | func findTraceTimes(events []Event) (start, end time.Time, _ bool) { 96 | // Find the start and end time of the trace. 97 | for _, e := range events { 98 | e, ok := e.(TimespanEvent) 99 | if !ok { 100 | continue 101 | } 102 | if start.IsZero() { 103 | start = e.Start() 104 | end = e.End() 105 | continue 106 | } 107 | if v := e.Start(); v.UnixNano() < start.UnixNano() { 108 | start = v 109 | } 110 | if v := e.End(); v.UnixNano() > end.UnixNano() { 111 | end = v 112 | } 113 | } 114 | return start, end, !start.IsZero() 115 | } 116 | 117 | type tracesByIDSpan []*Trace 118 | 119 | func (t tracesByIDSpan) Len() int { return len(t) } 120 | func (t tracesByIDSpan) Less(i, j int) bool { return t[i].Span.ID.Span < t[j].Span.ID.Span } 121 | func (t tracesByIDSpan) Swap(i, j int) { t[i], t[j] = t[j], t[i] } 122 | -------------------------------------------------------------------------------- /trace_test.go: -------------------------------------------------------------------------------- 1 | package appdash 2 | 3 | import "testing" 4 | 5 | func TestTrace_TreeString(t *testing.T) { 6 | t.Skip("TODO") 7 | 8 | x := &Trace{ 9 | Span: Span{ 10 | ID: SpanID{1, 1, 0}, 11 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 12 | }, 13 | Sub: []*Trace{ 14 | { 15 | Span: Span{ 16 | ID: SpanID{1, 2, 1}, 17 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 18 | }, 19 | Sub: []*Trace{ 20 | { 21 | Span: Span{ 22 | ID: SpanID{1, 3, 2}, 23 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 24 | }, 25 | }, 26 | }, 27 | }, 28 | { 29 | Span: Span{ 30 | ID: SpanID{1, 4, 1}, 31 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 32 | }, 33 | Sub: []*Trace{ 34 | { 35 | Span: Span{ 36 | ID: SpanID{1, 5, 4}, 37 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 38 | }, 39 | }, 40 | { 41 | Span: Span{ 42 | ID: SpanID{1, 6, 4}, 43 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 44 | }, 45 | }, 46 | }, 47 | }, 48 | }, 49 | } 50 | 51 | want := `` 52 | 53 | if ts := x.TreeString(); ts != want { 54 | t.Errorf("got TreeString\n%s\n\nwant TreeString\n%s", ts, want) 55 | } 56 | } 57 | 58 | func TestTrace_FindSpan(t *testing.T) { 59 | x := &Trace{ 60 | Span: Span{ 61 | ID: SpanID{1, 1, 0}, 62 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 63 | }, 64 | Sub: []*Trace{ 65 | { 66 | Span: Span{ 67 | ID: SpanID{1, 2, 1}, 68 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 69 | }, 70 | Sub: []*Trace{ 71 | { 72 | Span: Span{ 73 | ID: SpanID{1, 3, 2}, 74 | Annotations: []Annotation{{Key: "k", Value: []byte("v")}}, 75 | }, 76 | }, 77 | }, 78 | }, 79 | }, 80 | } 81 | 82 | testSpanIDs := []ID{1, 2, 3} 83 | for _, id := range testSpanIDs { 84 | span := x.FindSpan(id) 85 | if span == nil { 86 | t.Errorf("%v: got nil, want found", id) 87 | } 88 | if span.Span.ID.Span != id { 89 | t.Errorf("%v: got span ID %v, want %v", id, span.Span.ID.Span, id) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /traceapp/aggregate.go: -------------------------------------------------------------------------------- 1 | package traceapp 2 | 3 | import ( 4 | "fmt" 5 | 6 | "sourcegraph.com/sourcegraph/appdash" 7 | ) 8 | 9 | // aggItem represents a set of traces with the name (label) and their cumulative 10 | // time. 11 | type aggItem struct { 12 | Label string `json:"label"` 13 | Value int64 `json:"value"` 14 | } 15 | 16 | type aggMode int 17 | 18 | const ( 19 | traceOnly aggMode = iota 20 | spanOnly 21 | traceAndSpan 22 | ) 23 | 24 | // parseAggMode parses an aggregation mode: 25 | // 26 | // "trace-only" -> traceOnly 27 | // "span-only" -> spanOnly 28 | // "trace-and-span" -> traceAndSpan 29 | // 30 | func parseAggMode(s string) aggMode { 31 | switch s { 32 | case "trace-only": 33 | return traceOnly 34 | case "span-only": 35 | return spanOnly 36 | case "trace-and-span": 37 | return traceAndSpan 38 | default: 39 | return traceOnly 40 | } 41 | } 42 | 43 | // aggregate aggregates and encodes the given traces as JSON to the given writer. 44 | func (a *App) aggregate(traces []*appdash.Trace, mode aggMode) ([]*aggItem, error) { 45 | aggregated := make(map[string]*aggItem) 46 | 47 | // up updates the aggregated map with the given label and value. 48 | up := func(label string, value int64) { 49 | // Grab the aggregation item for the named trace, or create a new one if it 50 | // does not already exist. 51 | i, ok := aggregated[label] 52 | if !ok { 53 | i = &aggItem{Label: label} 54 | aggregated[label] = i 55 | } 56 | 57 | // Perform aggregation. 58 | i.Value += value 59 | if i.Value == 0 { 60 | i.Value = 1 // Must be positive values or else d3pie won't render. 61 | } 62 | } 63 | 64 | for _, trace := range traces { 65 | // Calculate the cumulative time -- which we can already get through the 66 | // profile view's calculation method. 67 | profiles, childProf, err := a.calcProfile(nil, trace) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | if mode == traceOnly { 73 | up(childProf.Name, childProf.TimeCum) 74 | } else if mode == spanOnly { 75 | for _, spanProf := range profiles[1:] { 76 | up(spanProf.Name, spanProf.Time) 77 | } 78 | } else if mode == traceAndSpan { 79 | for _, spanProf := range profiles[1:] { 80 | up(fmt.Sprintf("%s: %s", childProf.Name, spanProf.Name), spanProf.Time) 81 | } 82 | } 83 | } 84 | 85 | // Form an array (d3pie needs a JSON array, not a map). 86 | list := make([]*aggItem, 0, len(aggregated)) 87 | for _, item := range aggregated { 88 | list = append(list, item) 89 | } 90 | return list, nil 91 | } 92 | -------------------------------------------------------------------------------- /traceapp/dashboard.go: -------------------------------------------------------------------------------- 1 | package traceapp 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // dashboardRow represents a single row in the dashboard. It is encoded to JSON. 15 | type dashboardRow struct { 16 | Name string 17 | Average, Min, Max, StdDev time.Duration 18 | Timespans int 19 | URL string 20 | } 21 | 22 | // serverDashboard serves the dashboard page. 23 | func (a *App) serveDashboard(w http.ResponseWriter, r *http.Request) error { 24 | if a.Aggregator == nil { 25 | w.WriteHeader(http.StatusNotFound) 26 | fmt.Fprintf(w, "Dashboard is disabled.") 27 | return nil 28 | } 29 | 30 | uData, err := a.Router.URLTo(DashboardDataRoute) 31 | if err != nil { 32 | return err 33 | } 34 | 35 | return a.renderTemplate(w, r, "dashboard.html", http.StatusOK, &struct { 36 | TemplateCommon 37 | DataURL string 38 | HaveDashboard bool 39 | }{ 40 | DataURL: uData.String(), 41 | HaveDashboard: a.Aggregator != nil, 42 | }) 43 | } 44 | 45 | // serveDashboardData serves the JSON data requested by the dashboards table. 46 | func (a *App) serveDashboardData(w http.ResponseWriter, r *http.Request) error { 47 | if a.Aggregator == nil { 48 | w.WriteHeader(http.StatusNotFound) 49 | fmt.Fprintf(w, "Dashboard is disabled.") 50 | return nil 51 | } 52 | 53 | // Parse the query for the start & end timeline durations. 54 | var ( 55 | query = r.URL.Query() 56 | start, end time.Duration 57 | ) 58 | if s := query.Get("start"); len(s) > 0 { 59 | v, err := strconv.ParseInt(s, 10, 64) 60 | if err != nil { 61 | return err 62 | } 63 | start = time.Duration(v) * time.Hour 64 | start -= 72 * time.Hour 65 | } 66 | if s := query.Get("end"); len(s) > 0 { 67 | v, err := strconv.ParseInt(s, 10, 64) 68 | if err != nil { 69 | return err 70 | } 71 | end = time.Duration(v) * time.Hour 72 | end -= 72 * time.Hour 73 | } 74 | 75 | results, err := a.Aggregator.Aggregate(start, end) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | // Grab the URL to the traces page. 81 | tracesURL, err := a.Router.URLTo(TracesRoute) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | rows := make([]*dashboardRow, len(results)) 87 | for i, r := range results { 88 | var stringIDs []string 89 | for _, slowest := range r.Slowest { 90 | stringIDs = append(stringIDs, slowest.String()) 91 | } 92 | tracesURL.RawQuery = "show=" + strings.Join(stringIDs, ",") 93 | 94 | rows[i] = &dashboardRow{ 95 | Name: r.RootSpanName, 96 | Average: r.Average / time.Millisecond, 97 | Min: r.Min / time.Millisecond, 98 | Max: r.Max / time.Millisecond, 99 | StdDev: r.StdDev / time.Millisecond, 100 | Timespans: int(r.Samples), 101 | URL: tracesURL.String(), 102 | } 103 | } 104 | 105 | // Encode to JSON. 106 | j, err := json.Marshal(rows) 107 | if err != nil { 108 | return err 109 | } 110 | 111 | // Write out. 112 | _, err = io.Copy(w, bytes.NewReader(j)) 113 | return err 114 | } 115 | -------------------------------------------------------------------------------- /traceapp/handler.go: -------------------------------------------------------------------------------- 1 | package traceapp 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | "runtime/debug" 8 | ) 9 | 10 | type handlerFunc func(http.ResponseWriter, *http.Request) error 11 | 12 | // ServeHTTP implements http.Handler. 13 | func (h handlerFunc) ServeHTTP(w http.ResponseWriter, r *http.Request) { 14 | var rb responseBuffer 15 | 16 | defer func() { 17 | if rv := recover(); rv != nil { 18 | handleError(w, r, fmt.Errorf("handler panic\n\n%s\n\n%s", rv, debug.Stack())) 19 | } 20 | }() 21 | 22 | err := h(&rb, r) 23 | if err != nil { 24 | handleError(w, r, err) 25 | return 26 | } 27 | rb.WriteTo(w) 28 | } 29 | 30 | func handleError(w http.ResponseWriter, r *http.Request, err error) { 31 | log.Printf("%s %s: error: %s", r.Method, r.URL.RequestURI(), err.Error()) 32 | 33 | // Never cache error responses. 34 | w.Header().Set("cache-control", "no-cache, max-age=0") 35 | 36 | http.Error(w, err.Error(), http.StatusInternalServerError) 37 | } 38 | -------------------------------------------------------------------------------- /traceapp/httputil.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // 3 | // Use of this source code is governed by a BSD-style 4 | // license that can be found in the LICENSE file or at 5 | // https://developers.google.com/open-source/licenses/bsd. 6 | 7 | package traceapp 8 | 9 | import ( 10 | "bytes" 11 | "net/http" 12 | "strconv" 13 | ) 14 | 15 | type responseBuffer struct { 16 | buf bytes.Buffer 17 | Status int 18 | header http.Header 19 | } 20 | 21 | func (rb *responseBuffer) Write(p []byte) (int, error) { 22 | return rb.buf.Write(p) 23 | } 24 | 25 | func (rb *responseBuffer) WriteHeader(status int) { 26 | rb.Status = status 27 | } 28 | 29 | func (rb *responseBuffer) Header() http.Header { 30 | if rb.header == nil { 31 | rb.header = make(http.Header) 32 | } 33 | return rb.header 34 | } 35 | 36 | func (rb *responseBuffer) ContentLength() int { 37 | return rb.buf.Len() 38 | } 39 | 40 | func (rb *responseBuffer) WriteTo(w http.ResponseWriter) error { 41 | for k, v := range rb.header { 42 | w.Header()[k] = v 43 | } 44 | if l := rb.ContentLength(); l > 0 { 45 | w.Header().Set("Content-Length", strconv.Itoa(l)) 46 | } 47 | if rb.Status != 0 { 48 | w.WriteHeader(rb.Status) 49 | } 50 | if rb.buf.Len() > 0 { 51 | if _, err := w.Write(rb.buf.Bytes()); err != nil { 52 | return err 53 | } 54 | } 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /traceapp/prof.go: -------------------------------------------------------------------------------- 1 | package traceapp 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io" 7 | "net/url" 8 | "time" 9 | 10 | "sourcegraph.com/sourcegraph/appdash" 11 | ) 12 | 13 | type profile struct { 14 | Name string 15 | URL string 16 | Time, TimeChildren, TimeCum int64 17 | } 18 | 19 | // calcProfile calculates a profile for the given trace and appends it to the 20 | // given buffer (buf), which is then returned (prof). If an error is returned, 21 | // all other returned values are nil. The childProf is literally the *profile 22 | // associated with the given trace (t). 23 | func (a *App) calcProfile(buf []*profile, t *appdash.Trace) (prof []*profile, childProf *profile, err error) { 24 | // Unmarshal the trace's span events. 25 | var events []appdash.Event 26 | if err = appdash.UnmarshalEvents(t.Span.Annotations, &events); err != nil { 27 | return nil, nil, err 28 | } 29 | 30 | // Get the proper URL to the trace view. 31 | var u *url.URL 32 | if t.ID.Parent == 0 { 33 | u, err = a.URLToTrace(t.ID.Trace) 34 | if err != nil { 35 | return nil, nil, err 36 | } 37 | } else { 38 | u, err = a.URLToTraceSpan(t.ID.Trace, t.ID.Span) 39 | if err != nil { 40 | return nil, nil, err 41 | } 42 | } 43 | 44 | // Initialize the span's profile structure. We use either the span's given 45 | // name, or it's ID as a string if it has no given name. 46 | p := &profile{ 47 | Name: t.Span.Name(), 48 | URL: u.String(), 49 | } 50 | if len(p.Name) == 0 { 51 | p.Name = t.Span.ID.Span.String() 52 | } 53 | buf = append(buf, p) 54 | 55 | // Store the time for the largest timespan event the span has. 56 | for _, ev := range events { 57 | ts, ok := ev.(appdash.TimespanEvent) 58 | if !ok { 59 | continue 60 | } 61 | // To match the timeline properly we use floats and round up. 62 | msf := float64(ts.End().Sub(ts.Start())) / float64(time.Millisecond) 63 | ms := int64(msf + 0.5) 64 | if ms > p.Time { 65 | p.Time = ms 66 | } 67 | } 68 | 69 | // TimeChildren is our time + the children's time. 70 | p.TimeChildren = p.Time 71 | 72 | // The cumulative time is our time + all children's time. 73 | p.TimeCum = p.Time 74 | 75 | // Descend recursively into each sub-trace and calculate the profile for 76 | // each child span. 77 | for _, child := range t.Sub { 78 | buf, childProf, err = a.calcProfile(buf, child) 79 | if err != nil { 80 | return nil, nil, err 81 | } 82 | 83 | // Aggregate our direct children's time. 84 | p.TimeChildren += childProf.Time 85 | 86 | // As our child's profile has the cumulative time (which is initially, 87 | // it's self time) -- we can simply aggregate it here and we have our 88 | // trace's cumulative time (i.e. it is effectively recursive). 89 | p.TimeCum += childProf.TimeCum 90 | } 91 | return buf, p, nil 92 | } 93 | 94 | // profile generates and encodes the given trace as JSON to the given writer. 95 | func (a *App) profile(t *appdash.Trace, out io.Writer) error { 96 | // Generate the profile. 97 | prof, _, err := a.calcProfile(nil, t) 98 | if err != nil { 99 | return err 100 | } 101 | 102 | // Encode to JSON. 103 | j, err := json.Marshal(prof) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | // Write out. 109 | _, err = io.Copy(out, bytes.NewReader(j)) 110 | return err 111 | } 112 | -------------------------------------------------------------------------------- /traceapp/router.go: -------------------------------------------------------------------------------- 1 | package traceapp 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | 7 | "github.com/gorilla/mux" 8 | "sourcegraph.com/sourcegraph/appdash" 9 | ) 10 | 11 | // Traceapp's route names. 12 | const ( 13 | RootRoute = "traceapp.root" // route name for root 14 | StaticRoute = "traceapp.static" // route name for static data files 15 | TraceRoute = "traceapp.trace" // route name for a single trace page 16 | TraceSpanRoute = "traceapp.trace.span" // route name for a single trace sub-span page 17 | TraceProfileRoute = "traceapp.trace.profile" // route name for a JSON trace profile 18 | TraceSpanProfileRoute = "traceapp.trace.span.profile" // route name for a JSON trace sub-span profile 19 | TraceUploadRoute = "traceapp.trace.upload" // route name for a JSON trace upload 20 | TracesRoute = "traceapp.traces" // route name for traces page 21 | DashboardRoute = "traceapp.dashboard" // route name for dashboard page 22 | DashboardDataRoute = "traceapp.dashboard.data" // route name for dashboard JSON data 23 | AggregateRoute = "traceapp.aggregate" // route name for aggregate trace view 24 | ) 25 | 26 | // Router is a URL router for traceapp applications. It should be created via 27 | // the NewRouter function. 28 | type Router struct{ r *mux.Router } 29 | 30 | // NewRouter creates a new URL router for a traceapp application. 31 | func NewRouter(base *mux.Router) *Router { 32 | if base == nil { 33 | base = mux.NewRouter() 34 | } 35 | base.Path("/").Methods("GET").Name(RootRoute) 36 | base.PathPrefix("/static/").Methods("GET").Name(StaticRoute) 37 | base.Path("/traces/{Trace}").Methods("GET").Name(TraceRoute) 38 | base.Path("/traces/{Trace}/profile").Methods("GET").Name(TraceProfileRoute) 39 | base.Path("/traces/{Trace}/{Span}/profile").Methods("GET").Name(TraceSpanProfileRoute) 40 | base.Path("/traces/upload").Methods("POST").Name(TraceUploadRoute) 41 | base.Path("/traces/{Trace}/{Span}").Methods("GET").Name(TraceSpanRoute) 42 | base.Path("/traces").Methods("GET").Name(TracesRoute) 43 | base.Path("/dashboard").Methods("GET").Name(DashboardRoute) 44 | base.Path("/dashboard/data").Methods("GET").Name(DashboardDataRoute) 45 | base.Path("/aggregate").Methods("GET").Name(AggregateRoute) 46 | return &Router{base} 47 | } 48 | 49 | // URLTo constructs a URL to a given route. 50 | func (r *Router) URLTo(route string) (*url.URL, error) { 51 | rt := r.r.Get(route) 52 | if rt == nil { 53 | return nil, fmt.Errorf("no such route: %q", route) 54 | } 55 | return rt.URL() 56 | } 57 | 58 | // URLToTrace constructs a URL to a given trace by ID. 59 | func (r *Router) URLToTrace(id appdash.ID) (*url.URL, error) { 60 | return r.r.Get(TraceRoute).URL("Trace", id.String()) 61 | } 62 | 63 | // URLToTraceSpan constructs a URL to a sub-span in a trace. 64 | func (r *Router) URLToTraceSpan(trace, span appdash.ID) (*url.URL, error) { 65 | return r.r.Get(TraceSpanRoute).URL("Trace", trace.String(), "Span", span.String()) 66 | } 67 | 68 | // URLToTraceProfile constructs a URL to a trace's JSON profile. 69 | func (r *Router) URLToTraceProfile(trace appdash.ID) (*url.URL, error) { 70 | return r.r.Get(TraceProfileRoute).URL("Trace", trace.String()) 71 | } 72 | 73 | // URLToTraceSpanProfile constructs a URL to a sub-span's JSON profile in a 74 | // trace. 75 | func (r *Router) URLToTraceSpanProfile(trace, span appdash.ID) (*url.URL, error) { 76 | return r.r.Get(TraceSpanProfileRoute).URL("Trace", trace.String(), "Span", span.String()) 77 | } 78 | -------------------------------------------------------------------------------- /traceapp/tmpl.go: -------------------------------------------------------------------------------- 1 | package traceapp 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | htmpl "html/template" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | 12 | "reflect" 13 | "strconv" 14 | "strings" 15 | 16 | "sourcegraph.com/sourcegraph/appdash" 17 | "sourcegraph.com/sourcegraph/appdash/traceapp/tmpl" 18 | 19 | "github.com/gorilla/mux" 20 | ) 21 | 22 | var ( 23 | // ReloadTemplates is whether to reload html/template templates 24 | // before each request. It is useful during development. 25 | ReloadTemplates = true 26 | ) 27 | 28 | var templates = [][]string{ 29 | {"root.html", "layout.html"}, 30 | {"trace.html", "layout.html"}, 31 | {"traces.html", "layout.html"}, 32 | {"dashboard.html", "layout.html"}, 33 | {"aggregate.html", "layout.html"}, 34 | } 35 | 36 | // TemplateCommon is data that is passed to (and available to) all templates. 37 | type TemplateCommon struct { 38 | CurrentRoute string 39 | CurrentURI *url.URL 40 | BaseURL *url.URL 41 | HaveDashboard bool 42 | } 43 | 44 | func (a *App) renderTemplate(w http.ResponseWriter, r *http.Request, name string, status int, data interface{}) error { 45 | a.tmplLock.Lock() 46 | defer a.tmplLock.Unlock() 47 | 48 | if a.tmpls == nil || ReloadTemplates { 49 | if err := a.parseHTMLTemplates(templates); err != nil { 50 | return err 51 | } 52 | } 53 | 54 | w.WriteHeader(status) 55 | if ct := w.Header().Get("content-type"); ct == "" { 56 | w.Header().Set("Content-Type", "text/html; charset=utf-8") 57 | } 58 | t := a.tmpls[name] 59 | if t == nil { 60 | return fmt.Errorf("Template %s not found", name) 61 | } 62 | 63 | if data != nil { 64 | // Set TemplateCommon values. 65 | reflect.ValueOf(data).Elem().FieldByName("TemplateCommon").Set(reflect.ValueOf(TemplateCommon{ 66 | CurrentRoute: mux.CurrentRoute(r).GetName(), 67 | CurrentURI: r.URL, 68 | BaseURL: a.baseURL, 69 | HaveDashboard: a.Aggregator != nil, 70 | })) 71 | } 72 | 73 | // Write to a buffer to properly catch errors and avoid partial output written to the http.ResponseWriter 74 | var buf bytes.Buffer 75 | err := t.Execute(&buf, data) 76 | if err != nil { 77 | return err 78 | } 79 | _, err = buf.WriteTo(w) 80 | return err 81 | } 82 | 83 | // parseHTMLTemplates parses the HTML templates from their source. 84 | func (a *App) parseHTMLTemplates(sets [][]string) error { 85 | a.tmpls = map[string]*htmpl.Template{} 86 | for _, set := range sets { 87 | t := htmpl.New("") 88 | t.Funcs(htmpl.FuncMap{ 89 | "urlTo": a.URLTo, 90 | "urlToTrace": a.URLToTrace, 91 | "itoa": strconv.Itoa, 92 | "str": func(v interface{}) string { return fmt.Sprintf("%s", v) }, 93 | "durationClass": durationClass, 94 | "filterAnnotations": filterAnnotations, 95 | "descendTraces": func() bool { return false }, 96 | "dict": dict, 97 | }) 98 | for _, tmp := range set { 99 | tmplFile, err := tmpl.Data.Open("/" + tmp) 100 | if err != nil { 101 | return fmt.Errorf("template %v: %s", set, err) 102 | } 103 | tmplBytes, err := ioutil.ReadAll(tmplFile) 104 | tmplFile.Close() 105 | if err != nil { 106 | return fmt.Errorf("template %v: %s", set, err) 107 | } 108 | if _, err := t.Parse(string(tmplBytes)); err != nil { 109 | return fmt.Errorf("template %v: %s", set, err) 110 | } 111 | } 112 | t = t.Lookup("ROOT") 113 | if t == nil { 114 | return fmt.Errorf("ROOT template not found in %v", set) 115 | } 116 | a.tmpls[set[0]] = t 117 | } 118 | return nil 119 | } 120 | 121 | func durationClass(usec int64) string { 122 | msec := usec / 1000 123 | if msec < 30 { 124 | return "d0" 125 | } else if msec < 60 { 126 | return "d1" 127 | } else if msec < 90 { 128 | return "d2" 129 | } else if msec < 150 { 130 | return "d3" 131 | } else if msec < 250 { 132 | return "d4" 133 | } else if msec < 400 { 134 | return "d5" 135 | } else if msec < 600 { 136 | return "d6" 137 | } else if msec < 900 { 138 | return "d7" 139 | } else if msec < 1300 { 140 | return "d8" 141 | } else if msec < 1900 { 142 | return "d9" 143 | } 144 | return "d10" 145 | } 146 | 147 | func filterAnnotations(anns appdash.Annotations) appdash.Annotations { 148 | var anns2 appdash.Annotations 149 | for _, ann := range anns { 150 | if ann.Key != "" && !strings.HasPrefix(ann.Key, "_") { 151 | anns2 = append(anns2, ann) 152 | } 153 | } 154 | return anns2 155 | 156 | } 157 | 158 | // dict builds a map of paired items, allowing you to invoke a template with 159 | // multiple parameters. 160 | func dict(pairs ...interface{}) (map[string]interface{}, error) { 161 | if len(pairs)%2 != 0 { 162 | return nil, errors.New("expected pairs") 163 | } 164 | m := make(map[string]interface{}, len(pairs)/2) 165 | for i := 0; i < len(pairs); i += 2 { 166 | m[pairs[i].(string)] = pairs[i+1] 167 | } 168 | return m, nil 169 | } 170 | -------------------------------------------------------------------------------- /traceapp/tmpl/data.go: -------------------------------------------------------------------------------- 1 | // +build dev 2 | 3 | package tmpl 4 | 5 | import ( 6 | "go/build" 7 | "log" 8 | "net/http" 9 | ) 10 | 11 | func importPathToDir(importPath string) string { 12 | p, err := build.Import(importPath, "", build.FindOnly) 13 | if err != nil { 14 | log.Fatalln(err) 15 | } 16 | return p.Dir 17 | } 18 | 19 | // Data is a virtual filesystem that contains template data used by Appdash. 20 | var Data = http.Dir(importPathToDir("sourcegraph.com/sourcegraph/appdash/traceapp/tmpl/data")) 21 | -------------------------------------------------------------------------------- /traceapp/tmpl/data/aggregate.html: -------------------------------------------------------------------------------- 1 | 2 | {{define "Title"}}Aggregate View - appdash{{end}} 3 | {{define "Main"}} 4 | 5 | 32 | 33 | 34 || Name | 69 |Average (ms) | 70 |Min (ms) | 71 |Max (ms) | 72 |Std. Deviation (ms) | 73 |Timespans | 74 |
|---|
33 | 
35 |
36 | | {{.Key}} | {{str .Value}} |
|---|