121 | More information about NDT
122 | More information about Measurement Lab
123 |
This is an NDT server.
6 | 7 |You can run an NDT7 test (recommended) by going to ndt7.html
9 | 10 |You can run an NDT5 test (legacy, less recommended) by going to embed.html or by connecting to the configured ports 12 | (the defaults are 3001 and 3010).
13 | 14 | NDT7 is recommended for all new clients. NDT5 is for existing clients
15 | (including all versions before 5) that have not yet been ported to NDT7. The
16 | version "NDT6" was skipped entirely as IPv6 has been supported by NDT for
17 | many years, and the name NDT6 risked confusion with the naming scheme used by
18 | ping6 and the like.
This server is open source, and the code for it may be found at github.com/m-lab/ndt-server
22 | 23 | 24 | -------------------------------------------------------------------------------- /logging/logging_test.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "net/http" 7 | "testing" 8 | 9 | "github.com/m-lab/go/httpx" 10 | "github.com/m-lab/go/rtx" 11 | ) 12 | 13 | type fakeHandler struct{} 14 | 15 | func (s *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 16 | w.WriteHeader(200) 17 | } 18 | 19 | func TestMakeAccessLogHandler(t *testing.T) { 20 | buff := &bytes.Buffer{} 21 | old := log.Writer() 22 | defer func() { 23 | log.SetOutput(old) 24 | }() 25 | log.SetOutput(buff) 26 | f := MakeAccessLogHandler(&fakeHandler{}) 27 | log.SetOutput(old) 28 | srv := http.Server{ 29 | Addr: ":0", 30 | Handler: f, 31 | } 32 | rtx.Must(httpx.ListenAndServeAsync(&srv), "Could not start server") 33 | defer srv.Close() 34 | _, err := http.Get("http://" + srv.Addr + "/") 35 | rtx.Must(err, "Could not get") 36 | s, err := buff.ReadString('\n') 37 | if s == "" { 38 | t.Error("We should not have had an empty string") 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /ndt7/ping/ping.go: -------------------------------------------------------------------------------- 1 | // Package ping implements WebSocket PING messages. 2 | package ping 3 | 4 | import ( 5 | "encoding/json" 6 | "time" 7 | 8 | "github.com/gorilla/websocket" 9 | ) 10 | 11 | // SendTicks sends the current ticks as a ping message. 12 | func SendTicks(conn *websocket.Conn, deadline time.Time) error { 13 | // TODO(bassosimone): when we'll have a unique base time.Time reference for 14 | // the whole test, we should use that, since UnixNano() is not monotonic. 15 | ticks := int64(time.Now().UnixNano()) 16 | data, err := json.Marshal(ticks) 17 | if err == nil { 18 | err = conn.WriteControl(websocket.PingMessage, data, deadline) 19 | } 20 | return err 21 | } 22 | 23 | func ParseTicks(s string) (d int64, err error) { 24 | // TODO(bassosimone): when we'll have a unique base time.Time reference for 25 | // the whole test, we should use that, since UnixNano() is not monotonic. 26 | var prev int64 27 | err = json.Unmarshal([]byte(s), &prev) 28 | if err == nil { 29 | d = (int64(time.Now().UnixNano()) - prev) 30 | } 31 | return 32 | } 33 | -------------------------------------------------------------------------------- /logging/logging.go: -------------------------------------------------------------------------------- 1 | // Package logging contains data structures useful to implement logging 2 | // across ndt-server in a Docker friendly way. 3 | package logging 4 | 5 | import ( 6 | golog "log" 7 | "net/http" 8 | "os" 9 | 10 | "github.com/apex/log" 11 | "github.com/apex/log/handlers/json" 12 | "github.com/gorilla/handlers" 13 | ) 14 | 15 | // Logger is a logger that logs messages on the standard error 16 | // in a structured JSON format, to simplify processing. Emitting logs 17 | // on the standard error is consistent with the standard practices 18 | // when dockerising an Apache or Nginx instance. 19 | var Logger = log.Logger{ 20 | Handler: json.New(os.Stderr), 21 | Level: log.InfoLevel, 22 | } 23 | 24 | // MakeAccessLogHandler wraps |handler| with another handler that logs 25 | // access to each resource on the standard output. This is consistent with 26 | // the way in which Apache and Nginx are dockerised. We do not emit JSON 27 | // access logs, because access logs are a fairly standard format that 28 | // has been around for a long time now, so better to follow such standard. 29 | func MakeAccessLogHandler(handler http.Handler) http.Handler { 30 | return handlers.LoggingHandler(golog.Writer(), handler) 31 | } 32 | -------------------------------------------------------------------------------- /ndt5/web100/web100.go: -------------------------------------------------------------------------------- 1 | // Package web100 provides web100 variables (or a simulation thereof) to 2 | // interested systems. When run on not-BBR it is polling-based, when run on BBR 3 | // it only needs to measure once. 4 | package web100 5 | 6 | import "github.com/m-lab/tcp-info/tcp" 7 | 8 | // Metrics holds web100 data. According to the NDT5 protocol, each of these 9 | // metrics is required. That does not mean each is required to be non-zero, but 10 | // it does mean that the field should be present in any response. 11 | type Metrics struct { 12 | // Milliseconds 13 | MaxRTT, MinRTT, SumRTT, CurRTO, SndLimTimeCwnd, SndLimTimeRwin, SndLimTimeSender uint32 14 | 15 | // Counters 16 | DataBytesOut uint64 17 | DupAcksIn, PktsOut, PktsRetrans, Timeouts, CountRTT, CongestionSignals uint32 18 | AckPktsIn uint32 // Called SegsIn in tcp-kis.txt 19 | 20 | // Octets 21 | MaxCwnd, MaxRwinRcvd, CurMSS, Sndbuf uint32 22 | 23 | // Scaling factors 24 | RcvWinScale, SndWinScale int 25 | 26 | // Useful metrics that are not part of the required set. 27 | BytesPerSecond float64 28 | TCPInfo tcp.LinuxTCPInfo 29 | } 30 | -------------------------------------------------------------------------------- /cmd/generate-schemas/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | 7 | "github.com/m-lab/go/cloud/bqx" 8 | "github.com/m-lab/go/rtx" 9 | "github.com/m-lab/ndt-server/data" 10 | 11 | "cloud.google.com/go/bigquery" 12 | ) 13 | 14 | var ( 15 | ndt7schema string 16 | ndt5schema string 17 | ) 18 | 19 | func init() { 20 | flag.StringVar(&ndt7schema, "ndt7", "/var/spool/datatypes/ndt7.json", "filename to write ndt7 schema") 21 | flag.StringVar(&ndt5schema, "ndt5", "/var/spool/datatypes/ndt5.json", "filename to write ndt5 schema") 22 | } 23 | 24 | func main() { 25 | flag.Parse() 26 | // Generate and save ndt7 schema for autoloading. 27 | row7 := data.NDT7Result{} 28 | sch, err := bigquery.InferSchema(row7) 29 | rtx.Must(err, "failed to generate ndt7 schema") 30 | sch = bqx.RemoveRequired(sch) 31 | b, err := sch.ToJSONFields() 32 | rtx.Must(err, "failed to marshal schema") 33 | ioutil.WriteFile(ndt7schema, b, 0o644) 34 | 35 | // Generate and save ndt5 schema for autoloading. 36 | row5 := data.NDT7Result{} 37 | sch, err = bigquery.InferSchema(row5) 38 | rtx.Must(err, "failed to generate ndt5 schema") 39 | sch = bqx.RemoveRequired(sch) 40 | b, err = sch.ToJSONFields() 41 | rtx.Must(err, "failed to marshal schema") 42 | ioutil.WriteFile(ndt5schema, b, 0o644) 43 | } 44 | -------------------------------------------------------------------------------- /ndt7/upload/upload.go: -------------------------------------------------------------------------------- 1 | // Package upload implements the ndt7 upload 2 | package upload 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/gorilla/websocket" 8 | "github.com/m-lab/ndt-server/ndt7/model" 9 | "github.com/m-lab/ndt-server/ndt7/receiver" 10 | "github.com/m-lab/ndt-server/ndt7/upload/sender" 11 | ) 12 | 13 | // Do implements the upload subtest. The ctx argument is the parent context for 14 | // the subtest. The conn argument is the open WebSocket connection. The data 15 | // argument is the archival data where results are saved. All arguments are 16 | // owned by the caller of this function. 17 | func Do(ctx context.Context, conn *websocket.Conn, data *model.ArchivalData) error { 18 | // Implementation note: use child contexts so the sender is strictly time 19 | // bounded. After timeout, the sender closes the conn, which results in the 20 | // receiver completing. 21 | 22 | // Receive and save client-provided measurements in data. 23 | recv := receiver.StartUploadReceiverAsync(ctx, conn, data) 24 | 25 | // Perform upload and save server-measurements in data. 26 | // TODO: move sender.Start logic to this file. 27 | err := sender.Start(ctx, conn, data) 28 | 29 | // Block on the receiver completing to guarantee that access to data is synchronous. 30 | <-recv.Done() 31 | return err 32 | } 33 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | // Metrics for general use, in both NDT5 and in NDT7. 9 | var ( 10 | ActiveTests = promauto.NewGaugeVec( 11 | prometheus.GaugeOpts{ 12 | Name: "ndt_active_tests", 13 | Help: "A gauge of requests currently being served by the NDT server.", 14 | }, 15 | []string{"protocol"}) 16 | TestRate = promauto.NewHistogramVec( 17 | prometheus.HistogramOpts{ 18 | Name: "ndt_test_rate_mbps", 19 | Help: "A histogram of request rates.", 20 | Buckets: []float64{ 21 | .1, .15, .25, .4, .6, 22 | 1, 1.5, 2.5, 4, 6, 23 | 10, 15, 25, 40, 60, 24 | 100, 150, 250, 400, 600, 25 | 1000}, 26 | }, 27 | []string{"protocol", "direction", "monitoring"}, 28 | ) 29 | ) 30 | 31 | // GetResultLabel returns one of four strings based on the combination of 32 | // whether the error ("okay" or "error") and the rate ("with-rate" (non-zero) or 33 | // "without-rate" (zero)). 34 | func GetResultLabel(err error, rate float64) string { 35 | withErr := "okay" 36 | if err != nil { 37 | withErr = "error" 38 | } 39 | withResult := "-with-rate" 40 | if rate == 0 { 41 | withResult = "-without-rate" 42 | } 43 | return withErr + withResult 44 | } 45 | -------------------------------------------------------------------------------- /html/mlab/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
8 |
12 |
13 | NDT (Network Diagnostic Tool) Server from Measurement 10 | Lab
11 |
This is an
15 | NDT server, provided by Measurement Lab (M-Lab).
More information about NDT, other M-Lab hosted tests, the M-Lab platform, and 20 | the open data we publish can be found on our website.
22 | 23 |If you need assistance, please email support@measurementlab.net, or 24 | visit our website for complete 25 | information about the M-Lab platform, tests, and data.
26 | 27 |This server is open source, and the code for it may be found at github.com/m-lab/ndt-server
29 | 30 | 31 | -------------------------------------------------------------------------------- /ndt7/download/download.go: -------------------------------------------------------------------------------- 1 | // Package download implements the ndt7/server downloader. 2 | package download 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/gorilla/websocket" 8 | "github.com/m-lab/ndt-server/ndt7/download/sender" 9 | "github.com/m-lab/ndt-server/ndt7/model" 10 | "github.com/m-lab/ndt-server/ndt7/receiver" 11 | ) 12 | 13 | // Do implements the download subtest. The ctx argument is the parent context 14 | // for the subtest. The conn argument is the open WebSocket connection. The data 15 | // argument is the archival data where results are saved. All arguments are 16 | // owned by the caller of this function. 17 | func Do(ctx context.Context, conn *websocket.Conn, data *model.ArchivalData, params *sender.Params) error { 18 | // Implementation note: use child contexts so the sender is strictly time 19 | // bounded. After timeout, the sender closes the conn, which results in the 20 | // receiver completing. 21 | 22 | // Receive and save client-provided measurements in data. 23 | recv := receiver.StartDownloadReceiverAsync(ctx, conn, data) 24 | 25 | // Perform download and save server-measurements in data. 26 | // TODO: move sender.Start logic to this file. 27 | err := sender.Start(ctx, conn, data, params) 28 | 29 | // Block on the receiver completing to guarantee that access to data is synchronous. 30 | <-recv.Done() 31 | return err 32 | } 33 | -------------------------------------------------------------------------------- /ndt7/ndt7test/ndt7test.go: -------------------------------------------------------------------------------- 1 | package ndt7test 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | 10 | "github.com/m-lab/go/testingx" 11 | "github.com/m-lab/ndt-server/ndt7/handler" 12 | "github.com/m-lab/ndt-server/ndt7/spec" 13 | "github.com/m-lab/ndt-server/netx" 14 | "github.com/m-lab/tcp-info/eventsocket" 15 | ) 16 | 17 | // NewNDT7Server creates a local httptest server capable of running an ndt7 18 | // measurement in unittests. 19 | func NewNDT7Server(t *testing.T) (*handler.Handler, *httptest.Server) { 20 | dir := t.TempDir() 21 | // TODO: add support for token verifiers. 22 | // TODO: add support for TLS server. 23 | ndt7Handler := &handler.Handler{DataDir: dir, Events: eventsocket.NullServer()} 24 | ndt7Mux := http.NewServeMux() 25 | ndt7Mux.Handle(spec.DownloadURLPath, http.HandlerFunc(ndt7Handler.Download)) 26 | ndt7Mux.Handle(spec.UploadURLPath, http.HandlerFunc(ndt7Handler.Upload)) 27 | 28 | // Create unstarted so we can setup a custom netx.Listener. 29 | ts := httptest.NewUnstartedServer(ndt7Mux) 30 | listener, err := net.Listen("tcp", ":0") 31 | testingx.Must(t, err, "failed to allocate a listening tcp socket") 32 | addr := (listener.(*net.TCPListener)).Addr().(*net.TCPAddr) 33 | // Populate insecure port value with dynamic port. 34 | ndt7Handler.InsecurePort = fmt.Sprintf(":%d", addr.Port) 35 | ts.Listener = netx.NewListener(listener.(*net.TCPListener)) 36 | // Now that the test server has our custom listener, start it. 37 | ts.Start() 38 | return ndt7Handler, ts 39 | } 40 | -------------------------------------------------------------------------------- /ndt7/handler/handler_test.go: -------------------------------------------------------------------------------- 1 | // Package handler implements the WebSocket handler for ndt7. 2 | package handler 3 | 4 | import ( 5 | "net/url" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/m-lab/ndt-server/ndt7/download/sender" 10 | "github.com/m-lab/ndt-server/ndt7/spec" 11 | ) 12 | 13 | func Test_validateEarlyExit(t *testing.T) { 14 | type args struct { 15 | values url.Values 16 | } 17 | tests := []struct { 18 | name string 19 | values url.Values 20 | want *sender.Params 21 | wantErr bool 22 | }{ 23 | { 24 | name: "valid-param", 25 | values: url.Values{"early_exit": {spec.ValidEarlyExitValues[0]}}, 26 | want: &sender.Params{ 27 | IsEarlyExit: true, 28 | MaxBytes: 250000000, 29 | }, 30 | wantErr: false, 31 | }, 32 | { 33 | name: "invalid-param", 34 | values: url.Values{"early_exit": {"123"}}, 35 | want: nil, 36 | wantErr: true, 37 | }, 38 | { 39 | name: "missing-value", 40 | values: url.Values{"early_exit": {""}}, 41 | want: nil, 42 | wantErr: true, 43 | }, 44 | { 45 | name: "absent-param", 46 | values: url.Values{"foo": {"bar"}}, 47 | want: &sender.Params{ 48 | IsEarlyExit: false, 49 | MaxBytes: 0, 50 | }, 51 | wantErr: false, 52 | }, 53 | } 54 | for _, tt := range tests { 55 | t.Run(tt.name, func(t *testing.T) { 56 | got, err := validateEarlyExit(tt.values) 57 | if (err != nil) != tt.wantErr { 58 | t.Errorf("validateEarlyExit() error = %v, wantErr %v", err, tt.wantErr) 59 | return 60 | } 61 | if !reflect.DeepEqual(got, tt.want) { 62 | t.Errorf("validateEarlyExit() = %v, want %v", got, tt.want) 63 | } 64 | }) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/m-lab/ndt-server 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/apex/log v1.9.0 7 | github.com/google/uuid v1.3.0 8 | github.com/gorilla/handlers v1.5.1 9 | github.com/gorilla/websocket v1.5.0 10 | github.com/m-lab/access v0.0.11 11 | github.com/m-lab/go v0.1.75 12 | github.com/m-lab/tcp-info v1.5.3 13 | github.com/m-lab/uuid v1.0.1 14 | github.com/prometheus/client_golang v1.13.0 15 | go.uber.org/goleak v1.1.12 16 | gopkg.in/m-lab/pipe.v3 v3.0.0-20180108231244-604e84f43ee0 17 | ) 18 | 19 | require ( 20 | github.com/google/go-cmp v0.5.9 // indirect 21 | github.com/rogpeppe/go-internal v1.9.0 // indirect 22 | github.com/stretchr/testify v1.8.1 // indirect 23 | golang.org/x/net v0.8.0 // indirect 24 | golang.org/x/text v0.8.0 // indirect 25 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 26 | ) 27 | 28 | require ( 29 | github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 // indirect 30 | github.com/beorn7/perks v1.0.1 // indirect 31 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 32 | github.com/felixge/httpsnoop v1.0.1 // indirect 33 | github.com/gocarina/gocsv v0.0.0-20210408192840-02d7211d929d // indirect 34 | github.com/golang/protobuf v1.5.3 // indirect 35 | github.com/justinas/alice v1.2.0 // indirect 36 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 37 | github.com/pkg/errors v0.9.1 // indirect 38 | github.com/prometheus/client_model v0.2.0 // indirect 39 | github.com/prometheus/common v0.37.0 // indirect 40 | github.com/prometheus/procfs v0.8.0 // indirect 41 | golang.org/x/crypto v0.7.0 42 | golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 43 | golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect 44 | golang.org/x/sys v0.7.0 // indirect 45 | google.golang.org/protobuf v1.30.0 // indirect 46 | gopkg.in/square/go-jose.v2 v2.6.0 // indirect 47 | ) 48 | -------------------------------------------------------------------------------- /fullstack/Dockerfile: -------------------------------------------------------------------------------- 1 | # Traceroute-caller is the most brittle of our tools, as it requires 2 | # scamper which is not statically linked. So we work within that image. 3 | FROM measurementlab/traceroute-caller 4 | 5 | # UUIDs require a little setup to use appropriately. In particular, they need a 6 | # unique string written to a well-known location to serve as a prefix. 7 | COPY --from=measurementlab/uuid /create-uuid-prefix-file / 8 | 9 | # tcp-info needs its binary and also needs zstd 10 | COPY --from=measurementlab/tcp-info /bin/tcp-info /tcp-info 11 | COPY --from=measurementlab/tcp-info /bin/zstd /bin/zstd 12 | COPY --from=measurementlab/tcp-info /licences/zstd/ /licences/zstd/ 13 | 14 | # packet-headers needs its binary and libpcap. There's no good way to get both 15 | # easily from the image, due to C-linking issues and the differences between 16 | # alpine and ubuntu, so just rebuild it here. 17 | ENV DEBIAN_FRONTEND=noninteractive 18 | RUN apt-get update && apt-get install -y libpcap-dev golang-go git socat 19 | RUN go get github.com/m-lab/packet-headers 20 | RUN mv /root/go/bin/packet-headers /packet-headers 21 | 22 | # The NDT server needs the server binary and its HTML files 23 | COPY --from=measurementlab/ndt-server /ndt-server / 24 | COPY --from=measurementlab/ndt-server /html /html 25 | 26 | COPY fullstack/start.sh /start.sh 27 | RUN chmod +x /start.sh 28 | 29 | WORKDIR / 30 | 31 | # You can add further arguments to ndt-server, all the other commands are 32 | # fixed. Prometheus metrics for tcp-info and traceroute-caller can be 33 | # found on ports 9991 and 9992 (set in the start script), while the ndt 34 | # server metrics can be found on port 9990 by default, but can be set by 35 | # passing --prometheusx.listen-address to that start script. 36 | # 37 | # If you would like to run any SSL/TLS-based tests, you'll need to pass in 38 | # the --cert= and --key= arguments. 39 | ENTRYPOINT ["/start.sh"] 40 | -------------------------------------------------------------------------------- /ndt7/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/gorilla/websocket" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promauto" 9 | ) 10 | 11 | // Metrics for exporting to prometheus to aid in server monitoring. 12 | var ( 13 | ClientConnections = promauto.NewCounterVec( 14 | prometheus.CounterOpts{ 15 | Name: "ndt7_client_connections_total", 16 | Help: "Count of clients that connect and setup an ndt7 measurement.", 17 | }, 18 | []string{"direction", "status"}, 19 | ) 20 | ClientTestResults = promauto.NewCounterVec( 21 | prometheus.CounterOpts{ 22 | Name: "ndt7_client_test_results_total", 23 | Help: "Number of client-connections for NDT tests run by this server.", 24 | }, 25 | []string{"protocol", "direction", "result"}, 26 | ) 27 | ClientSenderErrors = promauto.NewCounterVec( 28 | prometheus.CounterOpts{ 29 | Name: "ndt7_client_sender_errors_total", 30 | Help: "Number of sender errors on all return paths.", 31 | }, 32 | []string{"protocol", "direction", "error"}, 33 | ) 34 | ClientReceiverErrors = promauto.NewCounterVec( 35 | prometheus.CounterOpts{ 36 | Name: "ndt7_client_receiver_errors_total", 37 | Help: "Number of receiver errors on all return paths.", 38 | }, 39 | []string{"protocol", "direction", "error"}, 40 | ) 41 | ) 42 | 43 | // ConnLabel infers an appropriate label for the websocket protocol. 44 | func ConnLabel(conn *websocket.Conn) string { 45 | // NOTE: this isn't perfect, but it is simple and a) works for production deployments, 46 | // and 2) will work for custom deployments with ports having the same suffix, e.g. 4433, 8080. 47 | if strings.HasSuffix(conn.LocalAddr().String(), "443") { 48 | return "ndt7+wss" 49 | } 50 | if strings.HasSuffix(conn.LocalAddr().String(), "80") { 51 | return "ndt7+ws" 52 | } 53 | return "ndt7+unknown" 54 | } 55 | -------------------------------------------------------------------------------- /ndt5/ndt/server.go: -------------------------------------------------------------------------------- 1 | package ndt 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/m-lab/ndt-server/metadata" 7 | "github.com/m-lab/ndt-server/ndt5/protocol" 8 | ) 9 | 10 | // ConnectionType records whether this test is performed over plain TCP, 11 | // websockets, or secure websockets. 12 | type ConnectionType string 13 | 14 | // String returns the connection type named used in archival data. 15 | func (c ConnectionType) String() string { 16 | return string(c) 17 | } 18 | 19 | // Label returns the connection type name used in monitoring metrics. 20 | func (c ConnectionType) Label() string { 21 | switch c { 22 | case WSS: 23 | return "ndt5+wss" 24 | case WS: 25 | return "ndt5+ws" 26 | case Plain: 27 | return "ndt5+plain" 28 | default: 29 | return "ndt5+unknown" 30 | } 31 | } 32 | 33 | // The types of connections we support. 34 | var ( 35 | WS = ConnectionType("WS") 36 | WSS = ConnectionType("WSS") 37 | Plain = ConnectionType("PLAIN") 38 | ) 39 | 40 | // Server describes the methods implemented by every server of every connection 41 | // type. 42 | type Server interface { 43 | SingleMeasurementServerFactory 44 | ConnectionType() ConnectionType 45 | DataDir() string 46 | Metadata() []metadata.NameValue 47 | LoginCeremony(protocol.Connection) (int, error) 48 | } 49 | 50 | // SingleMeasurementServerFactory is the method by which we abstract away what 51 | // kind of server is being created at any given time. Using this abstraction 52 | // allows us to use almost the same code for WS and WSS. 53 | type SingleMeasurementServerFactory interface { 54 | SingleServingServer(direction string) (SingleMeasurementServer, error) 55 | } 56 | 57 | // SingleMeasurementServer is the interface implemented by every single-serving 58 | // server. No matter whether they use WSS, WS, TCP with JSON, or TCP without 59 | // JSON. 60 | type SingleMeasurementServer interface { 61 | Port() int 62 | ServeOnce(context.Context) (protocol.MeasuredConnection, error) 63 | Close() 64 | } 65 | -------------------------------------------------------------------------------- /html/ndt-wrapper-ww.js: -------------------------------------------------------------------------------- 1 | /* This is an NDT client, written in javascript. It speaks the websocket 2 | * version of the NDT protocol. The NDT protocol is documented at: 3 | * https://code.google.com/p/ndt/wiki/NDTProtocol 4 | */ 5 | 6 | /*jslint bitwise: true, browser: true, nomen: true, vars: true */ 7 | /*global Uint8Array, WebSocket */ 8 | 9 | 'use strict'; 10 | 11 | importScripts('ndt-browser-client.js'); 12 | 13 | self.addEventListener('message', function (e) { 14 | var msg = e.data; 15 | switch (msg.cmd) { 16 | case 'start': 17 | startNDT(msg.hostname, msg.port, msg.protocol, msg.path, 18 | msg.update_interval); 19 | break; 20 | case 'stop': 21 | self.close(); 22 | break; 23 | default: 24 | // self.postMessage('Unknown command: ' + data.msg); 25 | break; 26 | }; 27 | }, false); 28 | 29 | function startNDT(hostname, port, protocol, path, update_interval) { 30 | var callbacks = { 31 | 'onstart': function(server) { 32 | self.postMessage({ 33 | 'cmd': 'onstart', 34 | 'server': server 35 | }); 36 | }, 37 | 'onstatechange': function(state, results) { 38 | self.postMessage({ 39 | 'cmd': 'onstatechange', 40 | 'state': state, 41 | 'results': results, 42 | }); 43 | }, 44 | 'onprogress': function(state, results) { 45 | self.postMessage({ 46 | 'cmd': 'onprogress', 47 | 'state': state, 48 | 'results': results, 49 | }); 50 | }, 51 | 'onfinish': function(results) { 52 | self.postMessage({ 53 | 'cmd': 'onfinish', 54 | 'results': results 55 | }); 56 | }, 57 | 'onerror': function(error_message) { 58 | self.postMessage({ 59 | 'cmd': 'onerror', 60 | 'error_message': error_message 61 | }); 62 | } 63 | }; 64 | 65 | var client = new NDTjs(hostname, port, protocol, path, callbacks, 66 | update_interval); 67 | client.startTest(); 68 | } 69 | -------------------------------------------------------------------------------- /ndt7/metrics/README.md: -------------------------------------------------------------------------------- 1 | # NDT7 Server Metrics 2 | 3 | Summary of ndt7 metrics useful for monitoring client requests, measurement 4 | successes, and error rates for the sender and receiver. 5 | 6 | * `ndt7_client_connections_total{direction, status}` counts every client 7 | connection that reaches `handler.Upload` or `handler.Download`. 8 | 9 | * The "direction=" label indicates an "upload" or "download" measurement. 10 | * The "status=" label is either "result" or a specific error that 11 | prevented setup before the connection was aborted. 12 | * All status="result" clients are counted in `ndt7_client_test_results_total`. 13 | * All status="result" clients should also equal the number of files written. 14 | 15 | * `ndt7_client_test_results_total{protocol, direction, result}` counts the 16 | test results of clients that successfully setup the websocket connection. 17 | 18 | * The "protocol=" label indicates the "ndt7+wss" or "ndt7+ws" protocol. 19 | * The "direction=" labels are as above. 20 | * The "result=" label is either "okay-with-rate", "error-with-rate" or 21 | "error-without-rate". 22 | * All result=~"*-with-rate" measurements are also recorded in the shared 23 | test rate histogram. 24 | * All results are also counted in `ndt7_client_sender_errors_total` and 25 | `ndt7_client_receiver_errors_total` 26 | 27 | * `ndt7_client_sender_errors_total{protocol, direction, error}` 28 | * The "protocol=" and "direction=" labels are as above. 29 | * The "error=" label contains unique values mapping to specific error or return 30 | paths in the sender. 31 | 32 | * `ndt7_client_receiver_errors_total{protocol, direction, error}` 33 | * Just like the `ndt7_client_sender_errors_total` metric, but for the receiver. 34 | 35 | Expected invariants: 36 | 37 | * `ndt7_client_connections_total{status="result"} == sum(ndt7_client_test_results_total)` 38 | * `sum(ndt7_client_test_results_total) == sum(ndt7_client_sender_errors_total)` 39 | * `sum(ndt7_client_test_results_total) == sum(ndt7_client_receiver_errors_total)` 40 | -------------------------------------------------------------------------------- /cloud-config.yaml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | write_files: 4 | - path: /var/lib/cloud/scripts/prom_vdlimit_metrics.sh 5 | permissions: 0700 6 | owner: root:root 7 | content: | 8 | #!/bin/bash 9 | VDLIMIT_TEXT="" 10 | VDLIMIT_FILE="/var/spool/node_exporter/vdlimit.prom" 11 | USED=$(df --output="used" /mnt/stateful_partition | tail -n1 ) 12 | FREE=$(df --output="avail" /mnt/stateful_partition | tail -n1 ) 13 | SIZE=$(df --output="size" /mnt/stateful_partition | tail -n1 ) 14 | VDLIMIT_TEXT="vdlimit_used{experiment=\"ndt.iupui\"} ${USED}\n" 15 | VDLIMIT_TEXT+="vdlimit_free{experiment=\"ndt.iupui\"} ${FREE}\n" 16 | VDLIMIT_TEXT+="vdlimit_total{experiment=\"ndt.iupui\"} ${SIZE}\n" 17 | echo -e $VDLIMIT_TEXT > $VDLIMIT_FILE 18 | - path: /etc/systemd/system/prom-vdlimit-metrics.service 19 | permissions: 0644 20 | owner: root:root 21 | content: | 22 | [Unit] 23 | Description=Writes vdlimit Prom metrics to /var/spool/node_exporter/vdlimit.prom 24 | [Service] 25 | Type=oneshot 26 | ExecStart=/var/lib/cloud/scripts/prom_vdlimit_metrics.sh 27 | - path: /etc/systemd/system/prom-vdlimit-metrics.timer 28 | permissions: 0644 29 | owner: root:root 30 | content: | 31 | [Unit] 32 | Description=Run /usr/bin/prom_vdlimit_metrics.sh every 5 minutes. 33 | [Timer] 34 | OnBootSec=1min 35 | OnUnitActiveSec=5min 36 | [Install] 37 | WantedBy=multi-user.target 38 | 39 | # The iptables command below is necessary due to the way that the Google 40 | # Container-Optimized OS works. For COS, it is necessary to both open holes in 41 | # the VPC firewall as well as opening the firewall on the VM itself. 42 | # https://cloud.google.com/container-optimized-os/docs/how-to/firewall 43 | runcmd: 44 | - systemctl daemon-reload 45 | - systemctl enable prom-vdlimit-metrics.service 46 | - systemctl enable prom-vdlimit-metrics.timer 47 | - systemctl start prom-vdlimit-metrics.service 48 | - systemctl start prom-vdlimit-metrics.timer 49 | - iptables -A INPUT -p tcp -j ACCEPT 50 | -------------------------------------------------------------------------------- /ndt7/model/archivaldata.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/m-lab/ndt-server/metadata" 7 | "github.com/m-lab/tcp-info/inetdiag" 8 | "github.com/m-lab/tcp-info/tcp" 9 | ) 10 | 11 | // ArchivalData saves all instantaneous measurements over the lifetime of a test. 12 | type ArchivalData struct { 13 | UUID string 14 | StartTime time.Time 15 | EndTime time.Time 16 | ServerMeasurements []Measurement 17 | ClientMeasurements []Measurement 18 | ClientMetadata []metadata.NameValue `json:",omitempty"` 19 | ServerMetadata []metadata.NameValue `json:",omitempty"` 20 | } 21 | 22 | // The Measurement struct contains measurement results. This structure is 23 | // meant to be serialised as JSON as sent as a textual message. This 24 | // structure is specified in the ndt7 specification. 25 | type Measurement struct { 26 | AppInfo *AppInfo `json:",omitempty"` 27 | ConnectionInfo *ConnectionInfo `json:",omitempty"` 28 | BBRInfo *BBRInfo `json:",omitempty"` 29 | TCPInfo *TCPInfo `json:",omitempty"` 30 | } 31 | 32 | // AppInfo contains an application level measurement. This structure is 33 | // described in the ndt7 specification. 34 | type AppInfo struct { 35 | NumBytes int64 36 | ElapsedTime int64 37 | } 38 | 39 | // ConnectionInfo contains connection info. This structure is described 40 | // in the ndt7 specification. 41 | type ConnectionInfo struct { 42 | Client string 43 | Server string 44 | UUID string `json:",omitempty"` 45 | StartTime time.Time `json:",omitempty"` 46 | } 47 | 48 | // The BBRInfo struct contains information measured using BBR. This structure is 49 | // an extension to the ndt7 specification. Variables here have the same 50 | // measurement unit that is used by the Linux kernel. 51 | type BBRInfo struct { 52 | inetdiag.BBRInfo 53 | ElapsedTime int64 54 | } 55 | 56 | // The TCPInfo struct contains information measured using TCP_INFO. This 57 | // structure is described in the ndt7 specification. 58 | type TCPInfo struct { 59 | tcp.LinuxTCPInfo 60 | ElapsedTime int64 61 | } 62 | -------------------------------------------------------------------------------- /fullstack/README.md: -------------------------------------------------------------------------------- 1 | This is a docker image that allows people to easily run their own ndt-server 2 | binary, along with some recommended "sidecar" measurement services to grab 3 | richer data. Advanced users will likely want to configure each service 4 | independently, but this docker image allows interested parties to try running 5 | the server with a simple command: 6 | ``` 7 | $ docker run --net=host measurementlab/ndt 8 | ``` 9 | After that, point your web browser to 10 | [http://localhost:3001/static/widget.html](http://localhost:3001/static/widget.html) 11 | and run a speed test to verify that things work. The default NDT5 JS client 12 | needs some Javascript client-side optimization (pull requests gratefully 13 | accepted), but you should still be able to run a speed test immediately. 14 | 15 | # How to use this image. 16 | 17 | If you just want to run a server that speaks the unencrypted NDT5 (legacy) 18 | protocol, then you can run: 19 | ``` 20 | $ docker run --net=host measurementlab/ndt 21 | ``` 22 | and you will get an NDT server running on port 3001, with data being saved to 23 | the in-container directory `/var/spool/ndt/` 24 | 25 | If you would like to run NDT7 tests (which you should, it is a simpler protocol 26 | and a more robust measurement) or NDT5 tests over TLS, then you will need a 27 | private key and a TLS certificate (let's assume they are called 28 | `/etc/certs/key.pem` and `/etc/certs/cert.pem`). To run an NDT7 server on port 29 | 443, you must mount the directory with those certificates inside the container, 30 | and then tell the NDT server about those files. For example: 31 | ``` 32 | $ docker run -v /etc/certs:/certs --net=host measurementlab/ndt \ 33 | --key=/certs/key.pem --cert=/certs/cert.pem 34 | ``` 35 | 36 | The NDT server produces data on disk. If you would like this data saved to a 37 | directory outside of the docker container, then you need to mount the external 38 | directory inside the container at `/var/spool/ndt` using the `-v` argument to 39 | `docker run`. 40 | 41 | All arguments to `docker run` after the name of the image are passed directly 42 | through to the NDT server. 43 | -------------------------------------------------------------------------------- /ndt7/ndt7test/ndt7test_test.go: -------------------------------------------------------------------------------- 1 | package ndt7test 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | 12 | "github.com/gorilla/websocket" 13 | "github.com/m-lab/go/testingx" 14 | "github.com/m-lab/ndt-server/ndt7/spec" 15 | ) 16 | 17 | func TestNewNDT7Server(t *testing.T) { 18 | // Create the ndt7test server. 19 | h, srv := NewNDT7Server(t) 20 | defer os.RemoveAll(h.DataDir) 21 | 22 | // Prepare to run a simplified download with ndt7test server. 23 | URL, _ := url.Parse(srv.URL) 24 | URL.Scheme = "ws" 25 | URL.Path = spec.DownloadURLPath 26 | headers := http.Header{} 27 | headers.Add("Sec-WebSocket-Protocol", spec.SecWebSocketProtocol) 28 | headers.Add("User-Agent", "fake-user-agent") 29 | ctx := context.Background() 30 | dialer := websocket.Dialer{HandshakeTimeout: 10 * time.Second} 31 | conn, _, err := dialer.DialContext(ctx, URL.String(), headers) 32 | testingx.Must(t, err, "failed to dial websocket ndt7 test") 33 | err = simpleDownload(ctx, t, conn) 34 | if err != nil && !websocket.IsCloseError(err, websocket.CloseNormalClosure) { 35 | testingx.Must(t, err, "failed to download") 36 | } 37 | 38 | // Allow the server time to save the file, the client may stop before the server does. 39 | time.Sleep(1 * time.Second) 40 | // Verify a file was saved. 41 | m, err := filepath.Glob(h.DataDir + "/ndt7/*/*/*/*") 42 | testingx.Must(t, err, "failed to glob datadir: %s", h.DataDir) 43 | if len(m) == 0 { 44 | t.Errorf("no files found") 45 | } 46 | } 47 | 48 | func simpleDownload(ctx context.Context, t *testing.T, conn *websocket.Conn) error { 49 | defer conn.Close() 50 | wholectx, cancel := context.WithTimeout(ctx, spec.MaxRuntime) 51 | defer cancel() 52 | conn.SetReadLimit(spec.MaxMessageSize) 53 | err := conn.SetReadDeadline(time.Now().Add(spec.MaxRuntime)) 54 | testingx.Must(t, err, "failed to set read deadline") 55 | 56 | var total int64 57 | // WARNING: this is not a reference client. 58 | for wholectx.Err() == nil { 59 | _, mdata, err := conn.ReadMessage() 60 | if err != nil { 61 | return err 62 | } 63 | total += int64(len(mdata)) 64 | } 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /ndt5/protocol/messager_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/m-lab/ndt-server/ndt5/web100" 8 | ) 9 | 10 | func assertJSONMessagerIsMessager(jm *jsonMessager) { 11 | func(m Messager) {}(jm) 12 | } 13 | 14 | func assertTLVMessagerIsMessager(tm *tlvMessager) { 15 | func(m Messager) {}(tm) 16 | } 17 | 18 | type fakeMessager struct { 19 | sentMessages []string 20 | errorAfter int 21 | } 22 | 23 | func (fm *fakeMessager) SendMessage(_ MessageType, msg []byte) error { 24 | fm.sentMessages = append(fm.sentMessages, string(msg)) 25 | if fm.errorAfter > 0 { 26 | defer func() { fm.errorAfter-- }() 27 | if fm.errorAfter == 1 { 28 | return errors.New("Error for testing") 29 | } 30 | } 31 | return nil 32 | } 33 | 34 | func (fm *fakeMessager) SendS2CResults(throughputKbps, unsentBytes, totalSentBytes int64) error { 35 | return nil 36 | } 37 | 38 | func (fm *fakeMessager) ReceiveMessage(MessageType) ([]byte, error) { return []byte{}, nil } 39 | 40 | func (fm *fakeMessager) Encoding() Encoding { 41 | return Unknown 42 | } 43 | 44 | func TestSendMetrics(t *testing.T) { 45 | data := &web100.Metrics{} 46 | fm := &fakeMessager{} 47 | err := SendMetrics(data, fm, "") 48 | if err != nil { 49 | t.Error("Error should be nil", err) 50 | } 51 | // 73 was chosen because we needed a number that was greater than zero and not 52 | // greater than the number of fields in the Metrics struct. This is a moving 53 | // target, so we don't want to be too specific and require equality with the 54 | // current count. There were a total of 73 fields as of 2019-08-23, so that's a 55 | // good lower bound. 56 | if len(fm.sentMessages) < 73 { 57 | t.Error("Bad messages:", len(fm.sentMessages), fm) 58 | } 59 | } 60 | 61 | func TestSendMetricsWithErrors(t *testing.T) { 62 | data := &web100.Metrics{} 63 | // Erroring after 25 fields means that the error occurs inside the tcpinfo 64 | // struct, which exercises both error cases in the recursive function. 65 | fm := &fakeMessager{ 66 | errorAfter: 25, 67 | } 68 | err := SendMetrics(data, fm, "") 69 | if err == nil { 70 | t.Error("Error should not be nil", err) 71 | } 72 | if len(fm.sentMessages) > 25 { 73 | t.Error("Too many messages sent:", fm) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /netx/iface/fdinfo.go: -------------------------------------------------------------------------------- 1 | // Package iface provides access to network connection operations via file 2 | // descriptor. The implementation MUST be correct by inspection. 3 | package iface 4 | 5 | import ( 6 | "net" 7 | "os" 8 | 9 | "github.com/m-lab/ndt-server/bbr" 10 | "github.com/m-lab/ndt-server/tcpinfox" 11 | "github.com/m-lab/tcp-info/inetdiag" 12 | "github.com/m-lab/tcp-info/tcp" 13 | "github.com/m-lab/uuid" 14 | ) 15 | 16 | // ConnFile provides access to underlying network file. 17 | type ConnFile interface { 18 | DupFile(tc *net.TCPConn) (*os.File, error) 19 | } 20 | 21 | // NetInfo provides access to network connection metadata. 22 | type NetInfo interface { 23 | GetUUID(fp *os.File) (string, error) 24 | GetBBRInfo(fp *os.File) (inetdiag.BBRInfo, error) 25 | GetTCPInfo(fp *os.File) (*tcp.LinuxTCPInfo, error) 26 | } 27 | 28 | // RealConnInfo implements both the ConnFile and NetInfo interfaces. 29 | type RealConnInfo struct{} 30 | 31 | // DupFile returns the corresponding *os.File. Note that the 32 | // returned *os.File is a dup() of the original, hence you now have ownership 33 | // of two objects that you need to remember to defer Close() of. 34 | func (f *RealConnInfo) DupFile(tc *net.TCPConn) (*os.File, error) { 35 | // Implementation note: according to a 2013 message on golang-nuts [1], the 36 | // code that follows is broken on Unix because calling File() makes the socket 37 | // blocking so causing Go to use more threads and, additionally, "timer wheel 38 | // inside net package never fires". However, an April, 19 2018 commit 39 | // on src/net/tcpsock.go apparently has removed such restriction and so now 40 | // (i.e. since go1.11beta1) it's safe to use the code below [2, 3]. 41 | // 42 | // [1] https://grokbase.com/t/gg/golang-nuts/1349whs82r 43 | // 44 | // [2] https://github.com/golang/go/commit/60e3ebb9cba 45 | // 46 | // [3] https://github.com/golang/go/issues/24942 47 | // 48 | // For this reason, this code only works correctly with go >= 1.11. 49 | return tc.File() 50 | } 51 | 52 | // GetUUID returns a UUID for the given file pointer. 53 | func (f *RealConnInfo) GetUUID(fp *os.File) (string, error) { 54 | return uuid.FromFile(fp) 55 | } 56 | 57 | // GetBBRInfo returns BBRInfo for the given file pointer. 58 | func (f *RealConnInfo) GetBBRInfo(fp *os.File) (inetdiag.BBRInfo, error) { 59 | return bbr.GetBBRInfo(fp) 60 | } 61 | 62 | // GetTCPInfo returns TCPInfo for the given file pointer. 63 | func (f *RealConnInfo) GetTCPInfo(fp *os.File) (*tcp.LinuxTCPInfo, error) { 64 | return tcpinfox.GetTCPInfo(fp) 65 | } 66 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | ndt-server: 4 | image: ndt-server 5 | volumes: 6 | - ./certs:/certs 7 | - ./html:/html 8 | - ./schemas:/schemas 9 | - ./resultsdir:/resultsdir 10 | - ./localgcs:/localgcs 11 | cap_drop: 12 | - ALL 13 | depends_on: 14 | generate-schemas: 15 | condition: service_completed_successfully 16 | 17 | # NOTE: All service containers will use the same network and IP. All ports 18 | # must be configured on the first. 19 | ports: 20 | # ndt-server prometheus, alt TLS and alt non-tls ports. 21 | - target: 9990 22 | published: 9990 23 | protocol: tcp 24 | mode: bridge 25 | - target: 4443 26 | published: 4443 27 | protocol: tcp 28 | mode: bridge 29 | - target: 8080 30 | published: 8080 31 | protocol: tcp 32 | mode: bridge 33 | # jostler prometheus. 34 | - target: 9991 35 | published: 9991 36 | protocol: tcp 37 | mode: bridge 38 | command: 39 | - /ndt-server 40 | - -cert=/certs/cert.pem 41 | - -key=/certs/key.pem 42 | - -datadir=/resultsdir/ndt 43 | - -ndt7_addr=:4443 44 | - -ndt7_addr_cleartext=:8080 45 | - -compress-results=false 46 | - -prometheusx.listen-address=:9990 47 | 48 | jostler: 49 | image: measurementlab/jostler:v1.0.7 50 | volumes: 51 | - ./resultsdir:/resultsdir 52 | - ./schemas:/schemas 53 | - ./localgcs:/localgcs 54 | network_mode: "service:ndt-server" 55 | depends_on: 56 | generate-schemas: 57 | condition: service_completed_successfully 58 | command: 59 | - -gcs-local-disk 60 | - -mlab-node-name=ndt-mlab1-lga01.mlab-sandbox.measurement-lab.org 61 | - -gcs-bucket=newclient,download,upload 62 | - -gcs-data-dir=/localgcs/autoload/v1 63 | - -local-data-dir=/resultsdir 64 | - -experiment=ndt 65 | - -datatype=ndt7 66 | - -datatype-schema-file=ndt7:/schemas/ndt7.json 67 | - -bundle-size-max=81920 68 | - -bundle-age-max=10s 69 | - -missed-age=20s 70 | - -missed-interval=15s 71 | - -verbose 72 | - -prometheusx.listen-address=:9991 73 | 74 | generate-schemas: 75 | image: ndt-server 76 | build: 77 | context: . 78 | dockerfile: Dockerfile.local 79 | volumes: 80 | - ./schemas:/schemas 81 | entrypoint: 82 | - /go/bin/generate-schemas 83 | - -ndt7=/schemas/ndt7.json 84 | - -ndt5=/schemas/ndt5.json 85 | -------------------------------------------------------------------------------- /data/result.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/m-lab/ndt-server/ndt5/c2s" 7 | "github.com/m-lab/ndt-server/ndt5/control" 8 | "github.com/m-lab/ndt-server/ndt5/s2c" 9 | 10 | "github.com/m-lab/ndt-server/ndt7/model" 11 | ) 12 | 13 | // NDTResult is preserved for legacy compatibility with an older unified version 14 | // of the NDT5 and NDT7 result structures below. 15 | // TODO(github.com/m-lab/ndt-server/issues/260) remove this alias once no one uses it. 16 | type NDTResult = NDT5Result 17 | 18 | // NDT5Result is the struct that is serialized as JSON to disk as the archival 19 | // record of an NDT test. 20 | // 21 | // This struct is dual-purpose. It contains the necessary data to allow joining 22 | // with tcp-info data and traceroute-caller data as well as any other UUID-based 23 | // data. It also contains enough data for interested parties to perform 24 | // lightweight data analysis without needing to join with other tools. 25 | // 26 | // WARNING: The BigQuery schema is inferred directly from this structure. To 27 | // preserve compatibility with historical data, never remove fields. 28 | // For more information see: https://github.com/m-lab/etl/issues/719 29 | type NDT5Result struct { 30 | // GitShortCommit is the Git commit (short form) of the running server code. 31 | GitShortCommit string 32 | // Version is the symbolic version (if any) of the running server code. 33 | Version string 34 | 35 | // All data members should all be self-describing. In the event of confusion, 36 | // rename them to add clarity rather than adding a comment. 37 | ServerIP string 38 | ServerPort int 39 | ClientIP string 40 | ClientPort int 41 | 42 | StartTime time.Time 43 | EndTime time.Time 44 | 45 | // ndt5 46 | Control *control.ArchivalData `json:",omitempty"` 47 | C2S *c2s.ArchivalData `json:",omitempty"` 48 | S2C *s2c.ArchivalData `json:",omitempty"` 49 | } 50 | 51 | // NDT7Result is the struct that is serialized as JSON to disk as the archival 52 | // record of an NDT7 test. This is similar to, but independent from, the NDT5Result. 53 | type NDT7Result struct { 54 | // GitShortCommit is the Git commit (short form) of the running server code. 55 | GitShortCommit string 56 | // Version is the symbolic version (if any) of the running server code. 57 | Version string 58 | 59 | // All data members should all be self-describing. In the event of confusion, 60 | // rename them to add clarity rather than adding a comment. 61 | ServerIP string 62 | ServerPort int 63 | ClientIP string 64 | ClientPort int 65 | 66 | StartTime time.Time 67 | EndTime time.Time 68 | 69 | // ndt7 70 | Upload *model.ArchivalData `json:",omitempty"` 71 | Download *model.ArchivalData `json:",omitempty"` 72 | } 73 | -------------------------------------------------------------------------------- /ndt7/upload/sender/sender.go: -------------------------------------------------------------------------------- 1 | // Package sender implements the upload sender. 2 | package sender 3 | 4 | import ( 5 | "context" 6 | "time" 7 | 8 | "github.com/gorilla/websocket" 9 | "github.com/m-lab/ndt-server/logging" 10 | "github.com/m-lab/ndt-server/ndt7/closer" 11 | "github.com/m-lab/ndt-server/ndt7/measurer" 12 | ndt7metrics "github.com/m-lab/ndt-server/ndt7/metrics" 13 | "github.com/m-lab/ndt-server/ndt7/model" 14 | "github.com/m-lab/ndt-server/ndt7/ping" 15 | "github.com/m-lab/ndt-server/ndt7/spec" 16 | ) 17 | 18 | // Start sends measurement messages (status messages) to the client conn. Each 19 | // measurement message will also be saved to data. 20 | // 21 | // Liveness guarantee: the sender will not be stuck sending for more than the 22 | // MaxRuntime of the subtest. This is enforced by setting the write deadline to 23 | // Time.Now() + MaxRuntime. 24 | func Start(ctx context.Context, conn *websocket.Conn, data *model.ArchivalData) error { 25 | logging.Logger.Debug("sender: start") 26 | proto := ndt7metrics.ConnLabel(conn) 27 | 28 | // Start collecting connection measurements. Measurements will be sent to 29 | // src until DefaultRuntime, when the src channel is closed. 30 | mr := measurer.New(conn, data.UUID) 31 | src := mr.Start(ctx, spec.DefaultRuntime) 32 | defer logging.Logger.Debug("sender: stop") 33 | defer mr.Stop(src) 34 | 35 | deadline := time.Now().Add(spec.MaxRuntime) 36 | err := conn.SetWriteDeadline(deadline) // Liveness! 37 | if err != nil { 38 | logging.Logger.WithError(err).Warn("sender: conn.SetWriteDeadline failed") 39 | ndt7metrics.ClientSenderErrors.WithLabelValues( 40 | proto, string(spec.SubtestUpload), "set-write-deadline").Inc() 41 | return err 42 | } 43 | 44 | // Record measurement start time, and prepare recording of the endtime on return. 45 | data.StartTime = time.Now().UTC() 46 | defer func() { 47 | data.EndTime = time.Now().UTC() 48 | }() 49 | for { 50 | m, ok := <-src 51 | if !ok { // This means that the previous step has terminated 52 | closer.StartClosing(conn) 53 | ndt7metrics.ClientSenderErrors.WithLabelValues( 54 | proto, string(spec.SubtestUpload), "measurer-closed").Inc() 55 | return nil 56 | } 57 | if err := conn.WriteJSON(m); err != nil { 58 | logging.Logger.WithError(err).Warn("sender: conn.WriteJSON failed") 59 | ndt7metrics.ClientSenderErrors.WithLabelValues( 60 | proto, string(spec.SubtestUpload), "write-json").Inc() 61 | return err 62 | } 63 | // Only save measurements sent to the client. 64 | data.ServerMeasurements = append(data.ServerMeasurements, m) 65 | if err := ping.SendTicks(conn, deadline); err != nil { 66 | logging.Logger.WithError(err).Warn("sender: ping.SendTicks failed") 67 | ndt7metrics.ClientSenderErrors.WithLabelValues( 68 | proto, string(spec.SubtestUpload), "ping-send-ticks").Inc() 69 | return err 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /fullstack/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Starts the ndt-server binary running with all its associated supporting 4 | # services set up just right. If you just want to run a server that speaks the 5 | # unencrypted NDT5 (legacy) protocol, then you can run: 6 | # $ docker run --net=host measurementlab/ndt 7 | # and you will get an NDT server running on port 3001, with data being saved to 8 | # the in-container directory /var/spool/ndt/ 9 | # 10 | # If you would like to run NDT7 tests (which you should, it is a simpler 11 | # protocol and a more robust measurement), then you will need a private key and 12 | # a TLS certificate (let's assume they are called "/etc/certs/key.pem" and 13 | # "/etc/certs/cert.pem"). To run an NDT7 server on port 443, you can do: 14 | # $ docker run -v /etc/certs:/certs --net=host measurementlab/ndt \ 15 | # --key=/certs/key.pem --cert=/certs/cert.pem 16 | # 17 | # The NDT server produces data on disk. If you would like this data saved to a 18 | # directory outside of the docker container, then you need to mount the external 19 | # directory inside the container at /var/spool/ndt using the -v argument to 20 | # "docker run". 21 | # 22 | # All arguments to this script are passed directly through to ndt-server. 23 | 24 | set -euxo pipefail 25 | 26 | 27 | # Set up the filesystem. 28 | 29 | # Set up UUIDs to have a common race-free prefix. 30 | UUID_FILE=$(mktemp /tmp/uuidprefix.XXXXXX) 31 | /create-uuid-prefix-file --filename="${UUID_FILE}" 32 | 33 | # Set up the data directory. 34 | DATA_DIR=/var/spool/ndt 35 | mkdir -p "${DATA_DIR}" 36 | 37 | 38 | # Start all services. 39 | 40 | # Start the tcp-info logging service. 41 | mkdir -p "${DATA_DIR}"/tcpinfo 42 | /tcp-info \ 43 | --prometheusx.listen-address=:9991 \ 44 | --uuid-prefix-file="${UUID_FILE}" \ 45 | --output="${DATA_DIR}"/tcpinfo \ 46 | --tcpinfo.eventsocket=/var/local/tcpeventsocket.sock \ 47 | & 48 | 49 | echo "Waiting for the tcpinfo eventsocket to become available..." 50 | while ! socat -u OPEN:/dev/null UNIX-CONNECT:/var/local/tcpeventsocket.sock; do 51 | sleep 1 52 | done 53 | 54 | # Start the traceroute service. 55 | mkdir -p "${DATA_DIR}"/traceroute 56 | /traceroute-caller \ 57 | --prometheusx.listen-address=:9992 \ 58 | --uuid-prefix-file="${UUID_FILE}" \ 59 | --hopannotation-output="${DATA_DIR}"/hopannotation1 \ 60 | --traceroute-output="${DATA_DIR}"/scamper1 \ 61 | --tcpinfo.eventsocket=/var/local/tcpeventsocket.sock \ 62 | & 63 | 64 | # Start packet header capture. 65 | mkdir -p "${DATA_DIR}"/pcap 66 | /packet-headers \ 67 | --prometheusx.listen-address=:9993 \ 68 | --datadir="${DATA_DIR}"/pcap \ 69 | --tcpinfo.eventsocket=/var/local/tcpeventsocket.sock \ 70 | & 71 | 72 | 73 | # Start the NDT server. 74 | /ndt-server \ 75 | --uuid-prefix-file="${UUID_FILE}" \ 76 | --datadir="${DATA_DIR}" \ 77 | $* 78 | & 79 | 80 | # Any of the backgrounded processes dying should kill the whole thing. 81 | wait -n 82 | -------------------------------------------------------------------------------- /ndt5/README.md: -------------------------------------------------------------------------------- 1 | # NDT5 ndt-server code 2 | 3 | All code in this directory tree is related to the support of the legacy NDT5 4 | protocol. We have many extant clients that use this protocol, and we don't 5 | want to leave them high and dry, but new clients are encouraged to use the 6 | services provided by ndt7. The test is streamlined, the client is easier to 7 | write, and basically everything about it is better. 8 | 9 | In this subtree, we support existing clients, but we will be adding no new 10 | functionality. If you are reading this and trying to decide how to implement 11 | a speed test, use ndt7 and not the legacy, ndt5 protocol. The legacy protocol is 12 | deprecated. It will be supported until usage drops to very low levels, but it 13 | is also not recommended for new integrations or code. 14 | 15 | ## NDT5 Metrics 16 | 17 | Summary of metrics useful for monitoring client request, success, and error rates. 18 | 19 | * `ndt5_control_total{protocol, result}` counts every client connection 20 | that reaches `HandleControlChannel`. 21 | 22 | * The "protocol=" label matches the client protocol, e.g., "WS", "WSS", or 23 | "PLAIN". 24 | * The "result=" label is either "okay" or "panic". 25 | * All result="panic" results also count specific causes in 26 | `ndt5_control_panic_total`. 27 | * All result="okay" results come from "protocol complete" clients. 28 | 29 | * `ndt5_client_test_requested_total{protocol, direction}` counts 30 | client-requested tests. 31 | 32 | * The "protocol=" label is the same as above. 33 | * The "direction=" label will have values like "c2s" and "s2c". 34 | * If the client continues the test, then the result will be counted in 35 | `ndt5_client_test_results_total`. 36 | 37 | * `ndt5_client_test_results_total{protocol, direction, result}` counts the 38 | results of client-requested tests. 39 | 40 | * The "protocol=" and "direction=" labels are as above. 41 | * The "result=" label is either "okay-with-rate", "error-with-rate" or 42 | "error-without-rate". 43 | * All result="okay-with-rate" count all "protocol complete" clients up to that 44 | point. 45 | * All result=~"error-.*" results also count specific causes in 46 | `ndt5_client_test_errors_total`. 47 | 48 | * `ndt5_client_test_errors_total{protocol, direction, error}` 49 | 50 | * The "protocol=" and "direction=" labels are as above. 51 | * The "error=" label contains unique values mapping to specific error paths in 52 | the ndt-server. 53 | 54 | Expected invariants: 55 | 56 | * `sum(ndt5_control_channel_duration_count) == sum(ndt5_control_total)` 57 | * `sum(ndt5_control_total{result="panic"}) == sum(ndt5_control_panic_total)` 58 | * `sum(ndt5_client_test_results_total{result=~"error.*"}) == sum(ndt5_client_test_errors_total)` 59 | 60 | NOTE: 61 | 62 | * `ndt5_client_test_results_total` may be less than `ndt5_client_test_requested_total` 63 | if the client hangs up before the test can run. 64 | -------------------------------------------------------------------------------- /cmd/generate-schemas/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/m-lab/ndt-server/cmd/generate-schemas 2 | 3 | go 1.20 4 | 5 | require ( 6 | cloud.google.com/go/bigquery v1.49.0 7 | github.com/m-lab/go v0.1.66 8 | github.com/m-lab/ndt-server v0.20.17 9 | ) 10 | 11 | require ( 12 | cloud.google.com/go v0.110.0 // indirect 13 | cloud.google.com/go/compute v1.18.0 // indirect 14 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 15 | cloud.google.com/go/iam v0.12.0 // indirect 16 | github.com/andybalholm/brotli v1.0.4 // indirect 17 | github.com/apache/arrow/go/v11 v11.0.0 // indirect 18 | github.com/apache/thrift v0.16.0 // indirect 19 | github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 // indirect 20 | github.com/beorn7/perks v1.0.1 // indirect 21 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 22 | github.com/goccy/go-json v0.9.11 // indirect 23 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect 24 | github.com/golang/protobuf v1.5.2 // indirect 25 | github.com/golang/snappy v0.0.4 // indirect 26 | github.com/google/flatbuffers v2.0.8+incompatible // indirect 27 | github.com/google/go-cmp v0.5.9 // indirect 28 | github.com/google/uuid v1.3.0 // indirect 29 | github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect 30 | github.com/googleapis/gax-go/v2 v2.7.0 // indirect 31 | github.com/gorilla/websocket v1.5.0 // indirect 32 | github.com/klauspost/asmfmt v1.3.2 // indirect 33 | github.com/klauspost/compress v1.15.9 // indirect 34 | github.com/klauspost/cpuid/v2 v2.0.9 // indirect 35 | github.com/m-lab/tcp-info v1.5.3 // indirect 36 | github.com/m-lab/uuid v1.0.1 // indirect 37 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 38 | github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect 39 | github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect 40 | github.com/pierrec/lz4/v4 v4.1.15 // indirect 41 | github.com/prometheus/client_golang v1.13.0 // indirect 42 | github.com/prometheus/client_model v0.2.0 // indirect 43 | github.com/prometheus/common v0.37.0 // indirect 44 | github.com/prometheus/procfs v0.8.0 // indirect 45 | github.com/zeebo/xxh3 v1.0.2 // indirect 46 | go.opencensus.io v0.24.0 // indirect 47 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect 48 | golang.org/x/net v0.7.0 // indirect 49 | golang.org/x/oauth2 v0.5.0 // indirect 50 | golang.org/x/sync v0.1.0 // indirect 51 | golang.org/x/sys v0.5.0 // indirect 52 | golang.org/x/text v0.7.0 // indirect 53 | golang.org/x/tools v0.1.12 // indirect 54 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect 55 | google.golang.org/api v0.111.0 // indirect 56 | google.golang.org/appengine v1.6.7 // indirect 57 | google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect 58 | google.golang.org/grpc v1.53.0 // indirect 59 | google.golang.org/protobuf v1.28.1 // indirect 60 | gopkg.in/yaml.v2 v2.4.0 // indirect 61 | ) 62 | -------------------------------------------------------------------------------- /bbr/bbr_linux.go: -------------------------------------------------------------------------------- 1 | package bbr 2 | 3 | // #include
28 |
31 |
90 |
91 |
92 |
93 |
--------------------------------------------------------------------------------
/spec/data-format.md:
--------------------------------------------------------------------------------
1 | # Data Format
2 |
3 | This specification describes how ndt-server serializes ndt7 data
4 | on disk. Other implementations of the ndt7 protocol MAY use other
5 | data serialization formats.
6 |
7 | This is version v0.3.0 of the data-format specification.
8 |
9 | For each subtest, ndt7 writes to the current working directory a Gzip
10 | compressed JSON file. The file name MUST match the following pattern:
11 |
12 | ```
13 | ndt7-[Download]
29 | [Upload]
30 |
22 |
23 |
24 |
132 |
133 |
25 |
47 |
26 |
27 |
40 | The Network Diagnostic Tool (NDT) provides a sophisticated speed and diagnostic test. An NDT test reports more than just the upload and download speeds — it also attempts to determine what, if any, problems limited these speeds, differentiating between computer configuration and network infrastructure problems. While the diagnostic messages are most useful for expert users, they can also help novice users by allowing them to provide detailed trouble reports to their network administrator.
28 | 29 |This NDT test uses WebSockets. Most modern browsers should support running this test. If you are experiencing problems, make sure that your browser has javascript enabled and supports WebSockets.
30 | 31 |
32 | Learn more about NDT – Learn more about Measurement Lab
33 |
34 |
35 |
36 |
38 |
39 | Your browser does not appear to support WebSockets. Please check your browser configuration or try a different browser.
37 |
41 | Initializing...
42 |
43 |
44 | Start Test
45 |
46 |
48 |
49 |
77 |
50 |
52 | Preparing your tests...
51 |
53 |
55 | Now testing your upload speed
54 |
56 |
58 |
59 |
60 | Now testing your download speed
57 |RTT
61 |
62 |
63 |
64 |
65 |
66 |
75 |
76 |
67 |
70 | Your computer
68 |
69 |
71 |
74 | Test server
72 |
73 |
78 |
79 |
131 | Your test results
80 | 81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
118 |
119 |
90 |
91 |
112 |
113 |
114 |
115 |
116 |
117 |
92 |
98 |
99 | Upload speed
93 |94 | 0.00 95 | mb/s 96 |
97 |
100 |
105 |
106 | Download speed
101 |102 | 0.00 103 | mb/s 104 |
107 |
110 |
111 | Network latency: 0.0 msec round trip time
108 |Jitter: 0.0 msec
109 |
120 |
124 |
125 |
126 |
127 | Test again
128 |
129 |
130 |
134 |
135 |
136 |
137 |
138 |
139 |
--------------------------------------------------------------------------------
/ndt5/protocol/messager.go:
--------------------------------------------------------------------------------
1 | package protocol
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 | "log"
8 | "reflect"
9 | "strconv"
10 | )
11 |
12 | // Encoding encodes the communication methods we support.
13 | type Encoding int
14 |
15 | // The different message types we support. This is initially Unknown for plain
16 | // ndt5 connections and becomes JSON or TLV depending on the whether we
17 | // receive MsgLogin or MsgExtendedLogin, but is always JSON for WS and WSS.
18 | const (
19 | Unknown Encoding = iota // Unknown is the zero-value for Encoding
20 | JSON
21 | TLV
22 | )
23 |
24 | func (e Encoding) String() string {
25 | switch e {
26 | case Unknown:
27 | return "Unknown"
28 | case JSON:
29 | return "JSON"
30 | case TLV:
31 | return "TLV"
32 | }
33 | return fmt.Sprintf("Bad Encoding value: %d", int(e))
34 | }
35 |
36 | // Messager creates an object that can encode and decode messages in the
37 | // corresponding format and send them along the passed-in connection.
38 | func (e Encoding) Messager(conn Connection) Messager {
39 | switch e {
40 | case Unknown:
41 | log.Println("Error: Messager() called for Unknown type")
42 | return nil
43 | case JSON:
44 | return &jsonMessager{conn}
45 | case TLV:
46 | return &tlvMessager{conn}
47 | }
48 | log.Printf("Bad Encoding value: %d\n", int(e))
49 | return nil
50 | }
51 |
52 | // Messager allows us to send JSON and non-JSON messages using a single unified
53 | // interface.
54 | type Messager interface {
55 | SendMessage(MessageType, []byte) error
56 | SendS2CResults(throughputKbps, unsentBytes, totalSentBytes int64) error
57 | ReceiveMessage(MessageType) ([]byte, error)
58 | Encoding() Encoding
59 | }
60 |
61 | // jsonMessager has all the methods for sending JSON-format NDT messages along
62 | // the passed-in connection.
63 | type jsonMessager struct {
64 | conn Connection
65 | }
66 |
67 | type s2cResult struct {
68 | ThroughputValue string
69 | UnsentDataAmount string
70 | TotalSentByte string
71 | }
72 |
73 | func (r *s2cResult) String() string {
74 | b, _ := json.Marshal(r)
75 | return string(b)
76 | }
77 |
78 | func (jm *jsonMessager) SendMessage(kind MessageType, contents []byte) error {
79 | return SendJSONMessage(kind, string(contents), jm.conn)
80 | }
81 |
82 | func (jm *jsonMessager) SendS2CResults(throughputKbps, unsentBytes, totalSentBytes int64) error {
83 | r := &s2cResult{
84 | ThroughputValue: strconv.FormatInt(throughputKbps, 10),
85 | UnsentDataAmount: strconv.FormatInt(unsentBytes, 10),
86 | TotalSentByte: strconv.FormatInt(totalSentBytes, 10),
87 | }
88 | return WriteTLVMessage(jm.conn, TestMsg, r.String())
89 | }
90 |
91 | func (jm *jsonMessager) ReceiveMessage(kind MessageType) ([]byte, error) {
92 | msg, err := ReceiveJSONMessage(jm.conn, kind)
93 | if msg == nil {
94 | if err == nil {
95 | return nil, errors.New("empty message received without error")
96 | }
97 | return nil, err
98 | }
99 | return []byte(msg.Msg), err
100 | }
101 |
102 | func (jm *jsonMessager) Encoding() Encoding {
103 | return JSON
104 | }
105 |
106 | // tlvMessager has all the methods for sending tlv-format NDT messages along the
107 | // passed-in connection.
108 | type tlvMessager struct {
109 | conn Connection
110 | }
111 |
112 | func (tm *tlvMessager) SendMessage(kind MessageType, contents []byte) error {
113 | return WriteTLVMessage(tm.conn, kind, string(contents))
114 | }
115 |
116 | func (tm *tlvMessager) SendS2CResults(throughputKbps, unsentBytes, totalSentBytes int64) error {
117 | msg := fmt.Sprintf("%d %d %d", throughputKbps, unsentBytes, totalSentBytes)
118 | return WriteTLVMessage(tm.conn, TestMsg, msg)
119 | }
120 |
121 | func (tm *tlvMessager) ReceiveMessage(kind MessageType) ([]byte, error) {
122 | b, _, err := ReadTLVMessage(tm.conn, kind)
123 | return b, err
124 | }
125 |
126 | func (tm *tlvMessager) Encoding() Encoding {
127 | return TLV
128 | }
129 |
130 | // SendMetrics sends all the required properties out along the NDT control channel.
131 | func SendMetrics(metrics interface{}, m Messager, prefix string) error {
132 | v := reflect.ValueOf(metrics)
133 | t := v.Type()
134 | // Dereference all passed-in pointers
135 | for t.Kind() == reflect.Ptr {
136 | v = v.Elem()
137 | t = v.Type()
138 | }
139 | for i := 0; i < v.NumField(); i++ {
140 | name := t.Field(i).Name
141 | switch t.Field(i).Type.Kind() {
142 | case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
143 | msg := fmt.Sprintf("%s%s: %v\n", prefix, name, v.Field(i).Interface())
144 | err := m.SendMessage(TestMsg, []byte(msg))
145 | if err != nil {
146 | return err
147 | }
148 | case reflect.String:
149 | msg := fmt.Sprintf("%s%s: %s\n", prefix, name, v.Field(i).String())
150 | err := m.SendMessage(TestMsg, []byte(msg))
151 | if err != nil {
152 | return err
153 | }
154 | case reflect.Struct:
155 | data := v.Field(i).Interface()
156 | var err error
157 | if s, ok := data.(fmt.Stringer); ok {
158 | msg := fmt.Sprintf("%s%s: %s\n", prefix, name, s.String())
159 | err = m.SendMessage(TestMsg, []byte(msg))
160 | } else {
161 | err = SendMetrics(v.Field(i).Interface(), m, prefix+name+".")
162 | }
163 | if err != nil {
164 | return err
165 | }
166 | default:
167 | log.Println("Unhandled case in SendMetrics:", t.Field(i).Type.Kind())
168 | }
169 | }
170 | return nil
171 | }
172 |
--------------------------------------------------------------------------------
/ndt7/download/sender/sender.go:
--------------------------------------------------------------------------------
1 | // Package sender implements the download sender.
2 | package sender
3 |
4 | import (
5 | "context"
6 | "math/rand"
7 | "time"
8 |
9 | "github.com/gorilla/websocket"
10 | "github.com/m-lab/ndt-server/logging"
11 | "github.com/m-lab/ndt-server/ndt7/closer"
12 | "github.com/m-lab/ndt-server/ndt7/measurer"
13 | ndt7metrics "github.com/m-lab/ndt-server/ndt7/metrics"
14 | "github.com/m-lab/ndt-server/ndt7/model"
15 | "github.com/m-lab/ndt-server/ndt7/ping"
16 | "github.com/m-lab/ndt-server/ndt7/spec"
17 | )
18 |
19 | // Params defines the parameters for the sender to end the test early.
20 | type Params struct {
21 | IsEarlyExit bool
22 | MaxBytes int64
23 | }
24 |
25 | func makePreparedMessage(size int) (*websocket.PreparedMessage, error) {
26 | data := make([]byte, size)
27 | _, err := rand.Read(data)
28 | if err != nil {
29 | return nil, err
30 | }
31 | return websocket.NewPreparedMessage(websocket.BinaryMessage, data)
32 | }
33 |
34 | // Start sends binary messages (bulk download) and measurement messages (status
35 | // messages) to the client conn. Each measurement message will also be saved to
36 | // data.
37 | //
38 | // Liveness guarantee: the sender will not be stuck sending for more than the
39 | // MaxRuntime of the subtest. This is enforced by setting the write deadline to
40 | // Time.Now() + MaxRuntime.
41 | func Start(ctx context.Context, conn *websocket.Conn, data *model.ArchivalData, params *Params) error {
42 | logging.Logger.Debug("sender: start")
43 | proto := ndt7metrics.ConnLabel(conn)
44 |
45 | // Start collecting connection measurements. Measurements will be sent to
46 | // src until DefaultRuntime, when the src channel is closed.
47 | mr := measurer.New(conn, data.UUID)
48 | src := mr.Start(ctx, spec.DefaultRuntime)
49 | defer logging.Logger.Debug("sender: stop")
50 | defer mr.Stop(src)
51 |
52 | logging.Logger.Debug("sender: generating random buffer")
53 | bulkMessageSize := 1 << 13
54 | preparedMessage, err := makePreparedMessage(bulkMessageSize)
55 | if err != nil {
56 | logging.Logger.WithError(err).Warn("sender: makePreparedMessage failed")
57 | ndt7metrics.ClientSenderErrors.WithLabelValues(
58 | proto, string(spec.SubtestDownload), "make-prepared-message").Inc()
59 | return err
60 | }
61 | deadline := time.Now().Add(spec.MaxRuntime)
62 | err = conn.SetWriteDeadline(deadline) // Liveness!
63 | if err != nil {
64 | logging.Logger.WithError(err).Warn("sender: conn.SetWriteDeadline failed")
65 | ndt7metrics.ClientSenderErrors.WithLabelValues(
66 | proto, string(spec.SubtestDownload), "set-write-deadline").Inc()
67 | return err
68 | }
69 |
70 | // Record measurement start time, and prepare recording of the endtime on return.
71 | data.StartTime = time.Now().UTC()
72 | defer func() {
73 | data.EndTime = time.Now().UTC()
74 | }()
75 | var totalSent int64
76 | for {
77 | select {
78 | case m, ok := <-src:
79 | if !ok { // This means that the measurer has terminated.
80 | closer.StartClosing(conn)
81 | ndt7metrics.ClientSenderErrors.WithLabelValues(
82 | proto, string(spec.SubtestDownload), "measurer-closed").Inc()
83 | return nil
84 | }
85 | if err := conn.WriteJSON(m); err != nil {
86 | logging.Logger.WithError(err).Warn("sender: conn.WriteJSON failed")
87 | ndt7metrics.ClientSenderErrors.WithLabelValues(
88 | proto, string(spec.SubtestDownload), "write-json").Inc()
89 | return err
90 | }
91 | // Only save measurements sent to the client.
92 | data.ServerMeasurements = append(data.ServerMeasurements, m)
93 | if err := ping.SendTicks(conn, deadline); err != nil {
94 | logging.Logger.WithError(err).Warn("sender: ping.SendTicks failed")
95 | ndt7metrics.ClientSenderErrors.WithLabelValues(
96 | proto, string(spec.SubtestDownload), "ping-send-ticks").Inc()
97 | return err
98 | }
99 | // End the test once enough bytes have been acked.
100 | if params.IsEarlyExit && m.TCPInfo != nil &&
101 | m.TCPInfo.BytesAcked >= params.MaxBytes {
102 | closer.StartClosing(conn)
103 | ndt7metrics.ClientSenderErrors.WithLabelValues(
104 | proto, string(spec.SubtestDownload), "measurer-closed-early").Inc()
105 | return nil
106 | }
107 | default:
108 | if err := conn.WritePreparedMessage(preparedMessage); err != nil {
109 | logging.Logger.WithError(err).Warn(
110 | "sender: conn.WritePreparedMessage failed")
111 | ndt7metrics.ClientSenderErrors.WithLabelValues(
112 | proto, string(spec.SubtestDownload), "write-prepared-message").Inc()
113 | return err
114 | }
115 | // The following block of code implements the scaling of message size
116 | // as recommended in the spec's appendix. We're not accounting for the
117 | // size of JSON messages because that is small compared to the bulk
118 | // message size. The net effect is slightly slowing down the scaling,
119 | // but this is currently fine. We need to gather data from large
120 | // scale deployments of this algorithm anyway, so there's no point
121 | // in engaging in fine grained calibration before knowing.
122 | totalSent += int64(bulkMessageSize)
123 | if int64(bulkMessageSize) >= spec.MaxScaledMessageSize {
124 | continue // No further scaling is required
125 | }
126 | if int64(bulkMessageSize) > totalSent/spec.ScalingFraction {
127 | continue // message size still too big compared to sent data
128 | }
129 | bulkMessageSize *= 2
130 | preparedMessage, err = makePreparedMessage(bulkMessageSize)
131 | if err != nil {
132 | logging.Logger.WithError(err).Warn("sender: makePreparedMessage failed")
133 | ndt7metrics.ClientSenderErrors.WithLabelValues(
134 | proto, string(spec.SubtestDownload), "make-prepared-message").Inc()
135 | return err
136 | }
137 | }
138 | }
139 | }
140 |
--------------------------------------------------------------------------------
/html/ndt7-upload-worker.js:
--------------------------------------------------------------------------------
1 | /* eslint-env es6, browser, node, worker */
2 |
3 | // WebWorker that runs the ndt7 upload test
4 | const workerMain = function(ev) {
5 | const url = ev.data['///ndt/v7/upload'];
6 | const sock = new WebSocket(url, 'net.measurementlab.ndt.v7');
7 | let now;
8 | if (typeof performance !== 'undefined' &&
9 | typeof performance.now === 'function') {
10 | now = () => performance.now();
11 | } else {
12 | now = () => Date.now();
13 | }
14 | uploadTest(sock, postMessage, now);
15 | };
16 |
17 | const uploadTest = function(sock, postMessage, now) {
18 | let closed = false;
19 | sock.onclose = function() {
20 | if (!closed) {
21 | closed = true;
22 | postMessage({
23 | MsgType: 'complete',
24 | });
25 | }
26 | };
27 |
28 | sock.onerror = function(ev) {
29 | postMessage({
30 | MsgType: 'error',
31 | Error: ev.type,
32 | });
33 | };
34 |
35 | sock.onmessage = function(ev) {
36 | if (typeof ev.data !== 'undefined') {
37 | postMessage({
38 | MsgType: 'measurement',
39 | Source: 'server',
40 | ServerMessage: ev.data,
41 | });
42 | }
43 | };
44 |
45 | /**
46 | * uploader is the main loop that uploads data in the web browser. It must
47 | * carefully balance a bunch of factors:
48 | * 1) message size determines measurement granularity on the client side,
49 | * 2) the JS event loop can only fire off so many times per second, and
50 | * 3) websocket buffer tracking seems inconsistent between browsers.
51 | *
52 | * Because of (1), we need to have small messages on slow connections, or
53 | * else this will not accurately measure slow connections. Because of (2), if
54 | * we use small messages on fast connections, then we will not fill the link.
55 | * Because of (3), we can't depend on the websocket buffer to "fill up" in a
56 | * reasonable amount of time.
57 | *
58 | * So on fast connections we need a big message size (one the message has
59 | * been handed off to the browser, it runs on the browser's fast compiled
60 | * internals) and on slow connections we need a small message. Because this
61 | * is used as a speed test, we don't know before the test which strategy we
62 | * will be using, because we don't know the speed before we test it.
63 | * Therefore, we use a strategy where we grow the message exponentially over
64 | * time. In an effort to be kind to the memory allocator, we always double
65 | * the message size instead of growing it by e.g. 1.3x.
66 | *
67 | * @param {*} data
68 | * @param {*} start
69 | * @param {*} end
70 | * @param {*} previous
71 | * @param {*} total
72 | */
73 | function uploader(data, start, end, previous, total) {
74 | if (closed) {
75 | // socket.send() with too much buffering causes socket.close(). We only
76 | // observed this behaviour with pre-Chromium Edge.
77 | return;
78 | }
79 | const t = now();
80 | if (t >= end) {
81 | sock.close();
82 | // send one last measurement.
83 | postClientMeasurement(total, sock.bufferedAmount, start);
84 | return;
85 | }
86 |
87 | const maxMessageSize = 8388608; /* = (1<<23) = 8MB */
88 | const clientMeasurementInterval = 250; // ms
89 |
90 | // Message size is doubled after the first 16 messages, and subsequently
91 | // every 8, up to maxMessageSize.
92 | const nextSizeIncrement =
93 | (data.length >= maxMessageSize) ? Infinity : 16 * data.length;
94 | if ((total - sock.bufferedAmount) >= nextSizeIncrement) {
95 | data = new Uint8Array(data.length * 2);
96 | }
97 |
98 | // We keep 7 messages in the send buffer, so there is always some more
99 | // data to send. The maximum buffer size is 8 * 8MB - 1 byte ~= 64M.
100 | const desiredBuffer = 7 * data.length;
101 | if (sock.bufferedAmount < desiredBuffer) {
102 | sock.send(data);
103 | total += data.length;
104 | }
105 |
106 | if (t >= previous + clientMeasurementInterval) {
107 | postClientMeasurement(total, sock.bufferedAmount, start);
108 | previous = t;
109 | }
110 |
111 | // Loop the uploader function in a way that respects the JS event handler.
112 | setTimeout(() => uploader(data, start, end, previous, total), 0);
113 | }
114 |
115 | /** Report measurement back to the main thread.
116 | *
117 | * @param {*} total
118 | * @param {*} bufferedAmount
119 | * @param {*} start
120 | */
121 | function postClientMeasurement(total, bufferedAmount, start) {
122 | // bytes sent - bytes buffered = bytes actually sent
123 | const numBytes = total - bufferedAmount;
124 | // ms / 1000 = seconds
125 | const elapsedTime = (now() - start) / 1000;
126 | // bytes * bits/byte * megabits/bit * 1/seconds = Mbps
127 | const meanMbps = numBytes * 8 / 1000000 / elapsedTime;
128 | postMessage({
129 | MsgType: 'measurement',
130 | ClientData: {
131 | ElapsedTime: elapsedTime,
132 | NumBytes: numBytes,
133 | MeanClientMbps: meanMbps,
134 | },
135 | Source: 'client',
136 | Test: 'upload',
137 | });
138 | }
139 |
140 | sock.onopen = function() {
141 | const initialMessageSize = 8192; /* (1<<13) = 8kBytes */
142 | // TODO(bassosimone): fill this message - see above comment
143 | const data = new Uint8Array(initialMessageSize);
144 | const start = now(); // ms since epoch
145 | const duration = 10000; // ms
146 | const end = start + duration; // ms since epoch
147 |
148 | postMessage({
149 | MsgType: 'start',
150 | Data: {
151 | StartTime: start / 1000, // seconds since epoch
152 | ExpectedEndTime: end / 1000, // seconds since epoch
153 | },
154 | });
155 |
156 | // Start the upload loop.
157 | uploader(data, start, end, start, 0);
158 | };
159 | };
160 |
161 | // Node and browsers get onmessage defined differently.
162 | if (typeof self !== 'undefined') {
163 | self.onmessage = workerMain;
164 | } else if (typeof this !== 'undefined') {
165 | this.onmessage = workerMain;
166 | } else if (typeof onmessage !== 'undefined') {
167 | onmessage = workerMain;
168 | }
169 |
--------------------------------------------------------------------------------
/html/ndt-wrapper.js:
--------------------------------------------------------------------------------
1 | /* This is an NDT client, written in javascript. It speaks the websocket
2 | * version of the NDT protocol. The NDT protocol is documented at:
3 | * https://code.google.com/p/ndt/wiki/NDTProtocol
4 | */
5 |
6 | /*jslint bitwise: true, browser: true, nomen: true, vars: true */
7 | /*global Uint8Array, WebSocket */
8 |
9 | 'use strict';
10 |
11 | function NDTWrapper(server) {
12 | var _this = this;
13 |
14 | // XXX: include a way to override this
15 | this.use_web_worker = false;
16 |
17 | /* If they have Web Workers, use it to do the NDT test, unless it's Firefox
18 | * which doesn't support web workers using websockets. This should be
19 | * migrated into something more sane (e.g. a function that spawns an inline worker that
20 | * creates a websocket, and returns true/false if that succeeds */
21 | var isFirefox = typeof InstallTrigger !== 'undefined';
22 | var supportsWebWorkers = !!window.Worker;
23 |
24 | if (supportsWebWorkers) {
25 | this.use_web_worker = true;
26 | }
27 |
28 | if (isFirefox) {
29 | this.use_web_worker = false;
30 | }
31 |
32 | this._hostname = server;
33 | this._port = 'https:' == location.protocol ? 3010 : 3001;
34 | this._protocol = 'https:' == location.protocol ? 'wss' : 'ws';
35 | this._path = '/ndt_protocol';
36 | this._update_interval = 1000;
37 |
38 | this.reset();
39 | }
40 |
41 | NDTWrapper.prototype.reset = function () {
42 | if (this.worker) {
43 | this.worker.postMessage({
44 | 'cmd': 'stop'
45 | });
46 |
47 | this.worker = null;
48 | }
49 |
50 | if (this.client) {
51 | this.client = null;
52 | }
53 |
54 | this._ndt_vars = { 'ClientToServerSpeed': 0, 'ServerToClientSpeed': 0 };
55 | this._errmsg = '';
56 | this._status = 'notStarted';
57 | this._diagnosis = '';
58 | };
59 |
60 | NDTWrapper.prototype.run_test = function () {
61 | var _this = this;
62 |
63 | this.reset();
64 |
65 | if (this.use_web_worker) {
66 | console.log('Generating new worker');
67 | // 'new Worker()' only accepts absolute paths to script files, so
68 | // determine the path to the location of the ndt-wrapper.js script, and
69 | // ndt-wrapper-ww.js should be in the same place.
70 | var scriptPath = getScriptPath();
71 | this.worker = new Worker(scriptPath + '/ndt-wrapper-ww.js');
72 | this.worker.addEventListener('message', function (e) {
73 | var msg = e.data;
74 | switch (msg.cmd) {
75 | case 'onstart':
76 | _this.onstart_cb(msg.server);
77 | break;
78 | case 'onstatechange':
79 | _this.onstatechange_cb(msg.state, msg.results);
80 | break;
81 | case 'onprogress':
82 | _this.onprogress_cb(msg.state, msg.results);
83 | break;
84 | case 'onfinish':
85 | _this.onfinish_cb(msg.results);
86 | break;
87 | case 'onerror':
88 | _this.onerror_cb(msg.error_message);
89 | break;
90 | }
91 | }, false);
92 |
93 | this.worker.addEventListener('error', function (e) {
94 | _this.onerror_cb(e.lineno, ' in ', e.filename, ': ', e.message);
95 | }, false);
96 |
97 | this.worker.postMessage({
98 | 'cmd': 'start',
99 | 'hostname': this._hostname,
100 | 'port': this._port,
101 | 'protocol': this._protocol,
102 | 'path': this._path,
103 | 'update_interval': this._update_interval
104 | });
105 | }
106 | else {
107 | this.callbacks = {
108 | 'onstart': function(server) {
109 | _this.onstart_cb(server);
110 | },
111 | 'onstatechange': function(state, results) {
112 | _this.onstatechange_cb(state, results);
113 | },
114 | 'onprogress': function(state, results) {
115 | _this.onprogress_cb(state, results);
116 | },
117 | 'onfinish': function(passed_results) {
118 | _this.onfinish_cb(passed_results);
119 | },
120 | 'onerror': function(error_message) {
121 | _this.onerror_cb(error_message);
122 | }
123 | };
124 |
125 | this.client = new NDTjs(this._hostname, this._port, this._protocol,
126 | this._path, this.callbacks,
127 | this._update_interval);
128 | this.client.startTest();
129 | }
130 | };
131 |
132 |
133 | NDTWrapper.prototype.get_status = function () {
134 | return this._status;
135 | };
136 |
137 | NDTWrapper.prototype.get_diagnosis = function () {
138 | return this._diagnosis;
139 | };
140 |
141 | NDTWrapper.prototype.get_errmsg = function () {
142 | return this._errmsg;
143 | };
144 |
145 | NDTWrapper.prototype.get_host = function () {
146 | return this._hostname;
147 | };
148 |
149 | NDTWrapper.prototype.getNDTvar = function (variable) {
150 | return this._ndt_vars[variable];
151 | };
152 |
153 | NDTWrapper.prototype.get_PcBuffSpdLimit = function (variable) {
154 | return Number.NaN;
155 | };
156 |
157 | NDTWrapper.prototype.onstart_cb = function (server) {
158 | };
159 |
160 | NDTWrapper.prototype.onstatechange_cb = function (state, results) {
161 | if (state === 'running_s2c') {
162 | this._status = 'Inbound';
163 | this._ndt_vars['ServerToClientSpeed'] = results['s2cRate'] / 1000;
164 | }
165 | else if (state === 'finished_s2c') {
166 | this._ndt_vars['ServerToClientSpeed'] = results['s2cRate'] / 1000;
167 | }
168 | else if (state === 'running_c2s') {
169 | this._status = 'Outbound';
170 | }
171 | else if (state == 'finished_c2s') {
172 | this._ndt_vars['ClientToServerSpeed'] = results['c2sRate'] / 1000;
173 | }
174 | };
175 |
176 | NDTWrapper.prototype.onprogress_cb = function (state, results) {
177 | if (state == 'interval_s2c') {
178 | this._ndt_vars['ServerToClientSpeed'] = results['s2cRate'] / 1000;
179 | }
180 | else if (state === 'interval_c2s') {
181 | this._ndt_vars['ClientToServerSpeed'] = results['c2sRate'] / 1000;
182 | }
183 | };
184 |
185 | NDTWrapper.prototype.onfinish_cb = function (results) {
186 | this._errmsg = "Test completed";
187 | this._ndt_vars = results;
188 | this._ndt_vars['ServerToClientSpeed'] = results['s2cRate'] / 1000;
189 | this._ndt_vars['ClientToServerSpeed'] = results['c2sRate'] / 1000;
190 | this._ndt_vars['Jitter'] = results['MaxRTT'] - results['MinRTT'];
191 | this.build_diagnosis();
192 | };
193 |
194 | NDTWrapper.prototype.build_diagnosis = function () {
195 | this._diagnosis = JSON.stringify(this._ndt_vars, null, " ");
196 | };
197 |
198 | NDTWrapper.prototype.onerror_cb = function (error_message) {
199 | this._errmsg = "Test failed: "+error_message;
200 | };
201 |
--------------------------------------------------------------------------------
/netx/net.go:
--------------------------------------------------------------------------------
1 | package netx
2 |
3 | import (
4 | "crypto/tls"
5 | "log"
6 | "net"
7 | "os"
8 | "sync"
9 | "time"
10 |
11 | guuid "github.com/google/uuid"
12 | "github.com/prometheus/client_golang/prometheus"
13 | "github.com/prometheus/client_golang/prometheus/promauto"
14 |
15 | "github.com/m-lab/go/rtx"
16 | "github.com/m-lab/ndt-server/bbr"
17 | "github.com/m-lab/ndt-server/netx/iface"
18 | "github.com/m-lab/tcp-info/inetdiag"
19 | "github.com/m-lab/tcp-info/tcp"
20 | )
21 |
22 | // Metrics for resource accounting in the netx package.
23 | var (
24 | CurrentOpenConns = promauto.NewGauge(
25 | prometheus.GaugeOpts{
26 | Name: "netx_current_open_conns",
27 | Help: "A gauge of currently open netx.Conns with open file pointers.",
28 | },
29 | )
30 | )
31 |
32 | // Listener is a TCPListener that is suitable for raw TCP servers, HTTP servers,
33 | // and TLS HTTP servers. The Conn's returned by Listener.Accept mediate access
34 | // to the underlying Conn file descriptor, allowing callers to perform meta
35 | // operations on the connection, e.g. GetUUID, EnableBBR, ReadInfo.
36 | type Listener struct {
37 | *net.TCPListener
38 | connfile iface.ConnFile
39 | }
40 |
41 | // NewListener creates a new Listener using the given net.TCPListener.
42 | func NewListener(l *net.TCPListener) *Listener {
43 | return &Listener{
44 | TCPListener: l,
45 | connfile: &iface.RealConnInfo{},
46 | }
47 | }
48 |
49 | // Conn is returned by Listener.Accept and provides mediated access to
50 | // additional operations on the Conn file descriptor.
51 | type Conn struct {
52 | net.Conn
53 | fp *os.File
54 | netinfo iface.NetInfo
55 | once sync.Once
56 | }
57 |
58 | // Addr supports the net.Addr interface and allows mediated access to operations
59 | // on the parent Conn.
60 | //
61 | // Why is this necessary? The Conn type accessible to application code is a
62 | // function of the server protocol. In particular, the tls.Conn is created by a
63 | // TLS Server, regardless of the underlying Listener type. Because a tls.Conn is
64 | // a struct, not an interface, and because tls.Conn does not provide access to
65 | // the underlying net.Conn, we still need a way to access the underlying Conn
66 | // type in order to access and operate on the underlying connection file
67 | // descriptor.
68 | //
69 | // So, to support all server protocols using the Listener, we "piggyback" on the
70 | // net.Conn interface supported by tls.Conns. In particular, the LocalAddr and
71 | // RemoteAddr methods return an Addr type, which includes a parent reference to
72 | // the associated Conn.
73 | //
74 | // Because the Addr includes references to the parent Conn, all Addrs should be
75 | // released before calling Conn.Close.
76 | type Addr struct {
77 | net.Addr
78 | parentConn *Conn
79 | }
80 |
81 | // ConnInfo provides operations on a Conn's underlying file descriptor.
82 | type ConnInfo interface {
83 | GetUUID() (string, error)
84 | EnableBBR() error
85 | ReadInfo() (inetdiag.BBRInfo, tcp.LinuxTCPInfo, error)
86 | }
87 |
88 | // Accept a connection, set 3min keepalive, and return a Conn that enables
89 | // ConnInfo operations on the underlying net.Conn file descriptor.
90 | func (ln *Listener) Accept() (net.Conn, error) {
91 | tc, err := ln.AcceptTCP()
92 | if err != nil {
93 | return nil, err
94 | }
95 | tc.SetKeepAlive(true)
96 | tc.SetKeepAlivePeriod(3 * time.Minute)
97 | fp, err := ln.connfile.DupFile(tc)
98 | if err != nil {
99 | tc.Close()
100 | return nil, err
101 | }
102 | CurrentOpenConns.Inc()
103 | mc := &Conn{
104 | Conn: tc,
105 | fp: fp,
106 | netinfo: &iface.RealConnInfo{},
107 | }
108 | return mc, nil
109 | }
110 |
111 | // Close the underlying net.Conn and dup'd file descriptor. Note: all net.Addr's
112 | // returned by LocalAddr and RemoteAddr should be released before calling Close.
113 | func (mc *Conn) Close() error {
114 | mc.fp.Close()
115 | mc.once.Do(CurrentOpenConns.Dec)
116 | return mc.Conn.Close()
117 | }
118 |
119 | // EnableBBR sets the BBR congestion control on the TCP connection, if supported
120 | // by the kernel. If unsupported, EnableBBR has no effect.
121 | func (mc *Conn) EnableBBR() error {
122 | return bbr.Enable(mc.fp)
123 | }
124 |
125 | // ReadInfo reads metadata about the TCP connections. If BBR was not enabled on
126 | // the underlying connection, then ReadInfo will return an empty BBRInfo struct.
127 | // If TCP info metrics cannot be read, an error is returned.
128 | func (mc *Conn) ReadInfo() (inetdiag.BBRInfo, tcp.LinuxTCPInfo, error) {
129 | bbrinfo, err := mc.netinfo.GetBBRInfo(mc.fp)
130 | if err != nil {
131 | bbrinfo = inetdiag.BBRInfo{}
132 | }
133 | tcpInfo, err := mc.netinfo.GetTCPInfo(mc.fp)
134 | if err != nil {
135 | return inetdiag.BBRInfo{}, tcp.LinuxTCPInfo{}, err
136 | }
137 | return bbrinfo, *tcpInfo, nil
138 | }
139 |
140 | // GetUUID returns the connection's UUID.
141 | func (mc *Conn) GetUUID() (string, error) {
142 | id, err := mc.netinfo.GetUUID(mc.fp)
143 | if err != nil {
144 | // Use UUID v1 as fallback when SO_COOKIE isn't supported by kernel
145 | fallbackUUID, err := guuid.NewUUID()
146 | // NOTE: this could only fail when `GetTime` fails from guuid package.
147 | rtx.Must(err, "unable to fallback to uuid")
148 |
149 | // NOTE: at this point, id is guaranteed to not be empty.
150 | id = fallbackUUID.String()
151 | }
152 | return id, nil
153 | }
154 |
155 | // LocalAddr returns an Addr supporting the net.Addr interface, which provides
156 | // access to the parent Conn.
157 | func (mc *Conn) LocalAddr() net.Addr {
158 | return &Addr{
159 | Addr: mc.Conn.LocalAddr(),
160 | parentConn: mc,
161 | }
162 | }
163 |
164 | // RemoteAddr returns an Addr supporting the net.Addr interface, which provides
165 | // access to the parent Conn.
166 | func (mc *Conn) RemoteAddr() net.Addr {
167 | return &Addr{
168 | Addr: mc.Conn.RemoteAddr(),
169 | parentConn: mc,
170 | }
171 | }
172 |
173 | // ToTCPAddr is a helper function for extracting the net.TCPAddr type from a
174 | // net.Addr of various origins. ToTCPAddr returns nil if addr does not contain a
175 | // *net.TCPAddr.
176 | func ToTCPAddr(addr net.Addr) *net.TCPAddr {
177 | switch a := addr.(type) {
178 | case *Addr:
179 | return a.Addr.(*net.TCPAddr)
180 | case *net.TCPAddr:
181 | return a
182 | default:
183 | log.Printf("unsupported conn type: %T", a)
184 | return nil
185 | }
186 | }
187 |
188 | // ToConnInfo is a helper function for extracting the ConnInfo interface from
189 | // the net.Conn of various origins. ToConnInfo panics if conn does not contain a
190 | // type supporting ConnInfo.
191 | func ToConnInfo(conn net.Conn) ConnInfo {
192 | switch c := conn.(type) {
193 | case *Conn:
194 | return c
195 | case *tls.Conn:
196 | return c.LocalAddr().(*Addr).parentConn
197 | default:
198 | log.Printf("unsupported conn type: %T", c)
199 | return nil
200 | }
201 | }
202 |
--------------------------------------------------------------------------------
/ndt5/c2s/c2s.go:
--------------------------------------------------------------------------------
1 | package c2s
2 |
3 | import (
4 | "context"
5 | "log"
6 | "strconv"
7 | "time"
8 |
9 | "github.com/m-lab/go/warnonerror"
10 | "github.com/m-lab/ndt-server/ndt5/metrics"
11 | "github.com/m-lab/ndt-server/ndt5/ndt"
12 | "github.com/m-lab/ndt-server/ndt5/protocol"
13 | "github.com/m-lab/ndt-server/ndt5/web100"
14 | )
15 |
16 | // ArchivalData is the data saved by the C2S test. If a researcher wants deeper
17 | // data, then they should use the UUID to get deeper data from tcp-info.
18 | type ArchivalData struct {
19 | // The server and client IP are here as well as in the containing struct
20 | // because happy eyeballs means that we may have a IPv4 control connection
21 | // causing a IPv6 connection to the test port or vice versa.
22 | ServerIP string
23 | ServerPort int
24 | ClientIP string
25 | ClientPort int
26 |
27 | // This is the only field that is really required.
28 | UUID string
29 |
30 | // These fields are here to enable analyses that don't require joining with tcp-info data.
31 | StartTime time.Time
32 | EndTime time.Time
33 | MeanThroughputMbps float64
34 | // TODO: Add TCPEngine (bbr, cubic, reno, etc.)
35 |
36 | Error string `json:",omitempty"`
37 | }
38 |
39 | // ManageTest manages the c2s test lifecycle.
40 | func ManageTest(ctx context.Context, controlConn protocol.Connection, s ndt.Server) (record *ArchivalData, err error) {
41 | localContext, localCancel := context.WithTimeout(ctx, 30*time.Second)
42 | defer localCancel()
43 | defer func() {
44 | if err != nil && record != nil {
45 | record.Error = err.Error()
46 | }
47 | }()
48 | record = &ArchivalData{}
49 |
50 | m := controlConn.Messager()
51 | connType := s.ConnectionType().Label()
52 |
53 | srv, err := s.SingleServingServer("c2s")
54 | if err != nil {
55 | log.Println("Could not start SingleServingServer", err)
56 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "StartSingleServingServer").Inc()
57 | return record, err
58 | }
59 |
60 | err = m.SendMessage(protocol.TestPrepare, []byte(strconv.Itoa(srv.Port())))
61 | if err != nil {
62 | log.Println("Could not send TestPrepare", err)
63 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "TestPrepare").Inc()
64 | return record, err
65 | }
66 |
67 | testConn, err := srv.ServeOnce(localContext)
68 | if err != nil {
69 | log.Println("Could not successfully ServeOnce", err)
70 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "ServeOnce").Inc()
71 | return record, err
72 | }
73 |
74 | // When ManageTest exits, close the test connection.
75 | defer func() {
76 | // Allow the connection-draining goroutine to empty all buffers in support of
77 | // poorly-written clients before we close the connection, but do not block the
78 | // exit of ManageTest on waiting for the test connection to close.
79 | go func() {
80 | time.Sleep(3 * time.Second)
81 | warnonerror.Close(testConn, "Could not close test connection")
82 | }()
83 | }()
84 |
85 | record.UUID = testConn.UUID()
86 | record.ServerIP, record.ServerPort = testConn.ServerIPAndPort()
87 | record.ClientIP, record.ClientPort = testConn.ClientIPAndPort()
88 |
89 | err = m.SendMessage(protocol.TestStart, []byte{})
90 | if err != nil {
91 | log.Println("Could not send TestStart", err, record.UUID)
92 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "TestStart").Inc()
93 | return record, err
94 | }
95 |
96 | record.StartTime = time.Now()
97 | web100Metrics, err := drainForeverButMeasureFor(ctx, testConn, 10*time.Second)
98 | record.EndTime = time.Now()
99 | seconds := record.EndTime.Sub(record.StartTime).Seconds()
100 | log.Println("Ended C2S test on", testConn, record.UUID)
101 | if err != nil {
102 | if web100Metrics.TCPInfo.BytesReceived == 0 {
103 | log.Println("Could not drain the test connection", err, record.UUID)
104 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "Drain").Inc()
105 | return record, err
106 | }
107 | // It is possible for the client to reach 10 seconds slightly before the server does.
108 | if seconds < 9 {
109 | log.Printf("C2S test client only uploaded for %f seconds %s\n", seconds, record.UUID)
110 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "EarlyExit").Inc()
111 | return record, err
112 | }
113 | // More than 9 seconds is fine.
114 | log.Printf("C2S test had an error (%v) after %f seconds. We will continue with the test.\n", err, seconds)
115 | }
116 |
117 | throughputValue := 8 * float64(web100Metrics.TCPInfo.BytesReceived) / 1000 / seconds
118 | record.MeanThroughputMbps = throughputValue / 1000 // Convert Kbps to Mbps
119 |
120 | log.Println(controlConn, "sent us", throughputValue, "Kbps")
121 | err = m.SendMessage(protocol.TestMsg, []byte(strconv.FormatInt(int64(throughputValue), 10)))
122 | if err != nil {
123 | log.Println("Could not send TestMsg with C2S results", err, record.UUID)
124 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "TestMsg").Inc()
125 | return record, err
126 | }
127 |
128 | err = m.SendMessage(protocol.TestFinalize, []byte{})
129 | if err != nil {
130 | log.Println("Could not send TestFinalize", err, record.UUID)
131 | metrics.ClientTestErrors.WithLabelValues(connType, "c2s", "TestFinalize").Inc()
132 | return record, err
133 | }
134 |
135 | return record, nil
136 | }
137 |
138 | // drainForeverButMeasureFor is a generic method for draining a connection while
139 | // measuring the connection for the first part of the drain. This method does
140 | // not close the passed-in Connection, and starts a goroutine which runs until
141 | // that Connection is closed.
142 | func drainForeverButMeasureFor(ctx context.Context, conn protocol.MeasuredConnection, d time.Duration) (*web100.Metrics, error) {
143 | derivedCtx, derivedCancel := context.WithTimeout(ctx, d)
144 | defer derivedCancel()
145 |
146 | conn.StartMeasuring(derivedCtx)
147 |
148 | errs := make(chan error, 1)
149 | // This is the "drain forever" part of this function. Read the passed-in
150 | // connection until the passed-in connection is closed.
151 | go func() {
152 | var connErr error
153 | // Read the connections until the connection is closed. Reading on a closed
154 | // connection returns an error, which terminates the loop and the goroutine.
155 | for connErr == nil {
156 | _, connErr = conn.ReadBytes()
157 | }
158 | errs <- connErr
159 | }()
160 |
161 | var socketStats *web100.Metrics
162 | var err error
163 | select {
164 | case <-derivedCtx.Done(): // Wait for timeout
165 | log.Println("Timed out")
166 | socketStats, err = conn.StopMeasuring()
167 | case err = <-errs: // Error in c2s transfer
168 | log.Println("C2S error:", err)
169 | socketStats, _ = conn.StopMeasuring()
170 | }
171 | if socketStats == nil {
172 | return nil, err
173 | }
174 | // socketStats is guaranteed to be non-nil and the TCPInfo element is a value not a pointer.
175 | return socketStats, err
176 | }
177 |
--------------------------------------------------------------------------------
/html/style.css:
--------------------------------------------------------------------------------
1 | @font-face {
2 | font-family: "League Gothic";
3 | src: url("fonts/League_Gothic.otf") format("opentype");
4 | }
5 |
6 | body * {
7 | margin: 0;
8 | padding: 0;
9 | }
10 | body {
11 | margin: 0;
12 | padding: 0;
13 | border: 0;
14 | overflow: hidden;
15 | height: 2000px;
16 | }
17 | a img {
18 | border: 0;
19 | }
20 | #widget {
21 | position: relative;
22 | width: 720px;
23 | height: 480px;
24 | overflow: hidden;
25 | background-color: black;
26 | color: white;
27 | text-transform: uppercase;
28 | font-size: 36px;
29 | font-family: "League Gothic", Impact, "Arial Narrow", sans-serif;
30 | margin-bottom: 15px;
31 | }
32 | .header {
33 | margin: 10px;
34 | }
35 | #no-websocket {
36 | color: red;
37 | margin: 5px;
38 | }
39 | #no-websocket {
40 | color: red;
41 | margin: 5px;
42 | display: none;
43 | }
44 | .page {
45 | width: 640px;
46 | height: 420px;
47 | position: absolute;
48 | left: 0;
49 | top: 0;
50 | padding: 40px;
51 | background-color: #00aef0;
52 | color: #000;
53 | }
54 | #welcome {
55 | text-align: center;
56 | }
57 | #test, #results {
58 | display: none;
59 | }
60 | #test {
61 | background-color: #3cb878;
62 | }
63 |
64 |
65 | /* Base styles */
66 |
67 | h1, h2, h3 {
68 | position: relative;
69 | margin: 0;
70 | padding: 0;
71 | font-weight: normal;
72 | }
73 | h1 {
74 | font-size: 72px;
75 | }
76 | h2 {
77 | font-size: 48px;
78 | }
79 | h2 em {
80 | font-style: normal;
81 | color: white;
82 | }
83 | h2.centered {
84 | text-align:center;
85 | }
86 | h3 {
87 | font-size: 36px;
88 | }
89 |
90 | .button {
91 | padding: 3px 20px;
92 | border: 4px solid white;
93 | -moz-border-radius: 50px;
94 | -webkit-border-radius: 50px;
95 | border-radius: 50px;
96 | background-color: #4d4d4d;
97 | color: white;
98 | cursor: pointer;
99 | text-decoration: none;
100 | }
101 | .button:hover {
102 | background-color: #000;
103 | }
104 | .controls .button {
105 | -moz-box-shadow: 0 3px 5px #000;
106 | }
107 |
108 | .backendButton {
109 | width: 200px;
110 | height: 50px;
111 | font-size: 24px;
112 | margin: 10px;
113 | }
114 |
115 | .active {
116 | background-color:black;
117 | }
118 |
119 | .backendButton:disabled {
120 | background-color:gray;
121 | cursor: default;
122 | }
123 |
124 | /* Welcome page */
125 |
126 | .page .status,
127 | .page .controls {
128 | width: 640px;
129 | position: absolute;
130 | left: 40px;
131 | bottom: 40px;
132 | text-align: center;
133 | }
134 | body.initializing .page .controls,
135 | body.ready .page .status {
136 | display: none;
137 | }
138 | body.ready .page .controls,
139 | body.initializing .page .status {
140 | display: block;
141 | }
142 |
143 | #welcome .info {
144 | padding-top: 10px;
145 | text-transform: none;
146 | font: 13px/18px verdana, helvetica, arial, sans-serif;
147 | }
148 | #welcome .info .logo {
149 | margin-bottom: 30px;
150 | }
151 | #welcome .info p {
152 | margin-bottom: 20px;
153 | }
154 | #welcome .info a {
155 | color: white;
156 | font-weight: bold;
157 | }
158 |
159 |
160 | /* Test */
161 |
162 | #test .locations {
163 | width: 640px;
164 | position: absolute;
165 | left: 40px;
166 | top: 400px;
167 | color: white;
168 | font-size: 32px;
169 | line-height: 32px;
170 | }
171 | #test .location {
172 | position: absolute;
173 | top: 0;
174 | }
175 | #test .location.local {
176 | right: 0;
177 | text-align: right;
178 | }
179 | #test .location.remote {
180 | left: 0;
181 | text-align: left;
182 | }
183 | #test .location .address {
184 | margin-top: 10px;
185 | color: black;
186 | font: bold 13px/13px verdana, helvetica, arial, sans-serif;
187 | }
188 |
189 | #uploadGauge {
190 | position: absolute;
191 | left: 40px;
192 | top: 120px;
193 | overflow: hidden;
194 | }
195 |
196 | #downloadGauge {
197 | position: absolute;
198 | left: 380px;
199 | top: 120px;
200 | overflow: hidden;
201 | }
202 |
203 | #rtt {
204 | position: absolute;
205 | left: 320px;
206 | top: 320px;
207 | color: white;
208 | font: bold 20px/20px verdana, helvetica, arial, sans-serif;
209 | }
210 |
211 | #rttValue {
212 | position: relative;
213 | text-align: center;
214 | left: -30px;
215 | width: 100px;
216 | color: black;
217 | }
218 |
219 | /* Test results */
220 |
221 | #results {
222 | height: 400px;
223 | }
224 | #result-pages {
225 | width: 640px;
226 | height: 280px;
227 | position: absolute;
228 | left: 40px;
229 | top: 100px;
230 | }
231 | #result-pages .result-page {
232 | width: 640px;
233 | height: 280px;
234 | position: absolute;
235 | left: 0;
236 | top: 0;
237 | display: none;
238 | }
239 | #results.summary #result-pages .result-page.summary,
240 | #results.details #result-pages .result-page.details,
241 | #results.advanced #result-pages .result-page.advanced {
242 | display: block;
243 | }
244 | #result-pages .details,
245 | #result-pages .advanced {
246 | overflow: auto;
247 | text-transform: none;
248 | font: 13px/18px verdana, arial, helvetica, sans-serif;
249 | }
250 | #result-pages .advanced {
251 | white-space: pre;
252 | }
253 | #results .result {
254 | width: 268px;
255 | height: 180px;
256 | position: absolute;
257 | top: 0px;
258 | padding: 20px;
259 | border: 1px solid white;
260 | }
261 | #results .result .number {
262 | font-size: 100px;
263 | line-height: 128px;
264 | color: white;
265 | }
266 | #results .result .units {
267 | font-size: 48px;
268 | text-transform: none;
269 | }
270 | #results .upload {
271 | left: 0;
272 | }
273 | #results .download {
274 | right: 0;
275 | }
276 | #results .other {
277 | position: absolute;
278 | top: 240px;
279 | text-transform: none;
280 | font: 13px/18px verdana, arial, helvetica, sans-serif;
281 | }
282 | #results .controls {
283 | width: 320px;
284 | left: 360px;
285 | text-align: right;
286 | }
287 | #results .branding {
288 | position: absolute;
289 | width: 320px;
290 | left: 40px;
291 | bottom: 24px;
292 | text-transform: none;
293 | font: 11px/16px verdana, arial, helvetica, sans-serif;
294 | }
295 | #results .branding a {
296 | color: black;
297 | }
298 |
299 |
300 | #results .view-selector {
301 | position: absolute;
302 | right: 40px;
303 | top: 60px;
304 | font-size: 18px;
305 | line-height: 18px;
306 | }
307 | #results .view-selector .summary {
308 | border-width: 2px 1px 2px 2px;
309 | padding-right: 15px;
310 | -moz-border-radius: 15px 0 0 15px;
311 | -webkit-border-radius: 15px 0 0 15px;
312 | border-radius: 15px 0 0 15px;
313 | }
314 | #results .view-selector .details {
315 | border-width: 2px 1px;
316 | padding-left: 15px;
317 | padding-right: 15px;
318 | -moz-border-radius: 0;
319 | -webkit-border-radius: 0;
320 | border-radius: 0;
321 | }
322 | #results .view-selector .advanced {
323 | border-width: 2px 2px 2px 1px;
324 | padding-left: 15px;
325 | -moz-border-radius: 0 15px 15px 0;
326 | -webkit-border-radius: 0 15px 15px 0;
327 | border-radius: 0 15px 15px 0;
328 | }
329 | #results.summary .view-selector .summary,
330 | #results.details .view-selector .details,
331 | #results.advanced .view-selector .advanced {
332 | background-color: #000;
333 | }
334 |
335 |
336 |
--------------------------------------------------------------------------------
/ndt5/s2c/s2c.go:
--------------------------------------------------------------------------------
1 | package s2c
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "log"
7 | "strconv"
8 | "time"
9 |
10 | "github.com/m-lab/go/warnonerror"
11 | "github.com/m-lab/ndt-server/ndt5/metrics"
12 | "github.com/m-lab/ndt-server/ndt5/ndt"
13 | "github.com/m-lab/ndt-server/ndt5/protocol"
14 | "github.com/m-lab/tcp-info/tcp"
15 | )
16 |
17 | // ArchivalData is the data saved by the S2C test. If a researcher wants deeper
18 | // data, then they should use the UUID to get deeper data from tcp-info.
19 | type ArchivalData struct {
20 | // This is the only field that is really required.
21 | UUID string
22 |
23 | // All subsequent fields are here to enable analyses that don't require joining
24 | // with tcp-info data.
25 |
26 | // The server and client IP are here as well as in the containing struct
27 | // because happy eyeballs means that we may have a IPv4 control connection
28 | // causing a IPv6 connection to the test port or vice versa.
29 | ServerIP string
30 | ServerPort int
31 | ClientIP string
32 | ClientPort int
33 |
34 | StartTime time.Time
35 | EndTime time.Time
36 | MeanThroughputMbps float64
37 | MinRTT time.Duration
38 | MaxRTT time.Duration
39 | SumRTT time.Duration
40 | CountRTT uint32
41 | ClientReportedMbps float64
42 | // TODO: Add TCPEngine (bbr, cubic, reno, etc.), MaxThroughputKbps, and Jitter
43 |
44 | TCPInfo *tcp.LinuxTCPInfo `json:",omitempty"`
45 | Error string `json:",omitempty"`
46 | }
47 |
48 | // ManageTest manages the s2c test lifecycle
49 | func ManageTest(ctx context.Context, controlConn protocol.Connection, s ndt.Server) (record *ArchivalData, err error) {
50 | localCtx, localCancel := context.WithTimeout(ctx, 30*time.Second)
51 | defer localCancel()
52 | record = &ArchivalData{}
53 | defer func() {
54 | if err != nil {
55 | record.Error = err.Error()
56 | }
57 | }()
58 |
59 | connType := s.ConnectionType().Label()
60 |
61 | srv, err := s.SingleServingServer("s2c")
62 | if err != nil {
63 | log.Println("Could not start single serving server", err)
64 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "StartSingleServingServer").Inc()
65 | return record, err
66 | }
67 | m := controlConn.Messager()
68 | err = m.SendMessage(protocol.TestPrepare, []byte(strconv.Itoa(srv.Port())))
69 | if err != nil {
70 | log.Println("Could not send TestPrepare", err)
71 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "TestPrepare").Inc()
72 | return record, err
73 | }
74 |
75 | testConn, err := srv.ServeOnce(localCtx)
76 | if err != nil || testConn == nil {
77 | log.Println("Could not successfully ServeOnce", err)
78 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "ServeOnce").Inc()
79 | if err == nil {
80 | err = errors.New("nil testConn, but also a nil error")
81 | }
82 | return record, err
83 | }
84 | record.UUID = testConn.UUID()
85 | record.ServerIP, record.ServerPort = testConn.ServerIPAndPort()
86 | record.ClientIP, record.ClientPort = testConn.ClientIPAndPort()
87 |
88 | dataToSend := make([]byte, 8192)
89 | for i := range dataToSend {
90 | dataToSend[i] = byte(((i * 101) % (122 - 33)) + 33)
91 | }
92 |
93 | err = m.SendMessage(protocol.TestStart, []byte{})
94 | if err != nil {
95 | warnonerror.Close(testConn, "Could not close test connection")
96 | log.Println("Could not write TestStart", err, record.UUID)
97 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "TestStart").Inc()
98 | return record, err
99 | }
100 |
101 | testConn.StartMeasuring(localCtx)
102 | record.StartTime = time.Now()
103 | testConn.FillUntil(time.Now().Add(10*time.Second), dataToSend)
104 | record.EndTime = time.Now()
105 |
106 | web100metrics, err := testConn.StopMeasuring()
107 | if err != nil {
108 | warnonerror.Close(testConn, "Could not close test connection")
109 | log.Println("Could not read metrics", err, record.UUID)
110 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "web100Metrics").Inc()
111 | return record, err
112 | }
113 |
114 | // Close the test connection to signal to single-threaded clients that the
115 | // download has completed. Note: a possible optimization is to wait for
116 | // one-two seconds for the client to close the connection and then close
117 | // it anyway. This gives us the advantage that the client will retain
118 | // the state assciated with initiating the close.
119 | warnonerror.Close(testConn, "Could not close testConnection")
120 |
121 | // Bits per second is the number of bits divided by the duration of the
122 | // test. The duration of the test is supposed to be 10 seconds, but it
123 | // can vary in practice, so we divide by the actual duration instead of
124 | // assuming it was 10.
125 | bps := 8 * float64(web100metrics.TCPInfo.BytesAcked) / record.EndTime.Sub(record.StartTime).Seconds()
126 | kbps := bps / 1000
127 | record.MinRTT = time.Duration(web100metrics.MinRTT) * time.Millisecond
128 | record.MaxRTT = time.Duration(web100metrics.MaxRTT) * time.Millisecond
129 | record.SumRTT = time.Duration(web100metrics.SumRTT) * time.Millisecond
130 | record.CountRTT = web100metrics.CountRTT
131 | record.MeanThroughputMbps = kbps / 1000 // Convert Kbps to Mbps
132 | record.TCPInfo = &web100metrics.TCPInfo
133 |
134 | // Send download results to the client.
135 | err = m.SendS2CResults(int64(kbps), 0, web100metrics.TCPInfo.BytesAcked)
136 | if err != nil {
137 | log.Println("Could not write a TestMsg", err, record.UUID)
138 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "TestMsgSend").Inc()
139 | return record, err
140 | }
141 |
142 | clientRateMsg, err := m.ReceiveMessage(protocol.TestMsg)
143 | // Do not return with an error if we got anything at all from the client.
144 | if err != nil && clientRateMsg == nil {
145 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "TestMsgRcv").Inc()
146 | log.Println("Could not receive a TestMsg", err, record.UUID)
147 | return record, err
148 | }
149 | log.Println("We measured", kbps, "and the client sent us", string(clientRateMsg))
150 | clientRateKbps, err := strconv.ParseFloat(string(clientRateMsg), 64)
151 | if err == nil {
152 | record.ClientReportedMbps = clientRateKbps / 1000
153 | } else {
154 | log.Println("Could not parse number sent from client")
155 | // Being unable to parse the number should not be a fatal error, so continue.
156 | }
157 |
158 | err = protocol.SendMetrics(web100metrics, m, "")
159 | if err != nil {
160 | log.Println("Could not SendMetrics for the legacy data", err, record.UUID)
161 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "SendMetricsLegacy").Inc()
162 | return record, err
163 | }
164 | err = protocol.SendMetrics(record, m, "NDTResult.S2C.")
165 | if err != nil {
166 | log.Println("Could not SendMetrics for the archival data", err, record.UUID)
167 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "SendMetricsArchival").Inc()
168 | return record, err
169 | }
170 |
171 | err = m.SendMessage(protocol.TestFinalize, []byte{})
172 | if err != nil {
173 | log.Println("Could not send TestFinalize", err, record.UUID)
174 | metrics.ClientTestErrors.WithLabelValues(connType, "s2c", "TestFinalize").Inc()
175 | return record, err
176 | }
177 |
178 | return record, nil
179 | }
180 |
--------------------------------------------------------------------------------
/ndt5/singleserving/server.go:
--------------------------------------------------------------------------------
1 | package singleserving
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "log"
7 | "net"
8 | "net/http"
9 | "sync"
10 | "time"
11 |
12 | ndt5metrics "github.com/m-lab/ndt-server/ndt5/metrics"
13 | "github.com/m-lab/ndt-server/ndt5/ndt"
14 | "github.com/m-lab/ndt-server/ndt5/protocol"
15 | "github.com/m-lab/ndt-server/ndt5/ws"
16 | "github.com/m-lab/ndt-server/netx"
17 | )
18 |
19 | // wsServer is a single-serving server for unencrypted websockets.
20 | type wsServer struct {
21 | srv *http.Server
22 | listener *netx.Listener
23 | port int
24 | direction string
25 | newConn protocol.MeasuredConnection
26 | newConnErr error
27 | once sync.Once
28 | kind ndt.ConnectionType
29 | serve func(net.Listener) error
30 | }
31 |
32 | func (s *wsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
33 | upgrader := ws.Upgrader(s.direction)
34 | wsc, err := upgrader.Upgrade(w, r, nil)
35 | if err != nil {
36 | s.newConnErr = err
37 | return
38 | }
39 | s.newConn = protocol.AdaptWsConn(wsc)
40 | // The websocket upgrade process hijacks the connection. Only un-hijacked
41 | // connections are terminated on server shutdown.
42 | go s.Close()
43 | }
44 |
45 | func (s *wsServer) Port() int {
46 | return s.port
47 | }
48 |
49 | func (s *wsServer) ServeOnce(ctx context.Context) (protocol.MeasuredConnection, error) {
50 | // This is a single-serving server. After serving one response, shut it down.
51 | defer s.Close()
52 |
53 | derivedCtx, derivedCancel := context.WithCancel(ctx)
54 | var closeErr error
55 | go func() {
56 | closeErr = s.serve(s.listener)
57 | derivedCancel()
58 | }()
59 | // This will wait until derivedCancel() is called or the parent context is
60 | // canceled. It is the parent context that determines how long ServeOnce should
61 | // wait before giving up.
62 | <-derivedCtx.Done()
63 |
64 | // During error conditions there is a race with the goroutine to write to
65 | // closeErr. We copy the current value to a separate variable in an effort to
66 | // ensure that the race gets resolved in just one way for the following if().
67 | err := closeErr
68 | if s.newConn == nil && err != nil && err != http.ErrServerClosed {
69 | log.Println("Server closed incorrectly:", err)
70 | return nil, errors.New("Server did not close correctly")
71 | }
72 |
73 | // If the context times out, then we can arrive here with both the connection
74 | // and error being nil.
75 | if s.newConn == nil && s.newConnErr == nil {
76 | return nil, errors.New("No connection created")
77 | }
78 | // Because the client has contacted the test server successfully, count the test.
79 | ndt5metrics.MeasurementServerAccept.WithLabelValues(s.kind.String(), s.direction)
80 | return s.newConn, s.newConnErr
81 | }
82 |
83 | func (s *wsServer) Close() {
84 | s.once.Do(func() {
85 | ndt5metrics.MeasurementServerStop.WithLabelValues(string(s.kind)).Inc()
86 | s.listener.Close()
87 | s.srv.Close()
88 | })
89 | }
90 |
91 | // ListenWS starts a single-serving unencrypted websocket server. When this
92 | // method returns without error, it is safe for a client to connect to the
93 | // server, as the server socket will be in "listening" mode. The returned server
94 | // will not actually respond until ServeOnce() is called, but the connect() will
95 | // not fail as long as ServeOnce is called soon ("soon" is defined by os-level
96 | // timeouts) after this returns.
97 | func ListenWS(direction string) (ndt.SingleMeasurementServer, error) {
98 | ndt5metrics.MeasurementServerStart.WithLabelValues(string(ndt.WS)).Inc()
99 | return listenWS(direction)
100 | }
101 |
102 | func listenWS(direction string) (*wsServer, error) {
103 | mux := http.NewServeMux()
104 | s := &wsServer{
105 | srv: &http.Server{
106 | Handler: mux,
107 | // NOTE: set absolute read and write timeouts for server connections.
108 | // This prevents clients, or middleboxes, from opening a connection and
109 | // holding it open indefinitely. This applies equally to TLS and non-TLS
110 | // servers.
111 | ReadTimeout: time.Minute,
112 | WriteTimeout: time.Minute,
113 | },
114 | direction: direction,
115 | kind: ndt.WS,
116 | }
117 | s.serve = s.srv.Serve
118 | mux.Handle("/ndt_protocol", s)
119 |
120 | // Start listening right away to ensure that subsequent connections succeed.
121 | l, err := net.Listen("tcp", ":0")
122 | if err != nil {
123 | return nil, err
124 | }
125 | tcpl := l.(*net.TCPListener)
126 | s.port = tcpl.Addr().(*net.TCPAddr).Port
127 | s.listener = netx.NewListener(tcpl)
128 | return s, nil
129 | }
130 |
131 | // wssServer is a single-serving server for encrypted websockets. A wssServer is
132 | // just a wsServer with a different start method and two extra fields.
133 | type wssServer struct {
134 | *wsServer
135 | certFile, keyFile string
136 | }
137 |
138 | // ListenWSS starts a single-serving encrypted websocket server. When this method
139 | // returns without error, it is safe for a client to connect to the server, as
140 | // the server socket will be in "listening" mode. The returned server will not
141 | // actually respond until ServeOnce() is called, but the connect() will not fail
142 | // as long as ServeOnce is called soon ("soon" is defined by os-level timeouts)
143 | // after this returns.
144 | func ListenWSS(direction, certFile, keyFile string) (ndt.SingleMeasurementServer, error) {
145 | ndt5metrics.MeasurementServerStart.WithLabelValues(string(ndt.WSS)).Inc()
146 | ws, err := listenWS(direction)
147 | if err != nil {
148 | return nil, err
149 | }
150 | wss := wssServer{
151 | wsServer: ws,
152 | certFile: certFile,
153 | keyFile: keyFile,
154 | }
155 | wss.kind = ndt.WSS
156 | wss.serve = func(l net.Listener) error {
157 | return wss.srv.ServeTLS(l, wss.certFile, wss.keyFile)
158 | }
159 | return &wss, nil
160 | }
161 |
162 | // plainServer is a single-serving server for plain TCP sockets.
163 | type plainServer struct {
164 | listener net.Listener
165 | port int
166 | direction string
167 | }
168 |
169 | func (ps *plainServer) Close() {
170 | ndt5metrics.MeasurementServerStop.WithLabelValues(string(ndt.Plain)).Inc()
171 | ps.listener.Close()
172 | }
173 |
174 | func (ps *plainServer) Port() int {
175 | return ps.port
176 | }
177 |
178 | func (ps *plainServer) ServeOnce(ctx context.Context) (protocol.MeasuredConnection, error) {
179 | // NOTE: set an absolute timeouts for single serving servers.
180 | derivedCtx, derivedCancel := context.WithTimeout(ctx, time.Minute)
181 | defer ps.Close()
182 |
183 | var conn net.Conn
184 | var err error
185 | go func() {
186 | conn, err = ps.listener.Accept()
187 | derivedCancel()
188 | }()
189 | <-derivedCtx.Done()
190 |
191 | if err != nil {
192 | return nil, err
193 | }
194 | if conn == nil {
195 | return nil, errors.New("nil conn, nil err: " + derivedCtx.Err().Error())
196 | }
197 | // Because the client has contacted the test server successfully, count the test.
198 | ndt5metrics.MeasurementServerAccept.WithLabelValues(ndt.Plain.String(), ps.direction)
199 | return protocol.AdaptNetConn(conn, conn), nil
200 | }
201 |
202 | // ListenPlain starts a single-serving server for plain NDT tests. When this
203 | // method returns without error, it is safe for a client to connect to the
204 | // server, as the server socket will be in "listening" mode. The returned server
205 | // will not actually respond until ServeOnce() is called, but the connect() will
206 | // not fail as long as ServeOnce is called soon ("soon" is defined by os-level
207 | // timeouts) after this returns.
208 | func ListenPlain(direction string) (ndt.SingleMeasurementServer, error) {
209 | ndt5metrics.MeasurementServerStart.WithLabelValues(string(ndt.Plain)).Inc()
210 | // Start listening right away to ensure that subsequent connections succeed.
211 | s := &plainServer{
212 | direction: direction,
213 | }
214 | l, err := net.Listen("tcp", ":0")
215 | if err != nil {
216 | return nil, err
217 | }
218 | tcpl := l.(*net.TCPListener)
219 | s.port = tcpl.Addr().(*net.TCPAddr).Port
220 | s.listener = netx.NewListener(tcpl)
221 | return s, nil
222 | }
223 |
--------------------------------------------------------------------------------
/netx/net_test.go:
--------------------------------------------------------------------------------
1 | package netx
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "log"
7 | "net"
8 | "net/http"
9 | "net/http/httptest"
10 | "os"
11 | "testing"
12 |
13 | "github.com/m-lab/go/rtx"
14 | "github.com/m-lab/tcp-info/inetdiag"
15 | "github.com/m-lab/tcp-info/tcp"
16 | )
17 |
18 | func init() {
19 | log.SetFlags(log.LUTC | log.Lshortfile)
20 | }
21 |
22 | type errorCI struct{}
23 |
24 | func (f *errorCI) DupFile(tc *net.TCPConn) (*os.File, error) {
25 | return nil, fmt.Errorf("fake file from conn error")
26 | }
27 |
28 | func dialAsync(t *testing.T, addr string) {
29 | go func() {
30 | // Because the socket already exists, Dial will block until Accept is
31 | // called below.
32 | c, err := net.Dial("tcp", addr)
33 | if err != nil {
34 | t.Errorf("unexpected failure to dial local conn: %v", err)
35 | return
36 | }
37 | // Wait until primary test routine closes conn and returns.
38 | buf := make([]byte, 1)
39 | c.Read(buf)
40 | c.Close()
41 | }()
42 | }
43 |
44 | func TestListener_Accept(t *testing.T) {
45 | // Successful Accept.
46 | addr := &net.TCPAddr{}
47 | tcpl, err := net.ListenTCP("tcp", addr)
48 | rtx.Must(err, "failed to listen during unit test")
49 | ln := NewListener(tcpl)
50 | defer ln.Close()
51 | dialAsync(t, tcpl.Addr().String())
52 |
53 | got, err := ln.Accept()
54 | if err != nil {
55 | t.Errorf("Listener.Accept() unexpected error = %v", err)
56 | return
57 | }
58 | if _, ok := got.(*Conn); !ok {
59 | t.Errorf("Listener.Accept() wrong Conn type = %T, want *Conn", got)
60 | }
61 |
62 | // Accept error
63 | addr = &net.TCPAddr{}
64 | tcpl, err = net.ListenTCP("tcp", addr)
65 | rtx.Must(err, "failed to listen during unit test")
66 | ln = NewListener(tcpl)
67 | defer ln.Close()
68 | // Close listener so Accept fails.
69 | tcpl.Close()
70 | // NOTE: a client dialing is unnecessary for accept to fail after listener is closed.
71 |
72 | _, err = ln.Accept()
73 | if err == nil {
74 | t.Errorf("Listener.Accept() expected error, got %#v", got)
75 | return
76 | }
77 |
78 | // ConnInfo error
79 | addr = &net.TCPAddr{}
80 | tcpl, err = net.ListenTCP("tcp", addr)
81 | rtx.Must(err, "failed to listen during unit test")
82 | ln = NewListener(tcpl)
83 | defer ln.Close()
84 | dialAsync(t, tcpl.Addr().String())
85 |
86 | // Force accept to receive an error when reading fd from conn.
87 | ln.connfile = &errorCI{}
88 |
89 | got, err = ln.Accept()
90 | if err == nil {
91 | t.Errorf("Listener.Accept() expected error, got = %#v", got)
92 | return
93 | }
94 | }
95 |
96 | type errorNetInfo struct{}
97 |
98 | func (e *errorNetInfo) GetUUID(fp *os.File) (string, error) {
99 | return "", fmt.Errorf("fake get uuid error")
100 | }
101 | func (e *errorNetInfo) GetBBRInfo(fp *os.File) (inetdiag.BBRInfo, error) {
102 | return inetdiag.BBRInfo{}, nil
103 | }
104 | func (e *errorNetInfo) GetTCPInfo(fp *os.File) (*tcp.LinuxTCPInfo, error) {
105 | return nil, fmt.Errorf("fake get tcpinfo error")
106 | }
107 |
108 | func TestConn(t *testing.T) {
109 | // Setup listener.
110 | laddr := &net.TCPAddr{}
111 | tcpl, err := net.ListenTCP("tcp", laddr)
112 | rtx.Must(err, "failed to listen during unit test")
113 | ln := NewListener(tcpl)
114 | defer ln.Close()
115 |
116 | dialAsync(t, tcpl.Addr().String())
117 | conn, err := ln.Accept()
118 | if err != nil {
119 | t.Errorf("Conn.Accept() unexpected error = %v", err)
120 | return
121 | }
122 | defer conn.Close()
123 |
124 | addr := ToTCPAddr(conn.LocalAddr())
125 | if addr == nil {
126 | t.Errorf("ToTCPAddr(conn.LocalAddr()) returned nil addr")
127 | }
128 | addr = ToTCPAddr(conn.RemoteAddr())
129 | if addr == nil {
130 | t.Errorf("ToTCPAddr(conn.RemoteAddr()) returned nil addr")
131 | }
132 |
133 | ci := ToConnInfo(conn)
134 | ci.EnableBBR()
135 | id, err := ci.GetUUID()
136 | if err != nil || id == "" {
137 | t.Errorf("ConnInfo.GetUUID error: %#v, %q", err, id)
138 | }
139 | bi, ti, err := ci.ReadInfo()
140 | if err != nil {
141 | // TODO: make testing work on non-linux platforms.
142 | t.Errorf("ConnInfo.ReadInfo error: %#v, %#v %#v", err, bi, ti)
143 | }
144 |
145 | // Reset the netinfo value to always fail.
146 | c := conn.(*Conn)
147 | c.netinfo = &errorNetInfo{}
148 | id, err = ci.GetUUID()
149 | if err != nil || id == "" {
150 | t.Errorf("ConnInfo.GetUUID error, got %#v, %q", err, id)
151 | }
152 |
153 | // Read info with an error fr
154 | bi, ti, err = ci.ReadInfo()
155 | if err == nil {
156 | // TODO: make testing work on non-linux platforms.
157 | t.Errorf("ConnInfo.ReadInfo expected error, got nil: %#v %#v", bi, ti)
158 | }
159 | }
160 |
161 | func TestToTCPAddr(t *testing.T) {
162 | baseAddr := &net.TCPAddr{
163 | IP: net.ParseIP("127.0.0.1"),
164 | Port: 1234,
165 | }
166 | tests := []struct {
167 | name string
168 | addr net.Addr
169 | wantNil bool
170 | }{
171 | {
172 | name: "success-Addr",
173 | addr: &Addr{
174 | Addr: baseAddr,
175 | },
176 | },
177 | {
178 | name: "success-TCPAddr",
179 | addr: baseAddr,
180 | },
181 | {
182 | name: "unsupported-returns-nil",
183 | addr: &net.UDPAddr{
184 | IP: net.ParseIP("127.0.0.1"),
185 | Port: 1234,
186 | },
187 | wantNil: true,
188 | },
189 | }
190 | for _, tt := range tests {
191 | t.Run(tt.name, func(t *testing.T) {
192 | got := ToTCPAddr(tt.addr)
193 | if (got == nil) != tt.wantNil {
194 | t.Errorf("ToTCPAddr() wrong value; got %#v, wantNil %t", got, tt.wantNil)
195 | }
196 | })
197 | }
198 | }
199 |
200 | func TestToConnInfo(t *testing.T) {
201 | // NOTE: because we cannot synthetically create a tls.Conn that wraps a
202 | // netx.Conn, we must setup an httptest server with TLS enabled. While we
203 | // do that, we use it to validate the regular HTTP server netx.Conn as
204 | // well.
205 |
206 | fakeHTTPReply := "HTTP/1.0 200 OK\n\ntest"
207 | tests := []struct {
208 | name string
209 | conn net.Conn
210 | withTLS bool
211 | }{
212 | {
213 | name: "success-Conn",
214 | withTLS: false,
215 | },
216 | {
217 | name: "success-tls.Conn",
218 | withTLS: true,
219 | },
220 | }
221 | mux := http.NewServeMux()
222 | mux.HandleFunc("/", func(rw http.ResponseWriter, req *http.Request) {
223 | hj, ok := rw.(http.Hijacker)
224 | if !ok {
225 | t.Fatalf("httptest Server does not support Hijacker interface")
226 | }
227 | conn, _, err := hj.Hijack()
228 | if err != nil {
229 | t.Fatalf("failed to hijack responsewriter")
230 | }
231 | defer conn.Close()
232 | // Write a fake reply for the client.
233 | conn.Write([]byte(fakeHTTPReply))
234 |
235 | // Extract the ConnInfo from the hijacked conn.
236 | got := ToConnInfo(conn)
237 | if got == nil {
238 | t.Errorf("ToConnInfo() failed to return ConnInfo from conn")
239 | }
240 | })
241 | for _, tt := range tests {
242 | t.Run(tt.name, func(t *testing.T) {
243 | s := httptest.NewUnstartedServer(mux)
244 | // Setup local listener using our Listener rather than the default.
245 | laddr := &net.TCPAddr{
246 | IP: net.ParseIP("127.0.0.1"),
247 | }
248 | tcpl, err := net.ListenTCP("tcp", laddr)
249 | rtx.Must(err, "failed to listen during unit test")
250 | // Use our listener in the httptest Server.
251 | s.Listener = NewListener(tcpl)
252 | // Start a plain or tls server.
253 | if tt.withTLS {
254 | s.StartTLS()
255 | } else {
256 | s.Start()
257 | }
258 | defer s.Close()
259 |
260 | // Use the server-provided client for TLS settings.
261 | c := s.Client()
262 | req, err := http.NewRequest(http.MethodGet, s.URL, nil)
263 | rtx.Must(err, "Failed to create request to %s", s.URL)
264 | // Run request to run conn test in handler.
265 | resp, err := c.Do(req)
266 | rtx.Must(err, "failed to GET %s", s.URL)
267 | b, err := ioutil.ReadAll(resp.Body)
268 | rtx.Must(err, "failed to read reply from %s", s.URL)
269 |
270 | if string(b) != "test" {
271 | t.Errorf("failed to receive reply from server")
272 | }
273 | })
274 | }
275 |
276 | // Finally, verify that unsupported net.Conn types return nil.
277 | got := ToConnInfo(&net.UDPConn{})
278 | if got != nil {
279 | t.Errorf("ToConnInfo() returned ConInfo for unsupported type: %#v", got)
280 | }
281 | }
282 |
--------------------------------------------------------------------------------