├── .github └── dependabot.yml ├── .gitignore ├── .golangci.yml ├── LICENSE ├── LICENSE.md ├── README.rst ├── bson_benchmark_test.go ├── bson_extract.go ├── bson_hash.go ├── bson_matrix.go ├── bson_metric.go ├── bson_restore.go ├── bson_test.go ├── cmd ├── run-linter │ └── run-linter.go └── verify-mod-tidy │ └── verify-mod-tidy.go ├── collector.go ├── collector_batch.go ├── collector_benchmark_test.go ├── collector_better.go ├── collector_buffered.go ├── collector_dynamic.go ├── collector_sample.go ├── collector_sample_test.go ├── collector_streaming.go ├── collector_sync.go ├── collector_test.go ├── collector_uncompressed.go ├── csv.go ├── csv_test.go ├── events ├── collector.go ├── collector_test.go ├── custom.go ├── custom_test.go ├── events_benchmark_test.go ├── histogram.go ├── performance.go ├── performance_test.go ├── recorder.go ├── recorder_histogram.go ├── recorder_histogram_grouped.go ├── recorder_histogram_interval.go ├── recorder_histogram_single.go ├── recorder_performance_grouped.go ├── recorder_performance_interval.go ├── recorder_performance_raw.go ├── recorder_performance_single.go ├── recorder_test.go ├── recorder_wrapper_stdlib.go └── recorder_wrapper_sync.go ├── evergreen.yaml ├── ftdc.go ├── ftdc_test.go ├── go.mod ├── go.sum ├── hdrhist ├── LICENSE ├── hdr.go ├── hdr_test.go ├── snapshot.go ├── window.go └── window_test.go ├── iterator.go ├── iterator_benchmark_test.go ├── iterator_chunk.go ├── iterator_combined.go ├── iterator_matrix.go ├── iterator_sample.go ├── iterator_sample_test.go ├── makefile ├── metrics ├── json.go ├── json_test.go ├── metrics.go └── metrics_test.go ├── read.go ├── t2.go ├── t2_test.go ├── testutil └── util.go ├── util.go ├── util ├── catcher.go └── catcher_test.go ├── util_for_test.go └── writer.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | - package-ecosystem: gomod 5 | directory: "/" 6 | schedule: 7 | interval: monthly 8 | time: "06:00" 9 | timezone: "America/New_York" 10 | open-pull-requests-limit: 99 11 | commit-message: 12 | prefix: "CHORE: " 13 | reviewers: 14 | - evg-plt 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | metrics.ftdc 3 | example.ftdc 4 | perf_metrics.ftdc 5 | perf_metrics_small.ftdc 6 | genny_metrics.ftdc -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | linters: 3 | disable-all: true 4 | enable: 5 | - errcheck 6 | - gofmt 7 | - goimports 8 | - govet 9 | - ineffassign 10 | - misspell 11 | - unconvert 12 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2016 MongoDB, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====================================================== 2 | ``ftdc`` -- Golang FTDC Parsing and Generating Library 3 | ====================================================== 4 | 5 | Overview 6 | -------- 7 | 8 | FTDC, originally short for *full time diagnostic data capture*, is 9 | MongoDB's internal diagnostic data collection facility. It encodes 10 | data in a space-efficient format, which allows MongoDB to record 11 | diagnostic information every second, and store weeks of data with only 12 | a few hundred megabytes of storage. 13 | 14 | This library provides a fully-featured and easy to use toolkit for 15 | interacting data stored in this format in Go programs. The library 16 | itself originated as a `project by 2016 Summer interns at MongoDB 17 | `_ but has diverged substantially 18 | since then. 19 | 20 | Features 21 | -------- 22 | 23 | Current 24 | ~~~~~~~ 25 | 26 | Currently the library provides parsing of the FTDC data format and 27 | several ways of iterating these results. Additionally, it provides the 28 | ability to create FTDC payloads, and is the only extant (?) tool for 29 | generating FTDC data outside of the MongoDB code base. 30 | 31 | The library includes tools for generating FTDC payloads and document 32 | streams as well as iterators and tools for accessing data from FTDC 33 | files. All functionality is part of the ``ftdc`` package, and the API 34 | is fully documented. 35 | 36 | Upcoming 37 | ~~~~~~~~ 38 | 39 | - (pending requests and use-cases) mode to read FTDC data without 40 | flattening the document structure. 41 | 42 | - command line tools for parsing and generating FTDC payloads. 43 | 44 | - helpers for generating default collector configurations. 45 | 46 | - combined check 47 | 48 | - improved collector performance. 49 | 50 | ... and more 51 | 52 | Documentation 53 | ------------- 54 | 55 | All documentation is in the `godoc `_. 56 | 57 | Participate 58 | ----------- 59 | 60 | File tickets in the `MAKE `_ 61 | project on the MongoDB jira. The repository will shortly move back to 62 | an offical MongoDB GitHub organization. 63 | 64 | Pull requests are welcome. 65 | -------------------------------------------------------------------------------- /bson_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/evergreen-ci/birch" 8 | "github.com/mongodb/ftdc/testutil" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | type metricHashFunc func(*birch.Document) (string, int) 14 | 15 | func BenchmarkHashBSON(b *testing.B) { 16 | for _, impl := range []struct { 17 | Name string 18 | HashFunc metricHashFunc 19 | }{ 20 | { 21 | Name: "FNVChecksum", 22 | HashFunc: metricKeyHash, 23 | }, 24 | } { 25 | b.Run(impl.Name, func(b *testing.B) { 26 | for _, test := range []struct { 27 | Name string 28 | Doc *birch.Document 29 | }{ 30 | { 31 | Name: "FlatSmall", 32 | Doc: testutil.RandFlatDocument(10), 33 | }, 34 | { 35 | Name: "FlatLarge", 36 | Doc: testutil.RandFlatDocument(100), 37 | }, 38 | { 39 | Name: "ComplexSmall", 40 | Doc: testutil.RandComplexDocument(10, 5), 41 | }, 42 | { 43 | Name: "ComplexLarge", 44 | Doc: testutil.RandComplexDocument(100, 5), 45 | }, 46 | { 47 | Name: "MoreComplexSmall", 48 | Doc: testutil.RandComplexDocument(10, 2), 49 | }, 50 | { 51 | Name: "MoreComplexLarge", 52 | Doc: testutil.RandComplexDocument(100, 2), 53 | }, 54 | { 55 | Name: "EventMock", 56 | Doc: testutil.CreateEventRecord(2, 2, 2, 2), 57 | }, 58 | } { 59 | b.Run(test.Name, func(b *testing.B) { 60 | var ( 61 | h string 62 | num int 63 | ) 64 | for n := 0; n < b.N; n++ { 65 | h, num = impl.HashFunc(test.Doc) 66 | } 67 | b.StopTimer() 68 | assert.NotZero(b, num) 69 | assert.NotZero(b, h) 70 | }) 71 | } 72 | }) 73 | } 74 | } 75 | 76 | func BenchmarkDocumentCreation(b *testing.B) { 77 | ctx, cancel := context.WithCancel(context.Background()) 78 | defer cancel() 79 | for _, test := range []struct { 80 | Name string 81 | Samples int 82 | Length int 83 | Reference *birch.Document 84 | Metrics []Metric 85 | }{ 86 | { 87 | Name: "Flat", 88 | Samples: 1000, 89 | Length: 15, 90 | Reference: testutil.RandFlatDocument(15), 91 | Metrics: produceMockMetrics(ctx, 1000, func() *birch.Document { return testutil.RandFlatDocument(15) }), 92 | }, 93 | { 94 | Name: "SmallFlat", 95 | Samples: 1000, 96 | Length: 5, 97 | Reference: testutil.RandFlatDocument(5), 98 | Metrics: produceMockMetrics(ctx, 1000, func() *birch.Document { return testutil.RandFlatDocument(5) }), 99 | }, 100 | { 101 | Name: "LargeFlat", 102 | Samples: 1000, 103 | Length: 15, 104 | Reference: testutil.RandFlatDocument(15), 105 | Metrics: produceMockMetrics(ctx, 1000, func() *birch.Document { return testutil.RandFlatDocument(100) }), 106 | }, 107 | { 108 | Name: "Complex", 109 | Samples: 1000, 110 | Length: 60, 111 | Reference: testutil.RandComplexDocument(20, 3), 112 | Metrics: produceMockMetrics(ctx, 1000, func() *birch.Document { return testutil.RandComplexDocument(20, 3) }), 113 | }, 114 | { 115 | Name: "SmallComplex", 116 | Samples: 1000, 117 | Length: 10, 118 | Reference: testutil.RandComplexDocument(5, 1), 119 | Metrics: produceMockMetrics(ctx, 1000, func() *birch.Document { return testutil.RandComplexDocument(5, 1) }), 120 | }, 121 | } { 122 | var doc *birch.Document 123 | b.Run(test.Name, func(b *testing.B) { 124 | for n := 0; n < b.N; n++ { 125 | for i := 0; i < test.Samples; i++ { 126 | doc, _ = restoreDocument(test.Reference, i, test.Metrics, 0) 127 | require.NotNil(b, doc) 128 | require.Equal(b, test.Length, doc.Len()) 129 | } 130 | } 131 | }) 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /bson_extract.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/evergreen-ci/birch/bsontype" 8 | "github.com/mongodb/ftdc/util" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | //////////////////////////////////////////////////////////////////////// 13 | // 14 | // Helpers for encoding values from birch documents 15 | 16 | type extractedMetrics struct { 17 | values []*birch.Value 18 | types []bsontype.Type 19 | ts time.Time 20 | } 21 | 22 | func extractMetricsFromDocument(doc *birch.Document) (extractedMetrics, error) { 23 | metrics := extractedMetrics{} 24 | iter := doc.Iterator() 25 | 26 | var ( 27 | err error 28 | data extractedMetrics 29 | ) 30 | 31 | catcher := util.NewCatcher() 32 | 33 | for iter.Next() { 34 | data, err = extractMetricsFromValue(iter.Element().Value()) 35 | catcher.Add(err) 36 | metrics.values = append(metrics.values, data.values...) 37 | metrics.types = append(metrics.types, data.types...) 38 | 39 | if metrics.ts.IsZero() { 40 | metrics.ts = data.ts 41 | } 42 | } 43 | 44 | catcher.Add(iter.Err()) 45 | 46 | if metrics.ts.IsZero() { 47 | metrics.ts = time.Now() 48 | } 49 | 50 | return metrics, catcher.Resolve() 51 | } 52 | 53 | func extractMetricsFromArray(array *birch.Array) (extractedMetrics, error) { 54 | metrics := extractedMetrics{} 55 | 56 | var ( 57 | err error 58 | data extractedMetrics 59 | ) 60 | 61 | catcher := util.NewCatcher() 62 | iter := array.Iterator() 63 | 64 | for iter.Next() { 65 | data, err = extractMetricsFromValue(iter.Value()) 66 | catcher.Add(err) 67 | metrics.values = append(metrics.values, data.values...) 68 | metrics.types = append(metrics.types, data.types...) 69 | 70 | if metrics.ts.IsZero() { 71 | metrics.ts = data.ts 72 | } 73 | } 74 | 75 | catcher.Add(iter.Err()) 76 | 77 | return metrics, catcher.Resolve() 78 | } 79 | 80 | func extractMetricsFromValue(val *birch.Value) (extractedMetrics, error) { 81 | metrics := extractedMetrics{} 82 | var err error 83 | 84 | btype := val.Type() 85 | switch btype { 86 | case bsontype.Array: 87 | metrics, err = extractMetricsFromArray(val.MutableArray()) 88 | err = errors.WithStack(err) 89 | case bsontype.EmbeddedDocument: 90 | metrics, err = extractMetricsFromDocument(val.MutableDocument()) 91 | err = errors.WithStack(err) 92 | case bsontype.Boolean: 93 | if val.Boolean() { 94 | metrics.values = append(metrics.values, birch.VC.Int64(1)) 95 | } else { 96 | metrics.values = append(metrics.values, birch.VC.Int64(0)) 97 | } 98 | metrics.types = append(metrics.types, bsontype.Boolean) 99 | case bsontype.Double: 100 | metrics.values = append(metrics.values, val) 101 | metrics.types = append(metrics.types, bsontype.Double) 102 | case bsontype.Int32: 103 | metrics.values = append(metrics.values, birch.VC.Int64(int64(val.Int32()))) 104 | metrics.types = append(metrics.types, bsontype.Int32) 105 | case bsontype.Int64: 106 | metrics.values = append(metrics.values, val) 107 | metrics.types = append(metrics.types, bsontype.Int64) 108 | case bsontype.DateTime: 109 | metrics.values = append(metrics.values, birch.VC.Int64(epochMs(val.Time()))) 110 | metrics.types = append(metrics.types, bsontype.DateTime) 111 | metrics.ts = val.Time() 112 | case bsontype.Timestamp: 113 | t, i := val.Timestamp() 114 | metrics.values = append(metrics.values, birch.VC.Int64(int64(t)), birch.VC.Int64(int64(i))) 115 | metrics.types = append(metrics.types, bsontype.Timestamp, bsontype.Timestamp) 116 | } 117 | 118 | return metrics, err 119 | } 120 | 121 | func extractDelta(current *birch.Value, previous *birch.Value) (int64, error) { 122 | switch current.Type() { 123 | case bsontype.Double: 124 | return normalizeFloat(current.Double()) - normalizeFloat(previous.Double()), nil 125 | case bsontype.Int64: 126 | return current.Int64() - previous.Int64(), nil 127 | default: 128 | return 0, errors.Errorf("invalid type %s", current.Type()) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /bson_hash.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "fmt" 5 | "hash" 6 | "hash/fnv" 7 | 8 | "github.com/evergreen-ci/birch" 9 | "github.com/evergreen-ci/birch/bsontype" 10 | ) 11 | 12 | func metricKeyHash(doc *birch.Document) (string, int) { 13 | checksum := fnv.New64() 14 | seen := metricKeyHashDocument(checksum, "", doc) 15 | return fmt.Sprintf("%x", checksum.Sum(nil)), seen 16 | } 17 | 18 | func metricKeyHashDocument(checksum hash.Hash, key string, doc *birch.Document) int { 19 | iter := doc.Iterator() 20 | seen := 0 21 | for iter.Next() { 22 | elem := iter.Element() 23 | seen += metricKeyHashValue(checksum, fmt.Sprintf("%s.%s", key, elem.Key()), elem.Value()) 24 | } 25 | 26 | return seen 27 | } 28 | 29 | func metricKeyHashArray(checksum hash.Hash, key string, array *birch.Array) int { 30 | seen := 0 31 | iter := array.Iterator() 32 | idx := 0 33 | for iter.Next() { 34 | seen += metricKeyHashValue(checksum, fmt.Sprintf("%s.%d", key, idx), iter.Value()) 35 | idx++ 36 | } 37 | 38 | return seen 39 | } 40 | 41 | func metricKeyHashValue(checksum hash.Hash, key string, value *birch.Value) int { 42 | switch value.Type() { 43 | case bsontype.Array: 44 | return metricKeyHashArray(checksum, key, value.MutableArray()) 45 | case bsontype.EmbeddedDocument: 46 | return metricKeyHashDocument(checksum, key, value.MutableDocument()) 47 | case bsontype.Boolean: 48 | _, _ = checksum.Write([]byte(key)) 49 | return 1 50 | case bsontype.Double: 51 | _, _ = checksum.Write([]byte(key)) 52 | return 1 53 | case bsontype.Int32: 54 | _, _ = checksum.Write([]byte(key)) 55 | return 1 56 | case bsontype.Int64: 57 | _, _ = checksum.Write([]byte(key)) 58 | return 1 59 | case bsontype.DateTime: 60 | _, _ = checksum.Write([]byte(key)) 61 | return 1 62 | case bsontype.Timestamp: 63 | _, _ = checksum.Write([]byte(key)) 64 | return 2 65 | default: 66 | return 0 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /bson_matrix.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/evergreen-ci/birch/bsontype" 8 | "github.com/evergreen-ci/birch/types" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func rehydrateMatrix(metrics []Metric, sample int) (*birch.Element, int, error) { 13 | if sample >= len(metrics) { 14 | return nil, sample, io.EOF 15 | } 16 | 17 | // the birch library's representation of arrays is more 18 | // efficient when constructing arrays from documents, 19 | // otherwise. 20 | array := birch.MakeArray(len(metrics[sample].Values)) 21 | key := metrics[sample].Key() 22 | switch metrics[sample].originalType { 23 | case bsontype.Boolean: 24 | for _, p := range metrics[sample].Values { 25 | array.AppendInterface(p != 0) 26 | } 27 | case bsontype.Double: 28 | for _, p := range metrics[sample].Values { 29 | array.AppendInterface(restoreFloat(p)) 30 | } 31 | case bsontype.Int64: 32 | for _, p := range metrics[sample].Values { 33 | array.AppendInterface(p) 34 | } 35 | case bsontype.Int32: 36 | for _, p := range metrics[sample].Values { 37 | array.AppendInterface(int32(p)) 38 | } 39 | case bsontype.DateTime: 40 | for _, p := range metrics[sample].Values { 41 | array.AppendInterface(timeEpocMs(p)) 42 | } 43 | case bsontype.Timestamp: 44 | for idx, p := range metrics[sample].Values { 45 | array.AppendInterface(types.Timestamp{T: uint32(p), I: uint32(metrics[sample+1].Values[idx])}) 46 | } 47 | sample++ 48 | default: 49 | return nil, sample, errors.New("invalid data type") 50 | } 51 | sample++ 52 | return birch.EC.Array(key, array), sample, nil 53 | } 54 | -------------------------------------------------------------------------------- /bson_metric.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/evergreen-ci/birch/bsontype" 8 | ) 9 | 10 | //////////////////////////////////////////////////////////////////////// 11 | // 12 | // Helpers for parsing the timeseries data from a metrics payload 13 | 14 | func metricForDocument(path []string, d *birch.Document) []Metric { 15 | iter := d.Iterator() 16 | o := []Metric{} 17 | 18 | for iter.Next() { 19 | e := iter.Element() 20 | 21 | o = append(o, metricForType(e.Key(), path, e.Value())...) 22 | } 23 | 24 | return o 25 | } 26 | 27 | func metricForArray(key string, path []string, a *birch.Array) []Metric { 28 | if a == nil { 29 | return []Metric{} 30 | } 31 | 32 | iter := a.Iterator() // ignore the error which can never be non-nil 33 | o := []Metric{} 34 | idx := 0 35 | for iter.Next() { 36 | o = append(o, metricForType(fmt.Sprintf("%s.%d", key, idx), path, iter.Value())...) 37 | idx++ 38 | } 39 | 40 | return o 41 | } 42 | 43 | func metricForType(key string, path []string, val *birch.Value) []Metric { 44 | switch val.Type() { 45 | case bsontype.ObjectID: 46 | return []Metric{} 47 | case bsontype.String: 48 | return []Metric{} 49 | case bsontype.Decimal128: 50 | return []Metric{} 51 | case bsontype.Array: 52 | return metricForArray(key, path, val.MutableArray()) 53 | case bsontype.EmbeddedDocument: 54 | path = append(path, key) 55 | 56 | o := []Metric{} 57 | for _, ne := range metricForDocument(path, val.MutableDocument()) { 58 | o = append(o, Metric{ 59 | ParentPath: path, 60 | KeyName: ne.KeyName, 61 | startingValue: ne.startingValue, 62 | originalType: ne.originalType, 63 | }) 64 | } 65 | return o 66 | case bsontype.Boolean: 67 | if val.Boolean() { 68 | return []Metric{ 69 | { 70 | ParentPath: path, 71 | KeyName: key, 72 | startingValue: 1, 73 | originalType: val.Type(), 74 | }, 75 | } 76 | } 77 | return []Metric{ 78 | { 79 | ParentPath: path, 80 | KeyName: key, 81 | startingValue: 0, 82 | originalType: val.Type(), 83 | }, 84 | } 85 | case bsontype.Double: 86 | return []Metric{ 87 | { 88 | ParentPath: path, 89 | KeyName: key, 90 | startingValue: normalizeFloat(val.Double()), 91 | originalType: val.Type(), 92 | }, 93 | } 94 | case bsontype.Int32: 95 | return []Metric{ 96 | { 97 | ParentPath: path, 98 | KeyName: key, 99 | startingValue: int64(val.Int32()), 100 | originalType: val.Type(), 101 | }, 102 | } 103 | case bsontype.Int64: 104 | return []Metric{ 105 | { 106 | ParentPath: path, 107 | KeyName: key, 108 | startingValue: val.Int64(), 109 | originalType: val.Type(), 110 | }, 111 | } 112 | case bsontype.DateTime: 113 | return []Metric{ 114 | { 115 | ParentPath: path, 116 | KeyName: key, 117 | startingValue: epochMs(val.Time()), 118 | originalType: val.Type(), 119 | }, 120 | } 121 | case bsontype.Timestamp: 122 | t, i := val.Timestamp() 123 | return []Metric{ 124 | { 125 | ParentPath: path, 126 | KeyName: key, 127 | startingValue: int64(t) * 1000, 128 | originalType: val.Type(), 129 | }, 130 | { 131 | ParentPath: path, 132 | KeyName: key + ".inc", 133 | startingValue: int64(i), 134 | originalType: val.Type(), 135 | }, 136 | } 137 | default: 138 | return []Metric{} 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /bson_restore.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "math" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/evergreen-ci/birch/bsontype" 8 | ) 9 | 10 | //////////////////////////////////////////////////////////////////////// 11 | // 12 | // Processores use to return rich (i.e. non-flat) structures from 13 | // metrics slices 14 | 15 | func restoreDocument(ref *birch.Document, sample int, metrics []Metric, idx int) (*birch.Document, int) { 16 | if ref == nil { 17 | return nil, 0 18 | } 19 | 20 | iter := ref.Iterator() 21 | doc := birch.DC.Make(ref.Len()) 22 | 23 | var elem *birch.Element 24 | 25 | for iter.Next() { 26 | refElem := iter.Element() 27 | 28 | elem, idx = restoreElement(refElem, sample, metrics, idx) 29 | if elem == nil { 30 | continue 31 | } 32 | doc.Append(elem) 33 | } 34 | 35 | return doc, idx 36 | } 37 | 38 | func restoreElement(ref *birch.Element, sample int, metrics []Metric, idx int) (*birch.Element, int) { 39 | switch ref.Value().Type() { 40 | case bsontype.ObjectID: 41 | return nil, idx 42 | case bsontype.String: 43 | return nil, idx 44 | case bsontype.Decimal128: 45 | return nil, idx 46 | case bsontype.Array: 47 | array := ref.Value().MutableArray() 48 | 49 | elems := make([]*birch.Element, 0, array.Len()) 50 | 51 | iter := array.Iterator() 52 | for iter.Next() { 53 | var item *birch.Element 54 | // TODO avoid Interface 55 | item, idx = restoreElement(birch.EC.Interface("", iter.Value()), sample, metrics, idx) 56 | if item == nil { 57 | continue 58 | } 59 | 60 | elems = append(elems, item) 61 | } 62 | 63 | if iter.Err() != nil { 64 | return nil, 0 65 | } 66 | 67 | out := make([]*birch.Value, len(elems)) 68 | 69 | for idx := range elems { 70 | out[idx] = elems[idx].Value() 71 | } 72 | 73 | return birch.EC.ArrayFromElements(ref.Key(), out...), idx 74 | case bsontype.EmbeddedDocument: 75 | var doc *birch.Document 76 | 77 | doc, idx = restoreDocument(ref.Value().MutableDocument(), sample, metrics, idx) 78 | return birch.EC.SubDocument(ref.Key(), doc), idx 79 | case bsontype.Boolean: 80 | value := metrics[idx].Values[sample] 81 | if value == 0 { 82 | return birch.EC.Boolean(ref.Key(), false), idx + 1 83 | } 84 | return birch.EC.Boolean(ref.Key(), true), idx + 1 85 | case bsontype.Double: 86 | return birch.EC.Double(ref.Key(), restoreFloat(metrics[idx].Values[sample])), idx + 1 87 | case bsontype.Int32: 88 | return birch.EC.Int32(ref.Key(), int32(metrics[idx].Values[sample])), idx + 1 89 | case bsontype.Int64: 90 | return birch.EC.Int64(ref.Key(), metrics[idx].Values[sample]), idx + 1 91 | case bsontype.DateTime: 92 | return birch.EC.Time(ref.Key(), timeEpocMs(metrics[idx].Values[sample])), idx + 1 93 | case bsontype.Timestamp: 94 | return birch.EC.Timestamp(ref.Key(), uint32(metrics[idx].Values[sample]), uint32(metrics[idx+1].Values[sample])), idx + 2 95 | default: 96 | return nil, idx 97 | } 98 | } 99 | 100 | func restoreFlat(t bsontype.Type, key string, value int64) (*birch.Element, bool) { 101 | switch t { 102 | case bsontype.Boolean: 103 | if value == 0 { 104 | return birch.EC.Boolean(key, false), true 105 | } 106 | return birch.EC.Boolean(key, true), true 107 | case bsontype.Double: 108 | return birch.EC.Double(key, math.Float64frombits(uint64(value))), true 109 | case bsontype.Int32: 110 | return birch.EC.Int32(key, int32(value)), true 111 | case bsontype.DateTime: 112 | return birch.EC.Time(key, timeEpocMs(value)), true 113 | default: 114 | return birch.EC.Int64(key, value), true 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /cmd/run-linter/run-linter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | type result struct { 15 | name string 16 | cmd string 17 | passed bool 18 | duration time.Duration 19 | output []string 20 | } 21 | 22 | // String prints the results of a linter run in gotest format. 23 | func (r *result) String() string { 24 | buf := &bytes.Buffer{} 25 | 26 | fmt.Fprintln(buf, "=== RUN", r.name) 27 | if r.passed { 28 | fmt.Fprintf(buf, "--- PASS: %s (%s)", r.name, r.duration) 29 | } else { 30 | fmt.Fprintf(buf, strings.Join(r.output, "\n")) 31 | fmt.Fprintf(buf, "--- FAIL: %s (%s)", r.name, r.duration) 32 | } 33 | 34 | return buf.String() 35 | } 36 | 37 | // fixup goes through the output and improves the output generated by 38 | // specific linters so that all output includes the relative path to the 39 | // error, instead of mixing relative and absolute paths. 40 | func (r *result) fixup(dirname string) { 41 | for idx, ln := range r.output { 42 | if strings.HasPrefix(ln, dirname) { 43 | r.output[idx] = ln[len(dirname)+1:] 44 | } 45 | } 46 | } 47 | 48 | // runs the golangci-lint on a list of packages; integrating with the "make lint" target. 49 | func main() { 50 | var ( 51 | lintArgs string 52 | lintBin string 53 | customLintersFlag string 54 | customLinters []string 55 | packageList string 56 | output string 57 | packages []string 58 | results []*result 59 | hasFailingTest bool 60 | ) 61 | 62 | flag.StringVar(&lintArgs, "lintArgs", "", "args to pass to golangci-lint") 63 | flag.StringVar(&lintBin, "lintBin", "", "path to golangci-lint") 64 | flag.StringVar(&packageList, "packages", "", "list of space separated packages") 65 | flag.StringVar(&customLintersFlag, "customLinters", "", "list of comma-separated custom linter commands") 66 | flag.StringVar(&output, "output", "", "output file for to write results.") 67 | flag.Parse() 68 | 69 | if len(customLintersFlag) != 0 { 70 | customLinters = strings.Split(customLintersFlag, ",") 71 | } 72 | packages = strings.Split(strings.Replace(packageList, "-", "/", -1), " ") 73 | dirname, _ := os.Getwd() 74 | cwd := filepath.Base(dirname) 75 | 76 | for _, pkg := range packages { 77 | pkgDir := "./" 78 | if cwd != pkg { 79 | pkgDir += pkg 80 | } 81 | splitLintArgs := strings.Split(lintArgs, " ") 82 | args := []string{lintBin, "run"} 83 | args = append(args, splitLintArgs...) 84 | args = append(args, pkgDir) 85 | 86 | startAt := time.Now() 87 | cmd := exec.Command(args[0], args[1:]...) 88 | cmd.Dir = dirname 89 | out, err := cmd.CombinedOutput() 90 | r := &result{ 91 | cmd: strings.Join(args, " "), 92 | name: "lint-" + strings.Replace(pkg, "/", "-", -1), 93 | passed: err == nil, 94 | duration: time.Since(startAt), 95 | output: strings.Split(string(out), "\n"), 96 | } 97 | 98 | for _, linter := range customLinters { 99 | customLinterStart := time.Now() 100 | linterArgs := strings.Split(linter, " ") 101 | linterArgs = append(linterArgs, pkgDir) 102 | cmd := exec.Command(linterArgs[0], linterArgs[1:]...) 103 | cmd.Dir = dirname 104 | out, err := cmd.CombinedOutput() 105 | r.passed = r.passed && err == nil 106 | r.duration += time.Since(customLinterStart) 107 | r.output = append(r.output, strings.Split(string(out), "\n")...) 108 | } 109 | r.fixup(dirname) 110 | 111 | if !r.passed { 112 | hasFailingTest = true 113 | } 114 | 115 | results = append(results, r) 116 | fmt.Println(r) 117 | } 118 | 119 | if output != "" { 120 | f, err := os.Create(output) 121 | if err != nil { 122 | os.Exit(1) 123 | } 124 | defer func() { 125 | if err != f.Close() { 126 | panic(err) 127 | } 128 | }() 129 | 130 | for _, r := range results { 131 | if _, err = f.WriteString(r.String() + "\n"); err != nil { 132 | fmt.Fprintf(os.Stderr, "%s", err) 133 | os.Exit(1) 134 | } 135 | } 136 | } 137 | 138 | if hasFailingTest { 139 | os.Exit(1) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /cmd/verify-mod-tidy/verify-mod-tidy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "time" 11 | 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | const ( 16 | goModFile = "go.mod" 17 | goSumFile = "go.sum" 18 | ) 19 | 20 | // verify-mod-tidy verifies that `go mod tidy` has been run to clean up the 21 | // go.mod and go.sum files. 22 | func main() { 23 | var ( 24 | goBin string 25 | timeout time.Duration 26 | ) 27 | 28 | flag.DurationVar(&timeout, "timeout", 0, "timeout for verifying modules are tidy") 29 | flag.StringVar(&goBin, "goBin", "go", "path to go binary to use for mod tidy check") 30 | flag.Parse() 31 | 32 | ctx := context.Background() 33 | if timeout != 0 { 34 | var cancel context.CancelFunc 35 | ctx, cancel = context.WithTimeout(ctx, timeout) 36 | defer cancel() 37 | } 38 | 39 | oldGoMod, oldGoSum, err := readModuleFiles() 40 | if err != nil { 41 | fmt.Fprintln(os.Stderr, err) 42 | os.Exit(1) 43 | } 44 | 45 | if err := runModTidy(ctx, goBin); err != nil { 46 | fmt.Fprintln(os.Stderr, err) 47 | os.Exit(1) 48 | } 49 | 50 | newGoMod, newGoSum, err := readModuleFiles() 51 | if err != nil { 52 | fmt.Fprintln(os.Stderr, err) 53 | os.Exit(1) 54 | } 55 | 56 | if !bytes.Equal(oldGoMod, newGoMod) || !bytes.Equal(oldGoSum, newGoSum) { 57 | fmt.Fprintf(os.Stderr, "%s and/or %s are not tidy - please run `go mod tidy`.\n", goModFile, goSumFile) 58 | writeModuleFiles(oldGoMod, oldGoSum) 59 | os.Exit(1) 60 | } 61 | } 62 | 63 | // readModuleFiles reads the contents of the go module files. 64 | func readModuleFiles() (goMod []byte, goSum []byte, err error) { 65 | goMod, err = os.ReadFile(goModFile) 66 | if err != nil { 67 | return nil, nil, errors.Wrapf(err, "reading file '%s'", goModFile) 68 | } 69 | goSum, err = os.ReadFile(goSumFile) 70 | if err != nil { 71 | return nil, nil, errors.Wrapf(err, "reading file '%s'", goSumFile) 72 | } 73 | return goMod, goSum, nil 74 | } 75 | 76 | // writeModuleFiles writes the contents of the go module files. 77 | func writeModuleFiles(goMod, goSum []byte) { 78 | if err := os.WriteFile(goModFile, goMod, 0600); err != nil { 79 | fmt.Fprintln(os.Stderr, err) 80 | } 81 | if err := os.WriteFile(goSumFile, goSum, 0600); err != nil { 82 | fmt.Fprintln(os.Stderr, err) 83 | } 84 | } 85 | 86 | // runModTidy runs the `go mod tidy` command with the given go binary. 87 | func runModTidy(ctx context.Context, goBin string) error { 88 | cmd := exec.CommandContext(ctx, goBin, "mod", "tidy") 89 | cmd.Stdout = os.Stdout 90 | cmd.Stderr = os.Stderr 91 | return errors.Wrap(cmd.Run(), "mod tidy") 92 | } 93 | -------------------------------------------------------------------------------- /collector.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | // Collector describes the interface for collecting and constructing 4 | // FTDC data series. Implementations may have different efficiencies 5 | // and handling of schema changes. 6 | // 7 | // The SetMetadata and Add methods both take interface{} values. These 8 | // are converted to bson documents; however it is an error to pass a 9 | // type based on a map. 10 | type Collector interface { 11 | // SetMetadata sets the metadata document for the collector or 12 | // chunk. This document is optional. Pass a nil to unset it, 13 | // or a different document to override a previous operation. 14 | SetMetadata(interface{}) error 15 | 16 | // Add extracts metrics from a document and appends it to the 17 | // current collector. These documents MUST all be 18 | // identical including field order. Returns an error if there 19 | // is a problem parsing the document or if the number of 20 | // metrics collected changes. 21 | Add(interface{}) error 22 | 23 | // Resolve renders the existing documents and outputs the full 24 | // FTDC chunk as a byte slice to be written out to storage. 25 | Resolve() ([]byte, error) 26 | 27 | // Reset clears the collector for future use. 28 | Reset() 29 | 30 | // Info reports on the current state of the collector for 31 | // introspection and to support schema change and payload 32 | // size. 33 | Info() CollectorInfo 34 | } 35 | 36 | // CollectorInfo reports on the current state of the collector and 37 | // provides introspection into the current state of the collector for 38 | // testing, transparency, and to support more complex collector 39 | // features, including payload size controls and schema change 40 | type CollectorInfo struct { 41 | MetricsCount int 42 | SampleCount int 43 | } 44 | -------------------------------------------------------------------------------- /collector_batch.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type batchCollector struct { 10 | maxSamples int 11 | chunks []*betterCollector 12 | } 13 | 14 | // NewBatchCollector constructs a collector implementation that 15 | // builds data chunks with payloads of the specified number of samples. 16 | // This implementation allows you break data into smaller components 17 | // for more efficient read operations. 18 | func NewBatchCollector(maxSamples int) Collector { 19 | return newBatchCollector(maxSamples) 20 | } 21 | 22 | func newBatchCollector(size int) *batchCollector { 23 | return &batchCollector{ 24 | maxSamples: size, 25 | chunks: []*betterCollector{ 26 | { 27 | maxDeltas: size, 28 | }, 29 | }, 30 | } 31 | } 32 | 33 | func (c *batchCollector) Info() CollectorInfo { 34 | out := CollectorInfo{} 35 | for _, c := range c.chunks { 36 | info := c.Info() 37 | out.MetricsCount += info.MetricsCount 38 | out.SampleCount += info.SampleCount 39 | } 40 | return out 41 | } 42 | 43 | func (c *batchCollector) Reset() { 44 | c.chunks = []*betterCollector{{maxDeltas: c.maxSamples}} 45 | } 46 | 47 | func (c *batchCollector) SetMetadata(in interface{}) error { 48 | return errors.WithStack(c.chunks[0].SetMetadata(in)) 49 | } 50 | 51 | func (c *batchCollector) Add(in interface{}) error { 52 | doc, err := readDocument(in) 53 | if err != nil { 54 | return errors.WithStack(err) 55 | } 56 | 57 | last := c.chunks[len(c.chunks)-1] 58 | if last.Info().SampleCount >= c.maxSamples { 59 | last = &betterCollector{maxDeltas: c.maxSamples} 60 | c.chunks = append(c.chunks, last) 61 | } 62 | 63 | return errors.WithStack(last.Add(doc)) 64 | } 65 | 66 | func (c *batchCollector) Resolve() ([]byte, error) { 67 | buf := &bytes.Buffer{} 68 | 69 | for _, chunk := range c.chunks { 70 | out, err := chunk.Resolve() 71 | if err != nil { 72 | return nil, errors.WithStack(err) 73 | } 74 | 75 | _, _ = buf.Write(out) 76 | } 77 | 78 | return buf.Bytes(), nil 79 | } 80 | -------------------------------------------------------------------------------- /collector_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkCollectorInterface(b *testing.B) { 9 | ctx, cancel := context.WithCancel(context.Background()) 10 | defer cancel() 11 | 12 | collectors := createCollectors(ctx) 13 | for _, collect := range collectors { 14 | if collect.skipBench { 15 | continue 16 | } 17 | 18 | b.Run(collect.name, func(b *testing.B) { 19 | tests := createTests() 20 | for _, test := range tests { 21 | if test.skipBench { 22 | continue 23 | } 24 | 25 | b.Run(test.name, func(b *testing.B) { 26 | collector := collect.factory() 27 | b.Run("Add", func(b *testing.B) { 28 | for n := 0; n < b.N; n++ { 29 | collector.Add(test.docs[n%len(test.docs)]) // nolint 30 | } 31 | }) 32 | b.Run("Resolve", func(b *testing.B) { 33 | for n := 0; n < b.N; n++ { 34 | collector.Resolve() // nolint 35 | } 36 | }) 37 | }) 38 | } 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /collector_better.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | "time" 6 | 7 | "github.com/evergreen-ci/birch" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type betterCollector struct { 12 | metadata *birch.Document 13 | reference *birch.Document 14 | startedAt time.Time 15 | lastSample *extractedMetrics 16 | deltas []int64 17 | numSamples int 18 | maxDeltas int 19 | } 20 | 21 | // NewBasicCollector provides a basic FTDC data collector that mirrors 22 | // the server's implementation. The Add method will error if you 23 | // attempt to add more than the specified number of records (plus one, 24 | // as the reference/schema document doesn't count). 25 | func NewBaseCollector(maxSize int) Collector { 26 | return &betterCollector{ 27 | maxDeltas: maxSize, 28 | } 29 | } 30 | 31 | func (c *betterCollector) SetMetadata(in interface{}) error { 32 | doc, err := readDocument(in) 33 | if err != nil { 34 | return errors.WithStack(err) 35 | } 36 | 37 | c.metadata = doc 38 | return nil 39 | } 40 | func (c *betterCollector) Reset() { 41 | c.reference = nil 42 | c.lastSample = nil 43 | c.deltas = nil 44 | c.numSamples = 0 45 | } 46 | 47 | func (c *betterCollector) Info() CollectorInfo { 48 | var num int 49 | 50 | if c.reference != nil { 51 | num++ 52 | } 53 | 54 | var metricsCount int 55 | if c.lastSample != nil { 56 | metricsCount = len(c.lastSample.values) 57 | } 58 | 59 | return CollectorInfo{ 60 | SampleCount: num + c.numSamples, 61 | MetricsCount: metricsCount, 62 | } 63 | } 64 | 65 | func (c *betterCollector) Add(in interface{}) error { 66 | doc, err := readDocument(in) 67 | if err != nil { 68 | return errors.WithStack(err) 69 | } 70 | 71 | var metrics extractedMetrics 72 | if c.reference == nil { 73 | c.reference = doc 74 | metrics, err = extractMetricsFromDocument(doc) 75 | if err != nil { 76 | return errors.WithStack(err) 77 | } 78 | c.startedAt = metrics.ts 79 | c.lastSample = &metrics 80 | c.deltas = make([]int64, c.maxDeltas*len(c.lastSample.values)) 81 | return nil 82 | } 83 | 84 | if c.numSamples >= c.maxDeltas { 85 | return errors.New("collector is overfull") 86 | } 87 | 88 | metrics, err = extractMetricsFromDocument(doc) 89 | if err != nil { 90 | return errors.WithStack(err) 91 | } 92 | 93 | if len(metrics.values) != len(c.lastSample.values) { 94 | return errors.Errorf("unexpected schema change detected for sample %d: [current=%d vs previous=%d]", 95 | c.numSamples+1, len(metrics.values), len(c.lastSample.values), 96 | ) 97 | } 98 | 99 | var delta int64 100 | for idx := range metrics.values { 101 | if metrics.types[idx] != c.lastSample.types[idx] { 102 | return errors.Errorf("unexpected schema change detected for sample types: [current=%v vs previous=%v]", 103 | metrics.types, c.lastSample.types) 104 | } 105 | delta, err = extractDelta(metrics.values[idx], c.lastSample.values[idx]) 106 | if err != nil { 107 | return errors.Wrap(err, "problem parsing data") 108 | } 109 | c.deltas[getOffset(c.maxDeltas, c.numSamples, idx)] = delta 110 | } 111 | 112 | c.numSamples++ 113 | c.lastSample = &metrics 114 | 115 | return nil 116 | } 117 | 118 | func (c *betterCollector) Resolve() ([]byte, error) { 119 | if c.reference == nil { 120 | return nil, errors.New("no reference document") 121 | } 122 | 123 | data, err := c.getPayload() 124 | if err != nil { 125 | return nil, errors.WithStack(err) 126 | } 127 | 128 | buf := bytes.NewBuffer([]byte{}) 129 | if c.metadata != nil { 130 | _, err = birch.NewDocument( 131 | birch.EC.Time("_id", c.startedAt), 132 | birch.EC.Int32("type", 0), 133 | birch.EC.SubDocument("doc", c.metadata)).WriteTo(buf) 134 | if err != nil { 135 | return nil, errors.Wrap(err, "problem writing metadata document") 136 | } 137 | } 138 | 139 | _, err = birch.NewDocument( 140 | birch.EC.Time("_id", c.startedAt), 141 | birch.EC.Int32("type", 1), 142 | birch.EC.Binary("data", data)).WriteTo(buf) 143 | if err != nil { 144 | return nil, errors.Wrap(err, "problem writing metric chunk document") 145 | } 146 | 147 | return buf.Bytes(), nil 148 | } 149 | 150 | func (c *betterCollector) getPayload() ([]byte, error) { 151 | payload := bytes.NewBuffer([]byte{}) 152 | if _, err := c.reference.WriteTo(payload); err != nil { 153 | return nil, errors.Wrap(err, "problem writing reference document") 154 | } 155 | 156 | payload.Write(encodeSizeValue(uint32(len(c.lastSample.values)))) 157 | payload.Write(encodeSizeValue(uint32(c.numSamples))) 158 | zeroCount := int64(0) 159 | for i := 0; i < len(c.lastSample.values); i++ { 160 | for j := 0; j < c.numSamples; j++ { 161 | delta := c.deltas[getOffset(c.maxDeltas, j, i)] 162 | 163 | if delta == 0 { 164 | zeroCount++ 165 | continue 166 | } 167 | 168 | if zeroCount > 0 { 169 | payload.Write(encodeValue(0)) 170 | payload.Write(encodeValue(zeroCount - 1)) 171 | zeroCount = 0 172 | } 173 | 174 | payload.Write(encodeValue(delta)) 175 | } 176 | } 177 | if zeroCount > 0 { 178 | payload.Write(encodeValue(0)) 179 | payload.Write(encodeValue(zeroCount - 1)) 180 | } 181 | 182 | data, err := compressBuffer(payload.Bytes()) 183 | if err != nil { 184 | return nil, errors.Wrap(err, "problem compressing payload") 185 | } 186 | 187 | return data, nil 188 | } 189 | -------------------------------------------------------------------------------- /collector_buffered.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/mongodb/ftdc/util" 7 | ) 8 | 9 | type bufferedCollector struct { 10 | Collector 11 | pipe chan interface{} 12 | catcher util.Catcher 13 | ctx context.Context 14 | } 15 | 16 | // NewBufferedCollector wraps an existing collector with a buffer to 17 | // normalize throughput to an underlying collector implementation. 18 | func NewBufferedCollector(ctx context.Context, size int, coll Collector) Collector { 19 | c := &bufferedCollector{ 20 | Collector: coll, 21 | pipe: make(chan interface{}, size), 22 | catcher: util.NewCatcher(), 23 | ctx: ctx, 24 | } 25 | 26 | go func() { 27 | for { 28 | select { 29 | case <-ctx.Done(): 30 | if len(c.pipe) != 0 { 31 | for in := range c.pipe { 32 | c.catcher.Add(c.Collector.Add(in)) 33 | } 34 | } 35 | 36 | return 37 | case in := <-c.pipe: 38 | c.catcher.Add(c.Collector.Add(in)) 39 | } 40 | } 41 | }() 42 | return c 43 | } 44 | 45 | func (c *bufferedCollector) Add(in interface{}) error { 46 | select { 47 | case <-c.ctx.Done(): 48 | return c.ctx.Err() 49 | case c.pipe <- in: 50 | return nil 51 | } 52 | } 53 | 54 | func (c *bufferedCollector) Resolve() ([]byte, error) { 55 | if c.catcher.HasErrors() { 56 | return nil, c.catcher.Resolve() 57 | } 58 | 59 | return c.Collector.Resolve() 60 | } 61 | -------------------------------------------------------------------------------- /collector_dynamic.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type dynamicCollector struct { 10 | maxSamples int 11 | chunks []*batchCollector 12 | hash string 13 | currentNum int 14 | } 15 | 16 | // NewDynamicCollector constructs a Collector that records metrics 17 | // from documents, creating new chunks when either the number of 18 | // samples collected exceeds the specified max sample count OR 19 | // the schema changes. 20 | // 21 | // There is some overhead associated with detecting schema changes, 22 | // particularly for documents with more complex schemas, so you may 23 | // wish to opt for a simpler collector in some cases. 24 | func NewDynamicCollector(maxSamples int) Collector { 25 | return &dynamicCollector{ 26 | maxSamples: maxSamples, 27 | chunks: []*batchCollector{ 28 | newBatchCollector(maxSamples), 29 | }, 30 | } 31 | } 32 | 33 | func (c *dynamicCollector) Info() CollectorInfo { 34 | out := CollectorInfo{} 35 | for _, c := range c.chunks { 36 | info := c.Info() 37 | out.MetricsCount += info.MetricsCount 38 | out.SampleCount += info.SampleCount 39 | } 40 | return out 41 | } 42 | 43 | func (c *dynamicCollector) Reset() { 44 | c.chunks = []*batchCollector{newBatchCollector(c.maxSamples)} 45 | c.hash = "" 46 | } 47 | 48 | func (c *dynamicCollector) SetMetadata(in interface{}) error { 49 | return errors.WithStack(c.chunks[0].SetMetadata(in)) 50 | } 51 | 52 | func (c *dynamicCollector) Add(in interface{}) error { 53 | doc, err := readDocument(in) 54 | if err != nil { 55 | return errors.WithStack(err) 56 | } 57 | 58 | if c.hash == "" { 59 | docHash, num := metricKeyHash(doc) 60 | c.hash = docHash 61 | c.currentNum = num 62 | return errors.WithStack(c.chunks[0].Add(doc)) 63 | } 64 | 65 | lastChunk := c.chunks[len(c.chunks)-1] 66 | 67 | docHash, _ := metricKeyHash(doc) 68 | if c.hash == docHash { 69 | return errors.WithStack(lastChunk.Add(doc)) 70 | } 71 | 72 | chunk := newBatchCollector(c.maxSamples) 73 | c.chunks = append(c.chunks, chunk) 74 | 75 | return errors.WithStack(chunk.Add(doc)) 76 | } 77 | 78 | func (c *dynamicCollector) Resolve() ([]byte, error) { 79 | buf := bytes.NewBuffer([]byte{}) 80 | for _, chunk := range c.chunks { 81 | out, err := chunk.Resolve() 82 | if err != nil { 83 | return nil, errors.WithStack(err) 84 | } 85 | 86 | _, _ = buf.Write(out) 87 | } 88 | 89 | return buf.Bytes(), nil 90 | } 91 | -------------------------------------------------------------------------------- /collector_sample.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type samplingCollector struct { 10 | minimumInterval time.Duration 11 | lastCollection time.Time 12 | Collector 13 | } 14 | 15 | // NewSamplingCollector wraps a different collector implementation and 16 | // provides an implementation of the Add method that skips collection 17 | // of results if the specified minimumInterval has not elapsed since 18 | // the last collection. 19 | func NewSamplingCollector(minimumInterval time.Duration, collector Collector) Collector { 20 | return &samplingCollector{ 21 | minimumInterval: minimumInterval, 22 | Collector: collector, 23 | } 24 | } 25 | 26 | func (c *samplingCollector) Add(d interface{}) error { 27 | if time.Since(c.lastCollection) < c.minimumInterval { 28 | return nil 29 | } 30 | 31 | c.lastCollection = time.Now() 32 | 33 | return errors.WithStack(c.Collector.Add(d)) 34 | } 35 | -------------------------------------------------------------------------------- /collector_sample_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/mongodb/ftdc/testutil" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestSamplingCollector(t *testing.T) { 12 | collector := NewSamplingCollector(10*time.Millisecond, &betterCollector{maxDeltas: 20}) 13 | assert.Equal(t, 0, collector.Info().SampleCount) 14 | for i := 0; i < 10; i++ { 15 | assert.NoError(t, collector.Add(testutil.RandFlatDocument(20))) 16 | } 17 | assert.Equal(t, 1, collector.Info().SampleCount) 18 | 19 | for i := 0; i < 4; i++ { 20 | time.Sleep(10 * time.Millisecond) 21 | assert.NoError(t, collector.Add(testutil.RandFlatDocument(20))) 22 | } 23 | 24 | assert.Equal(t, 5, collector.Info().SampleCount) 25 | } 26 | -------------------------------------------------------------------------------- /collector_streaming.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type streamingCollector struct { 10 | output io.Writer 11 | maxSamples int 12 | count int 13 | Collector 14 | } 15 | 16 | // NewStreamingCollector wraps the underlying collector, writing the 17 | // data to the underlying writer after the underlying collector is 18 | // filled. This is similar to the batch collector, but allows the 19 | // collector to drop FTDC data from memory. Chunks are flushed to disk 20 | // when the collector as collected the "maxSamples" number of 21 | // samples during the Add operation. 22 | func NewStreamingCollector(maxSamples int, writer io.Writer) Collector { 23 | return newStreamingCollector(maxSamples, writer) 24 | } 25 | 26 | func newStreamingCollector(maxSamples int, writer io.Writer) *streamingCollector { 27 | return &streamingCollector{ 28 | maxSamples: maxSamples, 29 | output: writer, 30 | Collector: &betterCollector{ 31 | maxDeltas: maxSamples, 32 | }, 33 | } 34 | } 35 | 36 | func (c *streamingCollector) Reset() { c.count = 0; c.Collector.Reset() } 37 | func (c *streamingCollector) Add(in interface{}) error { 38 | if c.count >= c.maxSamples { 39 | if err := FlushCollector(c, c.output); err != nil { 40 | return errors.Wrap(err, "problem flushing collector contents") 41 | } 42 | } 43 | 44 | if err := c.Collector.Add(in); err != nil { 45 | return errors.Wrapf(err, "adding sample #%d", c.count+1) 46 | } 47 | c.count++ 48 | 49 | return nil 50 | } 51 | 52 | // FlushCollector writes the contents of a collector out to an 53 | // io.Writer. This is useful in the context of any collector, but is 54 | // particularly useful in the context of streaming collectors, which 55 | // flush data periodically and may have cached data. 56 | func FlushCollector(c Collector, writer io.Writer) error { 57 | if writer == nil { 58 | return errors.New("invalid writer") 59 | } 60 | if c.Info().SampleCount == 0 { 61 | return nil 62 | } 63 | payload, err := c.Resolve() 64 | if err != nil { 65 | return errors.WithStack(err) 66 | } 67 | 68 | n, err := writer.Write(payload) 69 | if err != nil { 70 | return errors.WithStack(err) 71 | } 72 | if n != len(payload) { 73 | return errors.New("problem flushing data") 74 | } 75 | c.Reset() 76 | return nil 77 | } 78 | 79 | type streamingDynamicCollector struct { 80 | output io.Writer 81 | hash string 82 | metricCount int 83 | *streamingCollector 84 | } 85 | 86 | // NewStreamingDynamicCollector has the same semantics as the dynamic 87 | // collector but wraps the streaming collector rather than the batch 88 | // collector. Chunks are flushed during the Add() operation when the 89 | // schema changes or the chunk is full. 90 | func NewStreamingDynamicCollector(max int, writer io.Writer) Collector { 91 | return &streamingDynamicCollector{ 92 | output: writer, 93 | streamingCollector: newStreamingCollector(max, writer), 94 | } 95 | } 96 | 97 | func (c *streamingDynamicCollector) Reset() { 98 | c.streamingCollector = newStreamingCollector(c.streamingCollector.maxSamples, c.output) 99 | c.metricCount = 0 100 | c.hash = "" 101 | } 102 | 103 | func (c *streamingDynamicCollector) Add(in interface{}) error { 104 | doc, err := readDocument(in) 105 | if err != nil { 106 | return errors.WithStack(err) 107 | } 108 | 109 | docHash, num := metricKeyHash(doc) 110 | if c.hash == "" { 111 | c.hash = docHash 112 | c.metricCount = num 113 | if c.streamingCollector.count > 0 { 114 | if err := FlushCollector(c, c.output); err != nil { 115 | return errors.WithStack(err) 116 | } 117 | } 118 | return errors.WithStack(c.streamingCollector.Add(doc)) 119 | } 120 | 121 | if c.metricCount != num || c.hash != docHash { 122 | if err := FlushCollector(c, c.output); err != nil { 123 | return errors.WithStack(err) 124 | } 125 | } 126 | 127 | return errors.WithStack(c.streamingCollector.Add(doc)) 128 | } 129 | -------------------------------------------------------------------------------- /collector_sync.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type synchronizedCollector struct { 8 | Collector 9 | mu sync.RWMutex 10 | } 11 | 12 | // NewSynchronizedCollector wraps an existing collector in a 13 | // synchronized wrapper that guards against incorrect concurrent 14 | // access. 15 | func NewSynchronizedCollector(coll Collector) Collector { 16 | return &synchronizedCollector{ 17 | Collector: coll, 18 | } 19 | } 20 | 21 | func (c *synchronizedCollector) Add(in interface{}) error { 22 | c.mu.Lock() 23 | defer c.mu.Unlock() 24 | return c.Collector.Add(in) 25 | } 26 | 27 | func (c *synchronizedCollector) SetMetadata(in interface{}) error { 28 | c.mu.Lock() 29 | defer c.mu.Unlock() 30 | 31 | return c.Collector.SetMetadata(in) 32 | } 33 | 34 | func (c *synchronizedCollector) Resolve() ([]byte, error) { 35 | c.mu.Lock() 36 | defer c.mu.Unlock() 37 | 38 | return c.Collector.Resolve() 39 | } 40 | 41 | func (c *synchronizedCollector) Reset() { 42 | c.mu.Lock() 43 | defer c.mu.Unlock() 44 | 45 | c.Collector.Reset() 46 | } 47 | 48 | func (c *synchronizedCollector) Info() CollectorInfo { 49 | c.mu.RLock() 50 | defer c.mu.RUnlock() 51 | 52 | return c.Collector.Info() 53 | } 54 | -------------------------------------------------------------------------------- /csv.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "encoding/csv" 6 | "fmt" 7 | "io" 8 | "os" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/evergreen-ci/birch" 13 | "github.com/evergreen-ci/birch/bsontype" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | func (c *Chunk) getFieldNames() []string { 18 | fieldNames := make([]string, len(c.Metrics)) 19 | for idx, m := range c.Metrics { 20 | fieldNames[idx] = m.Key() 21 | } 22 | return fieldNames 23 | } 24 | 25 | func (c *Chunk) getRecord(i int) []string { 26 | fields := make([]string, len(c.Metrics)) 27 | for idx, m := range c.Metrics { 28 | switch m.originalType { 29 | case bsontype.Double, bsontype.Int32, bsontype.Int64, bsontype.Boolean, bsontype.Timestamp: 30 | fields[idx] = strconv.FormatInt(m.Values[i], 10) 31 | case bsontype.DateTime: 32 | fields[idx] = time.Unix(m.Values[i]/1000, 0).Format(time.RFC3339) 33 | } 34 | } 35 | return fields 36 | } 37 | 38 | // WriteCSV exports the contents of a stream of chunks as CSV. Returns 39 | // an error if the number of metrics changes between points, or if 40 | // there are any errors writing data. 41 | func WriteCSV(ctx context.Context, iter *ChunkIterator, writer io.Writer) error { 42 | var numFields int 43 | csvw := csv.NewWriter(writer) 44 | for iter.Next() { 45 | if ctx.Err() != nil { 46 | return errors.New("operation aborted") 47 | } 48 | chunk := iter.Chunk() 49 | if numFields == 0 { 50 | fieldNames := chunk.getFieldNames() 51 | if err := csvw.Write(fieldNames); err != nil { 52 | return errors.Wrap(err, "problem writing field names") 53 | } 54 | numFields = len(fieldNames) 55 | } else if numFields != len(chunk.Metrics) { 56 | return errors.New("unexpected schema change detected") 57 | } 58 | 59 | for i := 0; i < chunk.nPoints; i++ { 60 | record := chunk.getRecord(i) 61 | if err := csvw.Write(record); err != nil { 62 | return errors.Wrapf(err, "problem writing csv record %d of %d", i, chunk.nPoints) 63 | } 64 | } 65 | csvw.Flush() 66 | if err := csvw.Error(); err != nil { 67 | return errors.Wrapf(err, "problem flushing csv data") 68 | } 69 | } 70 | if err := iter.Err(); err != nil { 71 | return errors.Wrap(err, "problem reading chunks") 72 | } 73 | 74 | return nil 75 | } 76 | 77 | func getCSVFile(prefix string, count int) (io.WriteCloser, error) { 78 | fn := fmt.Sprintf("%s.%d.csv", prefix, count) 79 | writer, err := os.Create(fn) 80 | if err != nil { 81 | return nil, errors.Wrapf(err, "provlem opening file %s", fn) 82 | } 83 | return writer, nil 84 | } 85 | 86 | // DumpCSV writes a sequence of chunks to CSV files, creating new 87 | // files if the iterator detects a schema change, using only the 88 | // number of fields in the chunk to detect schema changes. DumpCSV 89 | // writes a header row to each file. 90 | // 91 | // The file names are constructed as "prefix..csv". 92 | func DumpCSV(ctx context.Context, iter *ChunkIterator, prefix string) error { 93 | var ( 94 | err error 95 | writer io.WriteCloser 96 | numFields int 97 | fileCount int 98 | csvw *csv.Writer 99 | ) 100 | for iter.Next() { 101 | if ctx.Err() != nil { 102 | return errors.New("operation aborted") 103 | } 104 | 105 | if writer == nil { 106 | writer, err = getCSVFile(prefix, fileCount) 107 | if err != nil { 108 | return errors.WithStack(err) 109 | } 110 | csvw = csv.NewWriter(writer) 111 | fileCount++ 112 | } 113 | 114 | chunk := iter.Chunk() 115 | if numFields == 0 { 116 | fieldNames := chunk.getFieldNames() 117 | if err = csvw.Write(fieldNames); err != nil { 118 | return errors.Wrap(err, "problem writing field names") 119 | } 120 | numFields = len(fieldNames) 121 | } else if numFields != len(chunk.Metrics) { 122 | if err = writer.Close(); err != nil { 123 | return errors.Wrap(err, "problem flushing and closing file") 124 | } 125 | 126 | writer, err = getCSVFile(prefix, fileCount) 127 | if err != nil { 128 | return errors.WithStack(err) 129 | } 130 | 131 | csvw = csv.NewWriter(writer) 132 | fileCount++ 133 | 134 | // now dump header 135 | fieldNames := chunk.getFieldNames() 136 | if err := csvw.Write(fieldNames); err != nil { 137 | return errors.Wrap(err, "problem writing field names") 138 | } 139 | numFields = len(fieldNames) 140 | } 141 | 142 | for i := 0; i < chunk.nPoints; i++ { 143 | record := chunk.getRecord(i) 144 | if err := csvw.Write(record); err != nil { 145 | return errors.Wrapf(err, "problem writing csv record %d of %d", i, chunk.nPoints) 146 | } 147 | } 148 | csvw.Flush() 149 | if err := csvw.Error(); err != nil { 150 | return errors.Wrapf(err, "problem flushing csv data") 151 | } 152 | } 153 | if err := iter.Err(); err != nil { 154 | return errors.Wrap(err, "problem reading chunks") 155 | } 156 | 157 | if writer == nil { 158 | return nil 159 | } 160 | if err := writer.Close(); err != nil { 161 | return errors.Wrap(err, "problem writing files to disk") 162 | 163 | } 164 | return nil 165 | } 166 | 167 | // ConvertFromCSV takes an input stream and writes ftdc compressed 168 | // data to the provided output writer. 169 | // 170 | // If the number of fields changes in the CSV fields, the first field 171 | // with the changed number of fields becomes the header for the 172 | // subsequent documents in the stream. 173 | func ConvertFromCSV(ctx context.Context, bucketSize int, input io.Reader, output io.Writer) error { 174 | csvr := csv.NewReader(input) 175 | 176 | header, err := csvr.Read() 177 | if err != nil { 178 | return errors.Wrap(err, "problem reading error") 179 | } 180 | 181 | collector := NewStreamingDynamicCollector(bucketSize, output) 182 | 183 | defer func() { 184 | if err != nil && (errors.Cause(err) != context.Canceled || errors.Cause(err) != context.DeadlineExceeded) { 185 | err = errors.Wrap(err, "omitting final flush, because of prior error") 186 | } 187 | err = FlushCollector(collector, output) 188 | }() 189 | 190 | var record []string 191 | for { 192 | if ctx.Err() != nil { 193 | // this is weird so that the defer can work 194 | err = errors.Wrap(err, "operation aborted") 195 | return err 196 | } 197 | 198 | record, err = csvr.Read() 199 | if err == io.EOF { 200 | // this is weird so that the defer can work 201 | err = nil 202 | return err 203 | } 204 | 205 | if err != nil { 206 | if pr, ok := err.(*csv.ParseError); ok && pr.Err == csv.ErrFieldCount { 207 | header = record 208 | continue 209 | } 210 | err = errors.Wrap(err, "problem parsing csv") 211 | return err 212 | } 213 | if len(record) != len(header) { 214 | return errors.New("unexpected field count change") 215 | } 216 | 217 | elems := make([]*birch.Element, 0, len(header)) 218 | for idx := range record { 219 | var val int 220 | val, err = strconv.Atoi(record[idx]) 221 | if err != nil { 222 | continue 223 | } 224 | elems = append(elems, birch.EC.Int64(header[idx], int64(val))) 225 | } 226 | 227 | if err = collector.Add(birch.NewDocument(elems...)); err != nil { 228 | return errors.WithStack(err) 229 | } 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /csv_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/csv" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/evergreen-ci/birch" 14 | "github.com/mongodb/ftdc/testutil" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func TestWriteCSVIntegration(t *testing.T) { 20 | ctx, cancel := context.WithCancel(context.Background()) 21 | defer cancel() 22 | 23 | tmp, err := ioutil.TempDir("", "ftdc-csv-") 24 | require.NoError(t, err) 25 | defer func() { require.NoError(t, os.RemoveAll(tmp)) }() 26 | 27 | t.Run("Write", func(t *testing.T) { 28 | iter := ReadChunks(ctx, bytes.NewBuffer(newChunk(10))) 29 | out := &bytes.Buffer{} 30 | err := WriteCSV(ctx, iter, out) 31 | require.NoError(t, err) 32 | 33 | lines := strings.Split(out.String(), "\n") 34 | assert.Len(t, lines, 12) 35 | }) 36 | t.Run("ResuseIterPass", func(t *testing.T) { 37 | iter := ReadChunks(ctx, bytes.NewBuffer(newChunk(10))) 38 | err := DumpCSV(ctx, iter, filepath.Join(tmp, "dump")) 39 | require.NoError(t, err) 40 | err = DumpCSV(ctx, iter, filepath.Join(tmp, "dump")) 41 | require.NoError(t, err) 42 | }) 43 | t.Run("Dump", func(t *testing.T) { 44 | iter := ReadChunks(ctx, bytes.NewBuffer(newChunk(10))) 45 | err := DumpCSV(ctx, iter, filepath.Join(tmp, "dump")) 46 | require.NoError(t, err) 47 | }) 48 | t.Run("DumpMixed", func(t *testing.T) { 49 | iter := ReadChunks(ctx, bytes.NewBuffer(newMixedChunk(10))) 50 | err := DumpCSV(ctx, iter, filepath.Join(tmp, "dump")) 51 | require.NoError(t, err) 52 | }) 53 | t.Run("WriteWithSchemaChange", func(t *testing.T) { 54 | iter := ReadChunks(ctx, bytes.NewBuffer(newMixedChunk(10))) 55 | out := &bytes.Buffer{} 56 | err := WriteCSV(ctx, iter, out) 57 | 58 | require.Error(t, err) 59 | }) 60 | } 61 | 62 | func TestReadCSVIntegration(t *testing.T) { 63 | ctx, cancel := context.WithCancel(context.Background()) 64 | defer cancel() 65 | 66 | for _, test := range []struct { 67 | Name string 68 | Iter *ChunkIterator 69 | Rows int 70 | Fields int 71 | }{ 72 | { 73 | Name: "SimpleFlat", 74 | Iter: produceMockChunkIter(ctx, 1000, func() *birch.Document { return testutil.RandFlatDocument(15) }), 75 | Rows: 1000, 76 | Fields: 15, 77 | }, 78 | { 79 | Name: "LargerFlat", 80 | Iter: produceMockChunkIter(ctx, 1000, func() *birch.Document { return testutil.RandFlatDocument(50) }), 81 | Rows: 1000, 82 | Fields: 50, 83 | }, 84 | { 85 | Name: "Complex", 86 | Iter: produceMockChunkIter(ctx, 1000, func() *birch.Document { return testutil.RandComplexDocument(20, 3) }), 87 | Rows: 1000, 88 | Fields: 100, 89 | }, 90 | { 91 | Name: "LargComplex", 92 | Iter: produceMockChunkIter(ctx, 1000, func() *birch.Document { return testutil.RandComplexDocument(100, 10) }), 93 | Rows: 1000, 94 | Fields: 190, 95 | }, 96 | } { 97 | t.Run(test.Name, func(t *testing.T) { 98 | buf := &bytes.Buffer{} 99 | err := WriteCSV(ctx, test.Iter, buf) 100 | require.NoError(t, err) 101 | 102 | out := &bytes.Buffer{} 103 | err = ConvertFromCSV(ctx, test.Rows, buf, out) 104 | require.NoError(t, err) 105 | 106 | iter := ReadMetrics(ctx, out) 107 | count := 0 108 | for iter.Next() { 109 | count++ 110 | doc := iter.Document() 111 | assert.Equal(t, test.Fields, doc.Len()) 112 | } 113 | assert.Equal(t, test.Rows, count) 114 | }) 115 | } 116 | t.Run("SchemaChangeGrow", func(t *testing.T) { 117 | buf := &bytes.Buffer{} 118 | csvw := csv.NewWriter(buf) 119 | require.NoError(t, csvw.Write([]string{"a", "b", "c", "d"})) 120 | for j := 0; j < 2; j++ { 121 | for i := 0; i < 10; i++ { 122 | require.NoError(t, csvw.Write([]string{"1", "2", "3", "4"})) 123 | } 124 | require.NoError(t, csvw.Write([]string{"1", "2", "3", "4", "5"})) 125 | } 126 | csvw.Flush() 127 | 128 | assert.Error(t, ConvertFromCSV(ctx, 1000, buf, &bytes.Buffer{})) 129 | }) 130 | t.Run("SchemaChangeShrink", func(t *testing.T) { 131 | buf := &bytes.Buffer{} 132 | csvw := csv.NewWriter(buf) 133 | require.NoError(t, csvw.Write([]string{"a", "b", "c", "d"})) 134 | for j := 0; j < 2; j++ { 135 | for i := 0; i < 10; i++ { 136 | require.NoError(t, csvw.Write([]string{"1", "2", "3", "4"})) 137 | } 138 | require.NoError(t, csvw.Write([]string{"1", "2"})) 139 | } 140 | csvw.Flush() 141 | 142 | assert.Error(t, ConvertFromCSV(ctx, 1000, buf, &bytes.Buffer{})) 143 | }) 144 | } 145 | -------------------------------------------------------------------------------- /events/collector.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "time" 7 | 8 | "github.com/mongodb/ftdc" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | // Collector wraps the ftdc.Collector interface and adds 13 | // specific awareness of the Performance type from this package. These 14 | // collectors should be responsible for cumulative summing of values, 15 | // when appropriate. 16 | // 17 | // In general, implementations should obstruct calls to underlying 18 | // collectors Add() method to avoid confusion, either by panicing or 19 | // by no-oping. 20 | type Collector interface { 21 | AddEvent(*Performance) error 22 | ftdc.Collector 23 | } 24 | 25 | type basicCumulativeCollector struct { 26 | ftdc.Collector 27 | current *Performance 28 | } 29 | 30 | // NewBasicCollector produces a collector implementation that adds 31 | // Performance points to the underlying FTDC collector. Counter values 32 | // in the point are added to the previous point so that the values are 33 | // cumulative. 34 | // 35 | // This event Collector implementation captures the maximal amount of 36 | // fidelity and should be used except when dictated by retention 37 | // strategy. 38 | func NewBasicCollector(fc ftdc.Collector) Collector { 39 | return &basicCumulativeCollector{ 40 | Collector: fc, 41 | } 42 | } 43 | 44 | func (c *basicCumulativeCollector) Add(interface{}) error { return nil } 45 | func (c *basicCumulativeCollector) AddEvent(in *Performance) error { 46 | if in == nil { 47 | return errors.New("cannot add nil performance event") 48 | } 49 | if c.current == nil { 50 | c.current = in 51 | return c.Collector.Add(c.current) 52 | } 53 | 54 | c.current.Add(in) 55 | return c.Collector.Add(c.current) 56 | } 57 | 58 | type passthroughCollector struct { 59 | ftdc.Collector 60 | } 61 | 62 | // NewPassthroughCollector constructs a collector that does not sum 63 | // Performance events and just passes them directly to the underlying 64 | // collector. 65 | func NewPassthroughCollector(fc ftdc.Collector) Collector { 66 | return &passthroughCollector{ 67 | Collector: fc, 68 | } 69 | } 70 | 71 | func (c *passthroughCollector) Add(interface{}) error { return nil } 72 | func (c *passthroughCollector) AddEvent(in *Performance) error { 73 | if in == nil { 74 | return errors.New("cannot add nil performance event") 75 | } 76 | 77 | return c.Collector.Add(in) 78 | } 79 | 80 | type samplingCollector struct { 81 | ftdc.Collector 82 | current *Performance 83 | sample int 84 | count int 85 | } 86 | 87 | // NewSamplingCollector constructs a collector that has the same 88 | // semantics as the basic, adding all sampled documents together, but 89 | // only persisting every n-th sample to the underlying collector. 90 | func NewSamplingCollector(fc ftdc.Collector, n int) Collector { 91 | return &samplingCollector{ 92 | sample: n, 93 | Collector: fc, 94 | } 95 | } 96 | 97 | func (c *samplingCollector) Add(interface{}) error { return nil } 98 | func (c *samplingCollector) AddEvent(in *Performance) error { 99 | if in == nil { 100 | return errors.New("cannot add nil performance event") 101 | } 102 | 103 | if c.current == nil { 104 | c.current = in 105 | } else { 106 | c.current.Add(in) 107 | } 108 | 109 | shouldCollect := c.count%c.sample == 0 110 | c.count++ 111 | 112 | if shouldCollect { 113 | return c.Collector.Add(c.current) 114 | } 115 | 116 | return nil 117 | } 118 | 119 | type randSamplingCollector struct { 120 | percent int 121 | current *Performance 122 | ftdc.Collector 123 | } 124 | 125 | // NewRandomSamplingCollector constructs a Collector that uses a 126 | // psudorandom number generator (go's standard library math/rand) to 127 | // select how often to record an event. All events are summed. Specify 128 | // a percentage between 1 and 99 as the percent to reflect how many 129 | // events to capture. 130 | func NewRandomSamplingCollector(fc ftdc.Collector, sumAll bool, percent int) Collector { 131 | return &randSamplingCollector{ 132 | percent: percent, 133 | Collector: fc, 134 | } 135 | } 136 | 137 | func (c *randSamplingCollector) Add(interface{}) error { return nil } 138 | func (c *randSamplingCollector) AddEvent(in *Performance) error { 139 | if in == nil { 140 | return errors.New("cannot add nil performance event") 141 | } 142 | 143 | if c.current == nil { 144 | c.current = in 145 | } else { 146 | c.current.Add(in) 147 | } 148 | 149 | if c.shouldCollect() { 150 | return c.Collector.Add(c.current) 151 | } 152 | return nil 153 | } 154 | 155 | func (c *randSamplingCollector) shouldCollect() bool { 156 | if c.percent > 100 { 157 | return true 158 | } 159 | 160 | if c.percent <= 0 { 161 | return false 162 | } 163 | 164 | return rand.Intn(101) < c.percent 165 | } 166 | 167 | type intervalSamplingCollector struct { 168 | ftdc.Collector 169 | dur time.Duration 170 | current *Performance 171 | lastCollected time.Time 172 | } 173 | 174 | // NewIntervalCollector constructs a Collector collapses events as in 175 | // the other collector implementations, but will record at most a 176 | // single event per interval. 177 | func NewIntervalCollector(fc ftdc.Collector, interval time.Duration) Collector { 178 | return &intervalSamplingCollector{ 179 | Collector: fc, 180 | dur: interval, 181 | } 182 | } 183 | 184 | func (c *intervalSamplingCollector) Add(interface{}) error { return nil } 185 | func (c *intervalSamplingCollector) AddEvent(in *Performance) error { 186 | if in == nil { 187 | return errors.New("cannot add nil performance event") 188 | } 189 | 190 | if c.current == nil { 191 | c.current = in 192 | c.lastCollected = time.Now() 193 | return c.Collector.Add(c.current) 194 | } 195 | c.current.Add(in) 196 | if time.Since(c.lastCollected) >= c.dur { 197 | c.lastCollected = time.Now() 198 | return c.Collector.Add(c.current) 199 | } 200 | return nil 201 | } 202 | 203 | type synchronizedCollector struct { 204 | Collector 205 | mu sync.RWMutex 206 | } 207 | 208 | // NewSynchronizedCollector wraps another collector and wraps all 209 | // required calls with the correct lock. 210 | func NewSynchronizedCollector(coll Collector) Collector { 211 | return &synchronizedCollector{ 212 | Collector: coll, 213 | } 214 | } 215 | 216 | func (c *synchronizedCollector) Add(in interface{}) error { 217 | c.mu.Lock() 218 | defer c.mu.Unlock() 219 | return c.Collector.Add(in) 220 | } 221 | 222 | func (c *synchronizedCollector) AddEvent(in *Performance) error { 223 | c.mu.Lock() 224 | defer c.mu.Unlock() 225 | return c.Collector.AddEvent(in) 226 | } 227 | 228 | func (c *synchronizedCollector) SetMetadata(in interface{}) error { 229 | c.mu.Lock() 230 | defer c.mu.Unlock() 231 | 232 | return c.Collector.SetMetadata(in) 233 | } 234 | 235 | func (c *synchronizedCollector) Resolve() ([]byte, error) { 236 | c.mu.Lock() 237 | defer c.mu.Unlock() 238 | 239 | return c.Collector.Resolve() 240 | } 241 | 242 | func (c *synchronizedCollector) Reset() { 243 | c.mu.Lock() 244 | defer c.mu.Unlock() 245 | 246 | c.Collector.Reset() 247 | } 248 | 249 | func (c *synchronizedCollector) Info() ftdc.CollectorInfo { 250 | c.mu.RLock() 251 | defer c.mu.RUnlock() 252 | 253 | return c.Collector.Info() 254 | } 255 | -------------------------------------------------------------------------------- /events/collector_test.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/evergreen-ci/birch" 8 | "github.com/mongodb/ftdc" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestCollector(t *testing.T) { 13 | for _, fcTest := range []struct { 14 | name string 15 | constructor func() ftdc.Collector 16 | }{ 17 | { 18 | name: "Basic", 19 | constructor: func() ftdc.Collector { 20 | return ftdc.NewBaseCollector(100) 21 | }, 22 | }, 23 | { 24 | name: "Uncompressed", 25 | constructor: func() ftdc.Collector { 26 | return ftdc.NewUncompressedCollectorBSON(100) 27 | }, 28 | }, 29 | { 30 | name: "Dynamic", 31 | constructor: func() ftdc.Collector { 32 | return ftdc.NewDynamicCollector(100) 33 | }, 34 | }, 35 | } { 36 | t.Run(fcTest.name, func(t *testing.T) { 37 | for _, collectorTest := range []struct { 38 | name string 39 | constructor func(ftdc.Collector) Collector 40 | }{ 41 | { 42 | name: "Basic", 43 | constructor: func(fc ftdc.Collector) Collector { 44 | return NewBasicCollector(fc) 45 | }, 46 | }, 47 | { 48 | name: "Passthrough", 49 | constructor: func(fc ftdc.Collector) Collector { 50 | return NewPassthroughCollector(fc) 51 | }, 52 | }, 53 | { 54 | name: "SamplingAll", 55 | constructor: func(fc ftdc.Collector) Collector { 56 | return NewSamplingCollector(fc, 1) 57 | }, 58 | }, 59 | { 60 | name: "RandomSamplingAll", 61 | constructor: func(fc ftdc.Collector) Collector { 62 | return NewRandomSamplingCollector(fc, true, 1000) 63 | }, 64 | }, 65 | { 66 | name: "IntervalAll", 67 | constructor: func(fc ftdc.Collector) Collector { 68 | return NewIntervalCollector(fc, 0) 69 | }, 70 | }, 71 | } { 72 | t.Run(collectorTest.name, func(t *testing.T) { 73 | t.Run("Fixture", func(t *testing.T) { 74 | collector := collectorTest.constructor(fcTest.constructor()) 75 | assert.NotNil(t, collector) 76 | }) 77 | t.Run("AddMethod", func(t *testing.T) { 78 | collector := collectorTest.constructor(fcTest.constructor()) 79 | assert.NoError(t, collector.Add(nil)) 80 | assert.NoError(t, collector.Add(map[string]string{"foo": "bar"})) 81 | }) 82 | t.Run("AddEvent", func(t *testing.T) { 83 | collector := collectorTest.constructor(fcTest.constructor()) 84 | assert.Error(t, collector.AddEvent(nil)) 85 | assert.Equal(t, 0, collector.Info().SampleCount) 86 | 87 | for idx, e := range []*Performance{ 88 | {}, 89 | { 90 | Timestamp: time.Now(), 91 | ID: 12, 92 | }, 93 | } { 94 | assert.NoError(t, collector.AddEvent(e)) 95 | assert.Equal(t, idx+1, collector.Info().SampleCount) 96 | } 97 | }) 98 | }) 99 | } 100 | }) 101 | } 102 | } 103 | 104 | func TestFastMarshaling(t *testing.T) { 105 | assert.Implements(t, (*birch.DocumentMarshaler)(nil), &Performance{}) 106 | assert.Implements(t, (*birch.DocumentMarshaler)(nil), &PerformanceHDR{}) 107 | assert.Implements(t, (*birch.DocumentMarshaler)(nil), Custom{}) 108 | } 109 | -------------------------------------------------------------------------------- /events/custom.go: -------------------------------------------------------------------------------- 1 | // Package events contains a number of different data types and 2 | // formats that you can use to populate ftdc metrics series. 3 | // 4 | // # Custom and CustomPoint 5 | // 6 | // The "custom" types allow you to construct arbirary key-value pairs 7 | // without using maps and have them be well represented in FTDC 8 | // output. Populate and interact with the data sequence as a slice of 9 | // key (string) value (numbers) pairs, which are marshaled in the database 10 | // as an object as a mapping of strings to numbers. The type provides 11 | // some additional helpers for manipulating these data. 12 | package events 13 | 14 | import ( 15 | "sort" 16 | "time" 17 | 18 | "github.com/evergreen-ci/birch" 19 | "github.com/pkg/errors" 20 | ) 21 | 22 | // CustomPoint represents a computed statistic as a key value 23 | // pair. Use with the Custom type to ensure that the Value types refer 24 | // to number values and ensure consistent round trip semantics through BSON 25 | // and FTDC. 26 | type CustomPoint struct { 27 | Name string 28 | Value interface{} 29 | } 30 | 31 | // Custom is a collection of data points designed to store computed 32 | // statistics at an interval. In general you will add a set of 33 | // rolled-up data values to the custom object on an interval and then 34 | // pass that sequence to an ftdc.Collector. Custom implements 35 | // sort.Interface, and the CustomPoint type implements custom bson 36 | // marshalling so that Points are marshaled as an object to facilitate 37 | // their use with the ftdc format. 38 | type Custom []CustomPoint 39 | 40 | // MakeCustom creates a Custom slice with the specified size hint. 41 | func MakeCustom(size int) Custom { return make(Custom, 0, size) } 42 | 43 | // Add appends a key to the Custom metric. Only accepts go native 44 | // number types and timestamps. 45 | func (ps *Custom) Add(key string, value interface{}) error { 46 | // TODO: figure out 47 | switch v := value.(type) { 48 | case int64, int32, int, bool, time.Time, float64, float32, uint32, uint64: 49 | *ps = append(*ps, CustomPoint{Name: key, Value: v}) 50 | return nil 51 | case []int64, []int32, []int, []bool, []time.Time, []float64, []float32, []uint32, []uint64: 52 | *ps = append(*ps, CustomPoint{Name: key, Value: v}) 53 | return nil 54 | default: 55 | return errors.Errorf("type '%T' for key %s is not supported", value, key) 56 | } 57 | } 58 | 59 | // Len is a component of the sort.Interface. 60 | func (ps Custom) Len() int { return len(ps) } 61 | 62 | // Less is a component of the sort.Interface. 63 | func (ps Custom) Less(i, j int) bool { return ps[i].Name < ps[j].Name } 64 | 65 | // Swap is a component of the sort.Interface. 66 | func (ps Custom) Swap(i, j int) { ps[i], ps[j] = ps[j], ps[i] } 67 | 68 | // Sort is a convenience function around a stable sort for the custom 69 | // array. 70 | func (ps Custom) Sort() { sort.Stable(ps) } 71 | 72 | func (ps Custom) MarshalBSON() ([]byte, error) { return birch.MarshalDocumentBSON(ps) } 73 | 74 | func (ps *Custom) UnmarshalBSON(in []byte) error { 75 | doc, err := birch.ReadDocument(in) 76 | if err != nil { 77 | return errors.Wrap(err, "problem parsing bson document") 78 | } 79 | 80 | iter := doc.Iterator() 81 | for iter.Next() { 82 | elem := iter.Element() 83 | *ps = append(*ps, CustomPoint{ 84 | Name: elem.Key(), 85 | Value: elem.Value().Interface(), 86 | }) 87 | } 88 | 89 | if err = iter.Err(); err != nil { 90 | return errors.Wrap(err, "problem reading document") 91 | } 92 | 93 | return nil 94 | } 95 | 96 | func (ps Custom) MarshalDocument() (*birch.Document, error) { 97 | ps.Sort() 98 | 99 | doc := birch.DC.Make(ps.Len()) 100 | 101 | for _, elem := range ps { 102 | de, err := birch.EC.InterfaceErr(elem.Name, elem.Value) 103 | if err != nil { 104 | return nil, errors.WithStack(err) 105 | } 106 | doc.Append(de) 107 | } 108 | 109 | return doc, nil 110 | } 111 | -------------------------------------------------------------------------------- /events/custom_test.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "go.mongodb.org/mongo-driver/v2/bson" 9 | ) 10 | 11 | func TestRollupRoundTrip(t *testing.T) { 12 | data := MakeCustom(4) 13 | assert.NoError(t, data.Add("a", 1.2)) 14 | assert.NoError(t, data.Add("f", 100)) 15 | assert.NoError(t, data.Add("b", 45.0)) 16 | assert.NoError(t, data.Add("d", []int64{45, 32})) 17 | assert.Error(t, data.Add("foo", Custom{})) 18 | assert.Len(t, data, 4) 19 | 20 | t.Run("NewBSON", func(t *testing.T) { 21 | payload, err := bson.Marshal(data) 22 | require.NoError(t, err) 23 | 24 | rt := Custom{} 25 | err = bson.Unmarshal(payload, &rt) 26 | require.NoError(t, err) 27 | 28 | require.Len(t, rt, 4) 29 | assert.Equal(t, "a", rt[0].Name) 30 | assert.Equal(t, "b", rt[1].Name) 31 | assert.Equal(t, "d", rt[2].Name) 32 | assert.Equal(t, "f", rt[3].Name) 33 | assert.Equal(t, 1.2, rt[0].Value) 34 | assert.Equal(t, 45.0, rt[1].Value) 35 | assert.EqualValues(t, 100, rt[3].Value) 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /events/events_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math/rand" 7 | "testing" 8 | "time" 9 | 10 | "github.com/mongodb/ftdc" 11 | ) 12 | 13 | func BenchmarkEventCollection(b *testing.B) { 14 | for _, collect := range []struct { 15 | Name string 16 | Factory func() ftdc.Collector 17 | }{ 18 | { 19 | Name: "Base", 20 | Factory: func() ftdc.Collector { return ftdc.NewBaseCollector(1000) }, 21 | }, 22 | { 23 | Name: "Dynamic", 24 | Factory: func() ftdc.Collector { return ftdc.NewDynamicCollector(1000) }, 25 | }, 26 | { 27 | Name: "Streaming", 28 | Factory: func() ftdc.Collector { return ftdc.NewStreamingCollector(1000, &bytes.Buffer{}) }, 29 | }, 30 | { 31 | Name: "StreamingDynamic", 32 | Factory: func() ftdc.Collector { return ftdc.NewStreamingDynamicCollector(1000, &bytes.Buffer{}) }, 33 | }, 34 | } { 35 | 36 | b.Run(collect.Name, func(b *testing.B) { 37 | for _, test := range []struct { 38 | Name string 39 | Generator func() interface{} 40 | }{ 41 | { 42 | Name: "Performance", 43 | Generator: func() interface{} { 44 | return &Performance{ 45 | Timestamp: time.Now(), 46 | Counters: PerformanceCounters{ 47 | Number: rand.Int63n(128), 48 | Operations: rand.Int63n(512), 49 | Size: rand.Int63n(1024), 50 | Errors: rand.Int63n(64), 51 | }, 52 | Timers: PerformanceTimers{ 53 | Duration: time.Duration(rand.Int63n(int64(time.Minute))), 54 | Total: time.Duration(rand.Int63n(int64(2 * time.Minute))), 55 | }, 56 | Gauges: PerformanceGauges{ 57 | State: rand.Int63n(2), 58 | Workers: rand.Int63n(4), 59 | }, 60 | } 61 | }, 62 | }, 63 | { 64 | Name: "HistogramSecondZeroed", 65 | Generator: func() interface{} { 66 | return NewHistogramSecond(PerformanceGauges{}) 67 | }, 68 | }, 69 | { 70 | Name: "HistogramMillisecondZeroed", 71 | Generator: func() interface{} { 72 | return NewHistogramMillisecond(PerformanceGauges{}) 73 | }, 74 | }, 75 | { 76 | Name: "CustomMidsized", 77 | Generator: func() interface{} { 78 | point := MakeCustom(20) 79 | for i := int64(1); i <= 10; i++ { 80 | point.Add(fmt.Sprintln("stat", i), rand.Int63n(i)) // nolint 81 | point.Add(fmt.Sprintln("stat", i), float64(i)+rand.Float64()) // nolint 82 | } 83 | return point 84 | }, 85 | }, 86 | { 87 | Name: "CustomSmall", 88 | Generator: func() interface{} { 89 | point := MakeCustom(4) 90 | for i := int64(1); i <= 2; i++ { 91 | point.Add(fmt.Sprintln("stat", i), rand.Int63n(i)) // nolint 92 | point.Add(fmt.Sprintln("stat", i), float64(i)+rand.Float64()) // nolint 93 | } 94 | return point 95 | }, 96 | }, 97 | } { 98 | collector := collect.Factory() 99 | b.Run(test.Name, func(b *testing.B) { 100 | b.Run("Add", func(b *testing.B) { 101 | for n := 0; n < b.N; n++ { 102 | collector.Add(test.Generator()) // nolint 103 | } 104 | }) 105 | b.Run("Resolve", func(b *testing.B) { 106 | for n := 0; n < b.N; n++ { 107 | collector.Resolve() // nolint 108 | } 109 | }) 110 | }) 111 | } 112 | }) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /events/histogram.go: -------------------------------------------------------------------------------- 1 | // Histogram 2 | // 3 | // The histogram representation is broadly similar to the Performance structure 4 | // but stores data in a histogram format, which offers a high fidelity 5 | // representation of a very large number of raw events without the storage 6 | // overhead. In general, use histograms to collect data for operation with 7 | // throughput in the thousands or more operations per second. 8 | package events 9 | 10 | import ( 11 | "time" 12 | 13 | "github.com/evergreen-ci/birch" 14 | "github.com/mongodb/ftdc/hdrhist" 15 | ) 16 | 17 | // PerformanceHDR the same as the Performance structure, but with all time 18 | // duration and counter values stored as histograms. 19 | type PerformanceHDR struct { 20 | Timestamp time.Time `bson:"ts" json:"ts" yaml:"ts"` 21 | ID int64 `bson:"id" json:"id" yaml:"id"` 22 | Counters PerformanceCountersHDR `bson:"counters" json:"counters" yaml:"counters"` 23 | Timers PerformanceTimersHDR `bson:"timers" json:"timers" yaml:"timers"` 24 | Gauges PerformanceGauges `bson:"guages" json:"guages" yaml:"guages"` 25 | } 26 | 27 | type PerformanceCountersHDR struct { 28 | Number *hdrhist.Histogram `bson:"n" json:"n" yaml:"n"` 29 | Operations *hdrhist.Histogram `bson:"ops" json:"ops" yaml:"ops"` 30 | Size *hdrhist.Histogram `bson:"size" json:"size" yaml:"size"` 31 | Errors *hdrhist.Histogram `bson:"errors" json:"errors" yaml:"errors"` 32 | } 33 | 34 | type PerformanceTimersHDR struct { 35 | Duration *hdrhist.Histogram `bson:"dur" json:"dur" yaml:"dur"` 36 | Total *hdrhist.Histogram `bson:"total" json:"total" yaml:"total"` 37 | } 38 | 39 | func NewHistogramSecond(g PerformanceGauges) *PerformanceHDR { 40 | return &PerformanceHDR{ 41 | Gauges: g, 42 | Counters: PerformanceCountersHDR{ 43 | Number: newSecondCounterHistogram(), 44 | Operations: newSecondCounterHistogram(), 45 | Size: newSecondCounterHistogram(), 46 | Errors: newSecondCounterHistogram(), 47 | }, 48 | Timers: PerformanceTimersHDR{ 49 | Duration: newSecondDurationHistogram(), 50 | Total: newSecondDurationHistogram(), 51 | }, 52 | } 53 | } 54 | 55 | func newSecondDurationHistogram() *hdrhist.Histogram { 56 | return hdrhist.New(int64(time.Microsecond), int64(20*time.Minute), 5) 57 | } 58 | 59 | func newSecondCounterHistogram() *hdrhist.Histogram { 60 | return hdrhist.New(0, 10*100*1000, 5) 61 | } 62 | 63 | func NewHistogramMillisecond(g PerformanceGauges) *PerformanceHDR { 64 | return &PerformanceHDR{ 65 | Gauges: g, 66 | Counters: PerformanceCountersHDR{ 67 | Number: newMillisecondCounterHistogram(), 68 | Operations: newMillisecondCounterHistogram(), 69 | Size: newMillisecondCounterHistogram(), 70 | Errors: newMillisecondCounterHistogram(), 71 | }, 72 | Timers: PerformanceTimersHDR{ 73 | Duration: newMillisecondDurationHistogram(), 74 | Total: newMillisecondDurationHistogram(), 75 | }, 76 | } 77 | } 78 | 79 | func newMillisecondDurationHistogram() *hdrhist.Histogram { 80 | return hdrhist.New(int64(time.Microsecond), int64(time.Minute), 5) 81 | } 82 | 83 | func newMillisecondCounterHistogram() *hdrhist.Histogram { 84 | return hdrhist.New(0, 10*1000, 5) 85 | } 86 | 87 | func (p *PerformanceHDR) MarshalDocument() (*birch.Document, error) { 88 | return birch.DC.Elements( 89 | birch.EC.Time("ts", p.Timestamp), 90 | birch.EC.Int64("id", p.ID), 91 | birch.EC.SubDocumentFromElements("counters", 92 | birch.EC.DocumentMarshaler("n", p.Counters.Number), 93 | birch.EC.DocumentMarshaler("ops", p.Counters.Operations), 94 | birch.EC.DocumentMarshaler("size", p.Counters.Size), 95 | birch.EC.DocumentMarshaler("errors", p.Counters.Errors), 96 | ), 97 | birch.EC.SubDocumentFromElements("timers", 98 | birch.EC.DocumentMarshaler("dur", p.Timers.Duration), 99 | birch.EC.DocumentMarshaler("total", p.Timers.Total), 100 | ), 101 | birch.EC.SubDocumentFromElements("gauges", 102 | birch.EC.Int64("state", p.Gauges.State), 103 | birch.EC.Int64("workers", p.Gauges.Workers), 104 | birch.EC.Boolean("failed", p.Gauges.Failed), 105 | ), 106 | ), nil 107 | } 108 | 109 | func (p *PerformanceHDR) setTimestamp(started time.Time) { 110 | if p.Timestamp.IsZero() { 111 | if !started.IsZero() { 112 | p.Timestamp = started 113 | } else { 114 | p.Timestamp = time.Now() 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /events/performance.go: -------------------------------------------------------------------------------- 1 | // Performance Points 2 | // 3 | // The Performance type represents a unified event to track an operation in a 4 | // performance test. These events record three types of metrics: counters, 5 | // timers, and gauges. Counters record the number of operations in different 6 | // ways, including test iterations, logical operation counts, operation size 7 | // (bytes), and error rate. Timers include both the latency of the core 8 | // operation, for use in calculating latencies as well as the total taken which 9 | // may be useful in calculating throughput. Finally gauges, capture changes in 10 | // state or other information about the environment including the number of 11 | // threads used in the test, or a failed Boolean when a test is aware of its 12 | // own failure. 13 | package events 14 | 15 | import ( 16 | "time" 17 | 18 | "github.com/evergreen-ci/birch" 19 | "github.com/pkg/errors" 20 | ) 21 | 22 | // Performance represents a single raw event in a metrics collection system for 23 | // performance metric collection system. 24 | // 25 | // Each point must report the timestamp of its collection. 26 | type Performance struct { 27 | Timestamp time.Time `bson:"ts" json:"ts" yaml:"ts"` 28 | ID int64 `bson:"id" json:"id" yaml:"id"` 29 | Counters PerformanceCounters `bson:"counters" json:"counters" yaml:"counters"` 30 | Timers PerformanceTimers `bson:"timers" json:"timers" yaml:"timers"` 31 | Gauges PerformanceGauges `bson:"gauges" json:"gauges" yaml:"gauges"` 32 | } 33 | 34 | // PerformanceCounters refer to the number of operations/events or total of 35 | // things since the last collection point. These values are used in computing 36 | // various kinds of throughput measurements. 37 | type PerformanceCounters struct { 38 | Number int64 `bson:"n" json:"n" yaml:"n"` 39 | Operations int64 `bson:"ops" json:"ops" yaml:"ops"` 40 | Size int64 `bson:"size" json:"size" yaml:"size"` 41 | Errors int64 `bson:"errors" json:"errors" yaml:"errors"` 42 | } 43 | 44 | // PerformanceTimers refers to all of the timing data for this event. In 45 | // general Total should equal the time since the last data point. 46 | type PerformanceTimers struct { 47 | Duration time.Duration `bson:"dur" json:"dur" yaml:"dur"` 48 | Total time.Duration `bson:"total" json:"total" yaml:"total"` 49 | } 50 | 51 | // PerformanceGauges holds simple counters that aren't expected to change 52 | // between points, but are useful as annotations of the experiment or 53 | // or descriptions of events in the system configuration. 54 | type PerformanceGauges struct { 55 | State int64 `bson:"state" json:"state" yaml:"state"` 56 | Workers int64 `bson:"workers" json:"workers" yaml:"workers"` 57 | Failed bool `bson:"failed" json:"failed" yaml:"failed"` 58 | } 59 | 60 | // MarshalBSON implements the bson marshaler interface to support 61 | // converting this type into BSON without relying on a 62 | // reflection-based BSON library. 63 | func (p *Performance) MarshalBSON() ([]byte, error) { return birch.MarshalDocumentBSON(p) } 64 | 65 | // MarshalDocument exports the Performance type as a birch.Document to 66 | // support more efficient operations. 67 | func (p *Performance) MarshalDocument() (*birch.Document, error) { 68 | return birch.DC.Elements( 69 | birch.EC.Time("ts", p.Timestamp), 70 | birch.EC.Int64("id", p.ID), 71 | birch.EC.SubDocument("counters", birch.DC.Elements( 72 | birch.EC.Int64("n", p.Counters.Number), 73 | birch.EC.Int64("ops", p.Counters.Operations), 74 | birch.EC.Int64("size", p.Counters.Size), 75 | birch.EC.Int64("errors", p.Counters.Errors), 76 | )), 77 | birch.EC.SubDocument("timers", birch.DC.Elements( 78 | birch.EC.Duration("dur", p.Timers.Duration), 79 | birch.EC.Duration("total", p.Timers.Total), 80 | )), 81 | birch.EC.SubDocument("gauges", birch.DC.Elements( 82 | birch.EC.Int64("state", p.Gauges.State), 83 | birch.EC.Int64("workers", p.Gauges.Workers), 84 | birch.EC.Boolean("failed", p.Gauges.Failed), 85 | )), 86 | ), nil 87 | } 88 | 89 | func (p *Performance) UnmarshalDocument(doc *birch.Document) error { 90 | iter := doc.Iterator() 91 | for iter.Next() { 92 | elem := iter.Element() 93 | switch elem.Key() { 94 | case "ts": 95 | p.Timestamp = elem.Value().Time() 96 | case "id": 97 | case "counters": 98 | if err := p.Counters.UnmarshalDocument(elem.Value().MutableDocument()); err != nil { 99 | return errors.WithStack(err) 100 | } 101 | case "timers": 102 | if err := p.Timers.UnmarshalDocument(elem.Value().MutableDocument()); err != nil { 103 | return errors.WithStack(err) 104 | } 105 | case "gauges": 106 | if err := p.Gauges.UnmarshalDocument(elem.Value().MutableDocument()); err != nil { 107 | return errors.WithStack(err) 108 | } 109 | } 110 | } 111 | 112 | return errors.WithStack(iter.Err()) 113 | } 114 | 115 | func (p *PerformanceCounters) UnmarshalDocument(doc *birch.Document) error { 116 | iter := doc.Iterator() 117 | for iter.Next() { 118 | elem := iter.Element() 119 | switch elem.Key() { 120 | case "n": 121 | p.Number = elem.Value().Int64() 122 | case "opts": 123 | p.Operations = elem.Value().Int64() 124 | case "size": 125 | p.Size = elem.Value().Int64() 126 | case "errors": 127 | p.Errors = elem.Value().Int64() 128 | } 129 | } 130 | 131 | return errors.WithStack(iter.Err()) 132 | } 133 | 134 | func (p *PerformanceTimers) UnmarshalDocument(doc *birch.Document) error { 135 | iter := doc.Iterator() 136 | for iter.Next() { 137 | elem := iter.Element() 138 | switch elem.Key() { 139 | case "dur": 140 | p.Duration = time.Duration(elem.Value().Int64()) 141 | case "total": 142 | p.Total = time.Duration(elem.Value().Int64()) 143 | } 144 | } 145 | 146 | return errors.WithStack(iter.Err()) 147 | } 148 | 149 | func (p *PerformanceGauges) UnmarshalDocument(doc *birch.Document) error { 150 | iter := doc.Iterator() 151 | for iter.Next() { 152 | elem := iter.Element() 153 | switch elem.Key() { 154 | case "state": 155 | p.State = elem.Value().Int64() 156 | case "workers": 157 | p.Workers = elem.Value().Int64() 158 | case "failed": 159 | p.Failed = elem.Value().Boolean() 160 | } 161 | } 162 | 163 | return errors.WithStack(iter.Err()) 164 | } 165 | 166 | // Add combines the values of the input Performance struct into this struct, 167 | // logically, overriding the Gauges values as well as the timestamp and ID 168 | // ID values, while summing the Counters and Timers values. 169 | func (p *Performance) Add(in *Performance) { 170 | if in.ID == 0 { 171 | in.ID = p.ID + 1 172 | } 173 | 174 | p.Timestamp = in.Timestamp 175 | p.ID = in.ID 176 | p.Counters.Number += in.Counters.Number 177 | p.Counters.Errors += in.Counters.Errors 178 | p.Counters.Operations += in.Counters.Operations 179 | p.Counters.Size += in.Counters.Size 180 | 181 | p.Timers.Duration += in.Timers.Duration 182 | p.Timers.Total += in.Timers.Total 183 | 184 | p.Gauges.Failed = in.Gauges.Failed 185 | p.Gauges.Workers = in.Gauges.Workers 186 | p.Gauges.State = in.Gauges.State 187 | } 188 | 189 | func (p *Performance) setTimestamp(started time.Time) { 190 | if p.Timestamp.IsZero() { 191 | if !started.IsZero() { 192 | p.Timestamp = started 193 | } else { 194 | p.Timestamp = time.Now() 195 | } 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /events/performance_test.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestPerformanceType(t *testing.T) { 11 | t.Run("MethodsPanicWhenNil", func(t *testing.T) { 12 | var perf *Performance 13 | assert.Nil(t, perf) 14 | assert.Panics(t, func() { 15 | _, err := perf.MarshalDocument() 16 | require.Error(t, err) 17 | }) 18 | assert.Panics(t, func() { 19 | _, err := perf.MarshalBSON() 20 | assert.Error(t, err) 21 | }) 22 | assert.Panics(t, func() { 23 | perf.Add(nil) 24 | }) 25 | assert.Panics(t, func() { 26 | perf = &Performance{} 27 | perf.Add(nil) 28 | }) 29 | }) 30 | t.Run("Document", func(t *testing.T) { 31 | perf := &Performance{} 32 | doc, err := perf.MarshalDocument() 33 | require.NoError(t, err) 34 | require.NotNil(t, doc) 35 | assert.Equal(t, 5, doc.Len()) 36 | }) 37 | t.Run("BSON", func(t *testing.T) { 38 | perf := &Performance{} 39 | out, err := perf.MarshalBSON() 40 | assert.NoError(t, err) 41 | assert.NotNil(t, out) 42 | }) 43 | t.Run("Add", func(t *testing.T) { 44 | t.Run("Zero", func(t *testing.T) { 45 | perf := &Performance{} 46 | perf.Add(&Performance{}) 47 | assert.Equal(t, &Performance{ID: 1}, perf) 48 | }) 49 | t.Run("OverridesID", func(t *testing.T) { 50 | perf := &Performance{} 51 | perf.Add(&Performance{ID: 100}) 52 | assert.EqualValues(t, 100, perf.ID) 53 | perf.Add(&Performance{ID: 100}) 54 | assert.EqualValues(t, 100, perf.ID) 55 | }) 56 | t.Run("Counter", func(t *testing.T) { 57 | perf := &Performance{} 58 | perf.Add(&Performance{Counters: PerformanceCounters{Number: 100}}) 59 | assert.EqualValues(t, 100, perf.Counters.Number) 60 | perf.Add(&Performance{Counters: PerformanceCounters{Number: 100}}) 61 | assert.EqualValues(t, 200, perf.Counters.Number) 62 | }) 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /events/recorder.go: -------------------------------------------------------------------------------- 1 | // Recorder 2 | // 3 | // The Recorder interface provides an interface for workloads and operations to 4 | // to collect data about their internal state, without requiring workloads to 5 | // be concerned with data retenation, storage, or compression. The 6 | // implementations of Recorder have different strategies for data collection 7 | // and persistence so that tests can easily change data collection strategies 8 | // without modifying the test. 9 | package events 10 | 11 | import "time" 12 | 13 | // Recorder describes an interface that tests can use to track metrics and 14 | // events during performance testing or normal operation. Implementations of 15 | // recorder wrap an FTDC collector and will write data out to the collector for 16 | // reporting purposes. The types produced by the collector use the Performance 17 | // or PerformanceHDR types in this package. 18 | // 19 | // Choose the implementation of Recorder that will capture all required data 20 | // used by your test with sufficient resolution for use later. Additionally, 21 | // consider the data volume produced by the recorder. 22 | type Recorder interface { 23 | // The Inc<> operations add values to the specified counters tracked by 24 | // the collector. There is an additional "iteration" counter that the 25 | // recorder tracks based on the number of times that 26 | // BeginIteration/EndIteration are called, but is also accessible via 27 | // the IncIteration counter. 28 | // 29 | // In general, IncOperations should refer to the number of logical 30 | // operations collected. This differs from the iteration count, in the 31 | // case of workloads that comprise of multiple logical operations. 32 | // 33 | // Use IncSize to record, typically, the number of bytes processed or 34 | // generated by the operation. Use this in combination with logical 35 | // operations to be able to explore the impact of data size on overall 36 | // performance. Finally use IncError to track the number of errors 37 | // encountered during the event. 38 | IncIterations(int64) 39 | IncOperations(int64) 40 | IncError(int64) 41 | IncSize(int64) 42 | 43 | // The Set<> operations replace existing values for the state, workers, 44 | // and failed gauges. Workers should typically report the number of 45 | // active threads. The meaning of state depends on the test 46 | // but can describe phases of an experiment or operation. Use SetFailed 47 | // to flag a test as failed during the operation. 48 | SetWorkers(int64) 49 | SetState(int64) 50 | SetFailed(bool) 51 | 52 | // The BeginIteration and EndIteration methods mark the beginning and 53 | // end of a test's iteration. Typically calling EndIteration records 54 | // the duration specified as its argument and increments the counter 55 | // for number of iterations. Additionally there is a "total duration" 56 | // value captured which represents the total time taken in the 57 | // iteration in addition to the operation latency. 58 | // 59 | // The EndTest method writes any unpersisted material if the 60 | // collector's EndTest method has not. In all cases EndTest reports all 61 | // errors since the last EndTest call, and resets the internal error 62 | // the internal error tracking and unsets the tracked starting time. 63 | // Generally you should call EndTest once at the end of every test run, 64 | // and fail if there are errors reported. Reset does the same as 65 | // EndTest, except for persisting data and returing errors. 66 | BeginIteration() 67 | EndIteration(time.Duration) 68 | EndTest() error 69 | Reset() 70 | 71 | // SetID sets the unique id for the event, to allow users to identify 72 | // events per thread. 73 | SetID(int64) 74 | 75 | // SetTime defines the timestamp of the current point. SetTime is 76 | // usually not needed: BeginIteration will set the time to the current 77 | // time; however, if you're using a recorder as part of 78 | // post-processing, you will want to use SetTime directly. 79 | SetTime(time.Time) 80 | 81 | // SetTotalDuration allows you to set the total time covered by the 82 | // event in question. The total time is usually derived by the 83 | // difference between the time set in BeginIteration and the time when 84 | // EndTest is called. Typically the duration passed to EndTest refers 85 | // to a subset of this time (i.e. the amount of time that the 86 | // operations in question took), and the total time, includes some 87 | // period of overhead. 88 | // 89 | // In simplest terms, this should typically be the time since the last 90 | // event was recorded. 91 | SetTotalDuration(time.Duration) 92 | 93 | // SetDuration allows you to define the duration of a the operation, 94 | // this is likely a subset of the total duration, with the difference 95 | // between the duration and the total duration, representing some kind 96 | // of operational overhead. 97 | SetDuration(time.Duration) 98 | } 99 | -------------------------------------------------------------------------------- /events/recorder_histogram.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mongodb/ftdc" 7 | "github.com/mongodb/ftdc/util" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type histogramStream struct { 12 | point *PerformanceHDR 13 | started time.Time 14 | collector ftdc.Collector 15 | catcher util.Catcher 16 | } 17 | 18 | // NewHistogramRecorder collects data and stores them with a histogram format. 19 | // Like the Raw recorder, the system saves each data point after each call to 20 | // EndIteration. 21 | // 22 | // The timer histgrams have a minimum value of 1 microsecond, and a maximum 23 | // value of 20 minutes, with 5 significant digits. The counter histograms store 24 | // between 0 and 1 million, with 5 significant digits. The gauges are stored as 25 | // integers. 26 | // 27 | // The histogram reporter is not safe for concurrent use without a synchronized 28 | // wrapper. 29 | func NewHistogramRecorder(collector ftdc.Collector) Recorder { 30 | return &histogramStream{ 31 | point: NewHistogramMillisecond(PerformanceGauges{}), 32 | collector: collector, 33 | catcher: util.NewCatcher(), 34 | } 35 | } 36 | 37 | func (r *histogramStream) SetID(id int64) { r.point.ID = id } 38 | func (r *histogramStream) SetState(val int64) { r.point.Gauges.State = val } 39 | func (r *histogramStream) SetWorkers(val int64) { r.point.Gauges.Workers = val } 40 | func (r *histogramStream) SetFailed(val bool) { r.point.Gauges.Failed = val } 41 | func (r *histogramStream) IncOperations(val int64) { 42 | r.catcher.Add(r.point.Counters.Operations.RecordValue(val)) 43 | } 44 | func (r *histogramStream) IncSize(val int64) { 45 | r.catcher.Add(r.point.Counters.Size.RecordValue(val)) 46 | } 47 | func (r *histogramStream) IncError(val int64) { 48 | r.catcher.Add(r.point.Counters.Errors.RecordValue(val)) 49 | } 50 | func (r *histogramStream) EndIteration(dur time.Duration) { 51 | r.point.setTimestamp(r.started) 52 | r.catcher.Add(r.point.Counters.Number.RecordValue(1)) 53 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 54 | if !r.started.IsZero() { 55 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(time.Since(r.started)))) 56 | r.started = time.Time{} 57 | } 58 | 59 | r.catcher.Add(r.collector.Add(r.point)) 60 | } 61 | 62 | func (r *histogramStream) SetDuration(dur time.Duration) { 63 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 64 | } 65 | 66 | func (r *histogramStream) SetTotalDuration(dur time.Duration) { 67 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(dur))) 68 | } 69 | 70 | func (r *histogramStream) IncIterations(val int64) { 71 | r.catcher.Add(r.point.Counters.Number.RecordValue(val)) 72 | } 73 | 74 | func (r *histogramStream) SetTime(t time.Time) { r.point.Timestamp = t } 75 | func (r *histogramStream) BeginIteration() { r.started = time.Now(); r.point.setTimestamp(r.started) } 76 | 77 | func (r *histogramStream) EndTest() error { 78 | if !r.point.Timestamp.IsZero() { 79 | r.catcher.Add(r.collector.Add(r.point)) 80 | } 81 | err := r.catcher.Resolve() 82 | r.Reset() 83 | return errors.WithStack(err) 84 | } 85 | 86 | func (r *histogramStream) Reset() { 87 | r.catcher = util.NewCatcher() 88 | r.point = NewHistogramMillisecond(r.point.Gauges) 89 | r.started = time.Time{} 90 | } 91 | -------------------------------------------------------------------------------- /events/recorder_histogram_grouped.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mongodb/ftdc" 7 | "github.com/mongodb/ftdc/util" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type histogramGroupedStream struct { 12 | point *PerformanceHDR 13 | lastCollected time.Time 14 | started time.Time 15 | interval time.Duration 16 | collector ftdc.Collector 17 | catcher util.Catcher 18 | } 19 | 20 | // NewHistogramGroupedRecorder captures data and stores them with a 21 | // histogram format. Like the Grouped recorder, it persists an event if the 22 | // specified interval has elapsed since the last time an event was captured. 23 | // The reset method also resets the last-collected time. 24 | // 25 | // The timer histgrams have a minimum value of 1 microsecond, and a maximum 26 | // value of 20 minutes, with 5 significant digits. The counter histograms store 27 | // between 0 and 1 million, with 5 significant digits. The gauges are stored as 28 | // integers. 29 | // 30 | // The histogram Grouped reporter is not safe for concurrent use without a 31 | // synchronixed wrapper. 32 | func NewHistogramGroupedRecorder(collector ftdc.Collector, interval time.Duration) Recorder { 33 | return &histogramGroupedStream{ 34 | point: NewHistogramMillisecond(PerformanceGauges{}), 35 | collector: collector, 36 | catcher: util.NewCatcher(), 37 | } 38 | } 39 | 40 | func (r *histogramGroupedStream) SetID(id int64) { r.point.ID = id } 41 | func (r *histogramGroupedStream) SetState(val int64) { r.point.Gauges.State = val } 42 | func (r *histogramGroupedStream) SetWorkers(val int64) { r.point.Gauges.Workers = val } 43 | func (r *histogramGroupedStream) SetFailed(val bool) { r.point.Gauges.Failed = val } 44 | func (r *histogramGroupedStream) IncOperations(val int64) { 45 | r.catcher.Add(r.point.Counters.Operations.RecordValue(val)) 46 | } 47 | func (r *histogramGroupedStream) IncSize(val int64) { 48 | r.catcher.Add(r.point.Counters.Size.RecordValue(val)) 49 | } 50 | func (r *histogramGroupedStream) IncError(val int64) { 51 | r.catcher.Add(r.point.Counters.Errors.RecordValue(val)) 52 | } 53 | func (r *histogramGroupedStream) EndIteration(dur time.Duration) { 54 | r.point.setTimestamp(r.started) 55 | r.catcher.Add(r.point.Counters.Number.RecordValue(1)) 56 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 57 | 58 | if !r.started.IsZero() { 59 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(time.Since(r.started)))) 60 | r.started = time.Time{} 61 | } 62 | 63 | if time.Since(r.lastCollected) >= r.interval { 64 | r.catcher.Add(r.collector.Add(r.point)) 65 | r.lastCollected = time.Now() 66 | } 67 | } 68 | 69 | func (r *histogramGroupedStream) SetTotalDuration(dur time.Duration) { 70 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(dur))) 71 | } 72 | 73 | func (r *histogramGroupedStream) SetDuration(dur time.Duration) { 74 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 75 | } 76 | 77 | func (r *histogramGroupedStream) IncIterations(val int64) { 78 | r.catcher.Add(r.point.Counters.Number.RecordValue(val)) 79 | } 80 | 81 | func (r *histogramGroupedStream) SetTime(t time.Time) { r.point.Timestamp = t } 82 | func (r *histogramGroupedStream) BeginIteration() { 83 | r.started = time.Now() 84 | r.point.setTimestamp(r.started) 85 | } 86 | 87 | func (r *histogramGroupedStream) EndTest() error { 88 | if !r.point.Timestamp.IsZero() { 89 | r.catcher.Add(r.collector.Add(r.point)) 90 | } 91 | err := r.catcher.Resolve() 92 | r.Reset() 93 | return errors.WithStack(err) 94 | } 95 | 96 | func (r *histogramGroupedStream) Reset() { 97 | r.catcher = util.NewCatcher() 98 | r.point = NewHistogramMillisecond(r.point.Gauges) 99 | r.lastCollected = time.Time{} 100 | r.started = time.Time{} 101 | } 102 | -------------------------------------------------------------------------------- /events/recorder_histogram_interval.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/mongodb/ftdc" 9 | "github.com/mongodb/ftdc/util" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | type intervalHistogramStream struct { 14 | point *PerformanceHDR 15 | started time.Time 16 | collector ftdc.Collector 17 | catcher util.Catcher 18 | sync.Mutex 19 | 20 | interval time.Duration 21 | rootCtx context.Context 22 | canceler context.CancelFunc 23 | } 24 | 25 | // NewIntervalHistogramRecorder has similar semantics to histogram Grouped 26 | // recorder, but has a background process that persists data on the specified 27 | // on the specified interval rather than as a side effect of the EndTest call. 28 | // 29 | // The background thread is started if it doesn't exist in the BeginIteration 30 | // operation and is terminated by the EndTest operation. 31 | // 32 | // The interval histogram recorder is safe for concurrent use. 33 | func NewIntervalHistogramRecorder(ctx context.Context, collector ftdc.Collector, interval time.Duration) Recorder { 34 | return &intervalHistogramStream{ 35 | collector: collector, 36 | rootCtx: ctx, 37 | catcher: util.NewCatcher(), 38 | interval: interval, 39 | point: NewHistogramMillisecond(PerformanceGauges{}), 40 | } 41 | } 42 | 43 | func (r *intervalHistogramStream) worker(ctx context.Context, interval time.Duration) { 44 | ticker := time.NewTicker(r.interval) 45 | defer ticker.Stop() 46 | 47 | for { 48 | select { 49 | case <-ctx.Done(): 50 | return 51 | case <-ticker.C: 52 | r.Lock() 53 | // check context error in case in between the time when 54 | // the lock is requested and when the lock is obtained, 55 | // the context has been canceled 56 | if ctx.Err() != nil { 57 | return 58 | } 59 | r.point.setTimestamp(r.started) 60 | r.catcher.Add(r.collector.Add(r.point)) 61 | r.Unlock() 62 | } 63 | } 64 | } 65 | 66 | func (r *intervalHistogramStream) BeginIteration() { 67 | r.Lock() 68 | if r.canceler == nil { 69 | // start new background ticker 70 | var newCtx context.Context 71 | newCtx, r.canceler = context.WithCancel(r.rootCtx) 72 | go r.worker(newCtx, r.interval) 73 | // release and return 74 | } 75 | 76 | r.started = time.Now() 77 | r.point.setTimestamp(r.started) 78 | r.Unlock() 79 | } 80 | 81 | func (r *intervalHistogramStream) EndIteration(dur time.Duration) { 82 | r.Lock() 83 | r.point.setTimestamp(r.started) 84 | r.catcher.Add(r.point.Counters.Number.RecordValue(1)) 85 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 86 | 87 | if !r.started.IsZero() { 88 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(time.Since(r.started)))) 89 | } 90 | 91 | r.Unlock() 92 | } 93 | 94 | func (r *intervalHistogramStream) SetTime(t time.Time) { 95 | r.Lock() 96 | r.point.Timestamp = t 97 | r.Unlock() 98 | } 99 | 100 | func (r *intervalHistogramStream) SetID(id int64) { 101 | r.Lock() 102 | r.point.ID = id 103 | r.Unlock() 104 | } 105 | 106 | func (r *intervalHistogramStream) SetTotalDuration(dur time.Duration) { 107 | r.Lock() 108 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(dur))) 109 | r.Unlock() 110 | } 111 | 112 | func (r *intervalHistogramStream) SetDuration(dur time.Duration) { 113 | r.Lock() 114 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 115 | r.Unlock() 116 | } 117 | 118 | func (r *intervalHistogramStream) EndTest() error { 119 | r.Lock() 120 | 121 | if !r.started.IsZero() { 122 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(time.Since(r.started)))) 123 | r.started = time.Time{} 124 | } 125 | if !r.point.Timestamp.IsZero() { 126 | r.catcher.Add(r.collector.Add(r.point)) 127 | } 128 | err := r.catcher.Resolve() 129 | r.reset() 130 | 131 | r.Unlock() 132 | return errors.WithStack(err) 133 | } 134 | 135 | func (r *intervalHistogramStream) Reset() { 136 | r.Lock() 137 | r.reset() 138 | r.Unlock() 139 | } 140 | 141 | func (r *intervalHistogramStream) reset() { 142 | if r.canceler != nil { 143 | r.canceler() 144 | r.canceler = nil 145 | } 146 | r.catcher = util.NewCatcher() 147 | r.point = NewHistogramMillisecond(r.point.Gauges) 148 | r.started = time.Time{} 149 | } 150 | 151 | func (r *intervalHistogramStream) IncOperations(val int64) { 152 | r.Lock() 153 | r.catcher.Add(r.point.Counters.Operations.RecordValue(val)) 154 | r.Unlock() 155 | } 156 | 157 | func (r *intervalHistogramStream) IncIterations(val int64) { 158 | r.Lock() 159 | r.catcher.Add(r.point.Counters.Number.RecordValue(val)) 160 | r.Unlock() 161 | } 162 | 163 | func (r *intervalHistogramStream) IncSize(val int64) { 164 | r.Lock() 165 | r.catcher.Add(r.point.Counters.Size.RecordValue(val)) 166 | r.Unlock() 167 | } 168 | 169 | func (r *intervalHistogramStream) IncError(val int64) { 170 | r.Lock() 171 | r.catcher.Add(r.point.Counters.Errors.RecordValue(val)) 172 | r.Unlock() 173 | } 174 | 175 | func (r *intervalHistogramStream) SetState(val int64) { 176 | r.Lock() 177 | r.point.Gauges.State = val 178 | r.Unlock() 179 | } 180 | 181 | func (r *intervalHistogramStream) SetWorkers(val int64) { 182 | r.Lock() 183 | r.point.Gauges.Workers = val 184 | r.Unlock() 185 | } 186 | 187 | func (r *intervalHistogramStream) SetFailed(val bool) { 188 | r.Lock() 189 | r.point.Gauges.Failed = val 190 | r.Unlock() 191 | } 192 | -------------------------------------------------------------------------------- /events/recorder_histogram_single.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mongodb/ftdc" 7 | "github.com/mongodb/ftdc/util" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type histogramSingle struct { 12 | point *PerformanceHDR 13 | started time.Time 14 | collector ftdc.Collector 15 | catcher util.Catcher 16 | } 17 | 18 | // NewSingleHistogramRecorder collects data and stores them with a histogram 19 | // format. Like the Single recorder, the implementation persists the histogram 20 | // every time you call EndTest. 21 | // 22 | // The timer histograms have a minimum value of 1 microsecond, and a maximum 23 | // value of 1 minute, with 5 significant digits. The counter histograms store 24 | // store between 0 and 10 thousand, with 5 significant digits. The gauges are 25 | // stored as integers. 26 | // 27 | // The histogram Single reporter is not safe for concurrent use without a 28 | // synchronized wrapper. 29 | func NewSingleHistogramRecorder(collector ftdc.Collector) Recorder { 30 | return &histogramSingle{ 31 | point: NewHistogramMillisecond(PerformanceGauges{}), 32 | collector: collector, 33 | catcher: util.NewCatcher(), 34 | } 35 | } 36 | 37 | func (r *histogramSingle) SetID(id int64) { r.point.ID = id } 38 | func (r *histogramSingle) SetState(val int64) { r.point.Gauges.State = val } 39 | func (r *histogramSingle) SetWorkers(val int64) { r.point.Gauges.Workers = val } 40 | func (r *histogramSingle) SetFailed(val bool) { r.point.Gauges.Failed = val } 41 | func (r *histogramSingle) IncOperations(val int64) { 42 | r.catcher.Add(r.point.Counters.Operations.RecordValue(val)) 43 | } 44 | func (r *histogramSingle) IncSize(val int64) { 45 | r.catcher.Add(r.point.Counters.Size.RecordValue(val)) 46 | } 47 | func (r *histogramSingle) IncError(val int64) { 48 | r.catcher.Add(r.point.Counters.Errors.RecordValue(val)) 49 | } 50 | 51 | func (r *histogramSingle) IncIterations(val int64) { 52 | r.catcher.Add(r.point.Counters.Number.RecordValue(val)) 53 | } 54 | 55 | func (r *histogramSingle) EndIteration(dur time.Duration) { 56 | r.point.setTimestamp(r.started) 57 | r.catcher.Add(r.point.Counters.Number.RecordValue(1)) 58 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 59 | if !r.started.IsZero() { 60 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(time.Since(r.started)))) 61 | r.started = time.Time{} 62 | } 63 | } 64 | 65 | func (r *histogramSingle) SetTotalDuration(dur time.Duration) { 66 | r.catcher.Add(r.point.Timers.Total.RecordValue(int64(dur))) 67 | } 68 | 69 | func (r *histogramSingle) SetDuration(dur time.Duration) { 70 | r.catcher.Add(r.point.Timers.Duration.RecordValue(int64(dur))) 71 | } 72 | 73 | func (r *histogramSingle) SetTime(t time.Time) { r.point.Timestamp = t } 74 | func (r *histogramSingle) BeginIteration() { r.started = time.Now() } 75 | 76 | func (r *histogramSingle) EndTest() error { 77 | r.point.setTimestamp(r.started) 78 | r.catcher.Add(r.collector.Add(r.point)) 79 | err := r.catcher.Resolve() 80 | r.Reset() 81 | return errors.WithStack(err) 82 | } 83 | 84 | func (r *histogramSingle) Reset() { 85 | r.catcher = util.NewCatcher() 86 | r.point = NewHistogramMillisecond(r.point.Gauges) 87 | r.started = time.Time{} 88 | } 89 | -------------------------------------------------------------------------------- /events/recorder_performance_grouped.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mongodb/ftdc" 7 | "github.com/mongodb/ftdc/util" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type groupStream struct { 12 | started time.Time 13 | lastCollected time.Time 14 | interval time.Duration 15 | point *Performance 16 | collector ftdc.Collector 17 | catcher util.Catcher 18 | } 19 | 20 | // NewGroupedRecorder blends the single and the interval recorders, but it 21 | // persists during the EndIteration call only if the specified interval has 22 | // elapsed. EndTest will persist any left over data. 23 | // 24 | // The Group recorder is not safe for concurrent access without a synchronized 25 | // wrapper. 26 | func NewGroupedRecorder(collector ftdc.Collector, interval time.Duration) Recorder { 27 | return &groupStream{ 28 | collector: collector, 29 | point: &Performance{Timestamp: time.Time{}}, 30 | catcher: util.NewCatcher(), 31 | interval: interval, 32 | lastCollected: time.Now(), 33 | } 34 | } 35 | 36 | func (r *groupStream) BeginIteration() { r.started = time.Now(); r.point.setTimestamp(r.started) } 37 | func (r *groupStream) IncOperations(val int64) { r.point.Counters.Operations += val } 38 | func (r *groupStream) IncIterations(val int64) { r.point.Counters.Number += val } 39 | func (r *groupStream) IncSize(val int64) { r.point.Counters.Size += val } 40 | func (r *groupStream) IncError(val int64) { r.point.Counters.Errors += val } 41 | func (r *groupStream) SetState(val int64) { r.point.Gauges.State = val } 42 | func (r *groupStream) SetWorkers(val int64) { r.point.Gauges.Workers = val } 43 | func (r *groupStream) SetFailed(val bool) { r.point.Gauges.Failed = val } 44 | func (r *groupStream) SetID(val int64) { r.point.ID = val } 45 | func (r *groupStream) SetTime(t time.Time) { r.point.Timestamp = t } 46 | func (r *groupStream) SetDuration(dur time.Duration) { r.point.Timers.Duration += dur } 47 | func (r *groupStream) SetTotalDuration(dur time.Duration) { r.point.Timers.Total += dur } 48 | func (r *groupStream) EndIteration(dur time.Duration) { 49 | r.point.Counters.Number++ 50 | if !r.started.IsZero() { 51 | r.point.Timers.Total += time.Since(r.started) 52 | } 53 | r.point.Timers.Duration += dur 54 | 55 | if time.Since(r.lastCollected) >= r.interval { 56 | r.point.setTimestamp(r.started) 57 | r.catcher.Add(r.collector.Add(r.point)) 58 | r.lastCollected = time.Now() 59 | r.point.Timestamp = time.Time{} 60 | } 61 | r.started = time.Time{} 62 | } 63 | 64 | func (r *groupStream) EndTest() error { 65 | if !r.point.Timestamp.IsZero() { 66 | r.catcher.Add(r.collector.Add(r.point)) 67 | } 68 | err := r.catcher.Resolve() 69 | r.Reset() 70 | return errors.WithStack(err) 71 | } 72 | 73 | func (r *groupStream) Reset() { 74 | r.catcher = util.NewCatcher() 75 | r.point = &Performance{ 76 | Gauges: r.point.Gauges, 77 | } 78 | r.started = time.Time{} 79 | r.lastCollected = time.Time{} 80 | } 81 | -------------------------------------------------------------------------------- /events/recorder_performance_interval.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "time" 7 | 8 | "github.com/mongodb/ftdc" 9 | "github.com/mongodb/ftdc/util" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | type intervalStream struct { 14 | point *Performance 15 | started time.Time 16 | collector ftdc.Collector 17 | catcher util.Catcher 18 | sync.Mutex 19 | 20 | interval time.Duration 21 | rootCtx context.Context 22 | canceler context.CancelFunc 23 | } 24 | 25 | // NewIntervalRecorder has similar semantics to histogram Grouped recorder, 26 | // but has a background process that persists data on the specified on the 27 | // specified interval rather than as a side effect of the EndTest call. 28 | // 29 | // The background thread is started in the BeginIteration operation if it 30 | // does not already exist and is terminated by the EndTest operation. 31 | // 32 | // The interval recorder is safe for concurrent use. 33 | func NewIntervalRecorder(ctx context.Context, collector ftdc.Collector, interval time.Duration) Recorder { 34 | return &intervalStream{ 35 | collector: collector, 36 | rootCtx: ctx, 37 | point: &Performance{Timestamp: time.Time{}}, 38 | catcher: util.NewCatcher(), 39 | interval: interval, 40 | } 41 | } 42 | 43 | func (r *intervalStream) worker(ctx context.Context, interval time.Duration) { 44 | ticker := time.NewTicker(r.interval) 45 | defer ticker.Stop() 46 | 47 | for { 48 | select { 49 | case <-ctx.Done(): 50 | return 51 | case <-ticker.C: 52 | r.Lock() 53 | // check context error in case in between the time when 54 | // the lock is requested and when the lock is obtained, 55 | // the context has been canceled 56 | if ctx.Err() != nil { 57 | return 58 | } 59 | r.point.setTimestamp(r.started) 60 | r.catcher.Add(r.collector.Add(r.point)) 61 | r.Unlock() 62 | } 63 | } 64 | } 65 | 66 | func (r *intervalStream) SetTime(t time.Time) { 67 | r.Lock() 68 | r.point.Timestamp = t 69 | r.Unlock() 70 | } 71 | 72 | func (r *intervalStream) SetID(id int64) { 73 | r.Lock() 74 | r.point.ID = id 75 | r.Unlock() 76 | } 77 | 78 | func (r *intervalStream) BeginIteration() { 79 | r.Lock() 80 | if r.canceler == nil { 81 | // start new background ticker 82 | var newCtx context.Context 83 | newCtx, r.canceler = context.WithCancel(r.rootCtx) 84 | go r.worker(newCtx, r.interval) 85 | // release and return 86 | } 87 | 88 | r.started = time.Now() 89 | r.point.setTimestamp(r.started) 90 | r.Unlock() 91 | } 92 | 93 | func (r *intervalStream) EndIteration(dur time.Duration) { 94 | r.Lock() 95 | 96 | r.point.setTimestamp(r.started) 97 | if !r.started.IsZero() { 98 | r.point.Timers.Total += time.Since(r.started) 99 | r.started = time.Time{} 100 | } 101 | r.point.Timers.Duration += dur 102 | 103 | r.Unlock() 104 | } 105 | 106 | func (r *intervalStream) EndTest() error { 107 | r.Lock() 108 | 109 | if !r.point.Timestamp.IsZero() { 110 | r.catcher.Add(r.collector.Add(r.point)) 111 | } 112 | err := r.catcher.Resolve() 113 | r.reset() 114 | 115 | r.Unlock() 116 | return errors.WithStack(err) 117 | } 118 | 119 | func (r *intervalStream) Reset() { 120 | r.Lock() 121 | r.reset() 122 | r.Unlock() 123 | } 124 | 125 | func (r *intervalStream) reset() { 126 | if r.canceler != nil { 127 | r.canceler() 128 | r.canceler = nil 129 | } 130 | r.catcher = util.NewCatcher() 131 | r.point = &Performance{ 132 | Gauges: r.point.Gauges, 133 | } 134 | r.started = time.Time{} 135 | } 136 | 137 | func (r *intervalStream) SetTotalDuration(dur time.Duration) { 138 | r.Lock() 139 | r.point.Timers.Total += dur 140 | r.Unlock() 141 | } 142 | 143 | func (r *intervalStream) SetDuration(dur time.Duration) { 144 | r.Lock() 145 | r.point.Timers.Duration += dur 146 | r.Unlock() 147 | } 148 | 149 | func (r *intervalStream) IncIterations(val int64) { 150 | r.Lock() 151 | r.point.Counters.Number += val 152 | r.Unlock() 153 | } 154 | 155 | func (r *intervalStream) IncOperations(val int64) { 156 | r.Lock() 157 | r.point.Counters.Operations += val 158 | r.Unlock() 159 | } 160 | 161 | func (r *intervalStream) IncSize(val int64) { 162 | r.Lock() 163 | r.point.Counters.Size += val 164 | r.Unlock() 165 | } 166 | 167 | func (r *intervalStream) IncError(val int64) { 168 | r.Lock() 169 | r.point.Counters.Errors += val 170 | r.Unlock() 171 | } 172 | 173 | func (r *intervalStream) SetState(val int64) { 174 | r.Lock() 175 | r.point.Gauges.State = val 176 | r.Unlock() 177 | } 178 | 179 | func (r *intervalStream) SetWorkers(val int64) { 180 | r.Lock() 181 | r.point.Gauges.Workers = val 182 | r.Unlock() 183 | } 184 | 185 | func (r *intervalStream) SetFailed(val bool) { 186 | r.Lock() 187 | r.point.Gauges.Failed = val 188 | r.Unlock() 189 | } 190 | -------------------------------------------------------------------------------- /events/recorder_performance_raw.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mongodb/ftdc" 7 | "github.com/mongodb/ftdc/util" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type rawStream struct { 12 | started time.Time 13 | point *Performance 14 | collector ftdc.Collector 15 | catcher util.Catcher 16 | } 17 | 18 | // NewRawRecorder records a new event every time that the EndIteration method 19 | // is called. 20 | // 21 | // The Raw recorder is not safe for concurrent access without a synchronized 22 | // wrapper. 23 | func NewRawRecorder(collector ftdc.Collector) Recorder { 24 | return &rawStream{ 25 | collector: collector, 26 | point: &Performance{Timestamp: time.Time{}}, 27 | catcher: util.NewCatcher(), 28 | } 29 | } 30 | 31 | func (r *rawStream) BeginIteration() { r.started = time.Now(); r.point.setTimestamp(r.started) } 32 | func (r *rawStream) SetTime(t time.Time) { r.point.Timestamp = t } 33 | func (r *rawStream) SetID(val int64) { r.point.ID = val } 34 | func (r *rawStream) SetTotalDuration(dur time.Duration) { r.point.Timers.Total = dur } 35 | func (r *rawStream) SetDuration(dur time.Duration) { r.point.Timers.Duration = dur } 36 | func (r *rawStream) IncOperations(val int64) { r.point.Counters.Operations += val } 37 | func (r *rawStream) IncIterations(val int64) { r.point.Counters.Number += val } 38 | func (r *rawStream) IncSize(val int64) { r.point.Counters.Size += val } 39 | func (r *rawStream) IncError(val int64) { r.point.Counters.Errors += val } 40 | func (r *rawStream) SetState(val int64) { r.point.Gauges.State = val } 41 | func (r *rawStream) SetWorkers(val int64) { r.point.Gauges.Workers = val } 42 | func (r *rawStream) SetFailed(val bool) { r.point.Gauges.Failed = val } 43 | func (r *rawStream) EndIteration(dur time.Duration) { 44 | r.point.Counters.Number++ 45 | if !r.started.IsZero() { 46 | r.point.Timers.Total += time.Since(r.started) 47 | } 48 | 49 | r.point.setTimestamp(r.started) 50 | r.point.Timers.Duration += dur 51 | r.catcher.Add(r.collector.Add(r.point)) 52 | r.started = time.Time{} 53 | } 54 | 55 | func (r *rawStream) EndTest() error { 56 | if !r.point.Timestamp.IsZero() { 57 | r.catcher.Add(r.collector.Add(r.point)) 58 | } 59 | err := r.catcher.Resolve() 60 | r.Reset() 61 | return errors.WithStack(err) 62 | } 63 | 64 | func (r *rawStream) Reset() { 65 | r.catcher = util.NewCatcher() 66 | r.point = &Performance{ 67 | Gauges: r.point.Gauges, 68 | } 69 | r.started = time.Time{} 70 | } 71 | -------------------------------------------------------------------------------- /events/recorder_performance_single.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mongodb/ftdc" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | type singleStream struct { 11 | started time.Time 12 | point *Performance 13 | collector ftdc.Collector 14 | } 15 | 16 | // NewSingleRecorder records a single event every time the EndTest method is 17 | // called, and otherwise just adds all counters and timing information to the 18 | // underlying point. 19 | // 20 | // The Single recorder is not safe for concurrent access without a synchronized 21 | // wrapper. 22 | func NewSingleRecorder(collector ftdc.Collector) Recorder { 23 | return &singleStream{ 24 | point: &Performance{Timestamp: time.Time{}}, 25 | collector: collector, 26 | } 27 | } 28 | 29 | func (r *singleStream) BeginIteration() { r.started = time.Now() } 30 | func (r *singleStream) SetTime(t time.Time) { r.point.Timestamp = t } 31 | func (r *singleStream) SetID(id int64) { r.point.ID = id } 32 | func (r *singleStream) SetTotalDuration(dur time.Duration) { r.point.Timers.Total += dur } 33 | func (r *singleStream) SetDuration(dur time.Duration) { r.point.Timers.Duration += dur } 34 | func (r *singleStream) IncOperations(val int64) { r.point.Counters.Operations += val } 35 | func (r *singleStream) IncIterations(val int64) { r.point.Counters.Number += val } 36 | func (r *singleStream) IncSize(val int64) { r.point.Counters.Size += val } 37 | func (r *singleStream) IncError(val int64) { r.point.Counters.Errors += val } 38 | func (r *singleStream) SetState(val int64) { r.point.Gauges.State = val } 39 | func (r *singleStream) SetWorkers(val int64) { r.point.Gauges.Workers = val } 40 | func (r *singleStream) SetFailed(val bool) { r.point.Gauges.Failed = val } 41 | func (r *singleStream) EndIteration(dur time.Duration) { 42 | r.point.setTimestamp(r.started) 43 | r.point.Counters.Number++ 44 | if !r.started.IsZero() { 45 | r.point.Timers.Total += time.Since(r.started) 46 | r.started = time.Time{} 47 | } 48 | r.point.Timers.Duration += dur 49 | } 50 | 51 | func (r *singleStream) EndTest() error { 52 | r.point.setTimestamp(r.started) 53 | err := errors.WithStack(r.collector.Add(r.point)) 54 | r.Reset() 55 | return errors.WithStack(err) 56 | } 57 | 58 | func (r *singleStream) Reset() { 59 | r.point = &Performance{ 60 | Gauges: r.point.Gauges, 61 | } 62 | r.started = time.Time{} 63 | } 64 | -------------------------------------------------------------------------------- /events/recorder_wrapper_stdlib.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import "time" 4 | 5 | // TimerManager is a subset of the testing.B tool, used to manage setup code. 6 | type TimerManager interface { 7 | ResetTimer() 8 | StartTimer() 9 | StopTimer() 10 | } 11 | 12 | // NewShimRecorder takes a recorder and acts as a thin recorder, using the 13 | // TimeManager interface for relevant Begin & End values. 14 | // 15 | // Go's standard library testing package has a *B type for benchmarking that 16 | // can pass as a TimerManager. 17 | func NewShimRecorder(r Recorder, tm TimerManager) Recorder { 18 | return &stdShim{ 19 | b: tm, 20 | Recorder: r, 21 | } 22 | } 23 | 24 | type stdShim struct { 25 | b TimerManager 26 | Recorder 27 | } 28 | 29 | func (r *stdShim) Reset() { 30 | r.b.ResetTimer() 31 | r.Recorder.Reset() 32 | } 33 | func (r *stdShim) Begin() { 34 | r.b.StartTimer() 35 | r.Recorder.BeginIteration() 36 | } 37 | func (r *stdShim) End(dur time.Duration) { 38 | r.b.StopTimer() 39 | r.Recorder.EndIteration(dur) 40 | } 41 | -------------------------------------------------------------------------------- /events/recorder_wrapper_sync.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | type syncRecorder struct { 9 | recorder Recorder 10 | sync.Mutex 11 | } 12 | 13 | // NewSynchronizedRecorder wraps a recorder implementation that is not 14 | // concurrent safe in a recorder implementation that provides safe concurrent 15 | // access without modifying the semantics of the recorder. 16 | // 17 | // Most Recorder implementations are not safe for concurrent use, although some 18 | // have this property as a result of persisting data on an interval. 19 | func NewSynchronizedRecorder(r Recorder) Recorder { 20 | return &syncRecorder{ 21 | recorder: r, 22 | } 23 | } 24 | 25 | func (r *syncRecorder) doOpInt(val int64, op func(int64)) { 26 | r.Lock() 27 | op(val) 28 | r.Unlock() 29 | } 30 | 31 | func (r *syncRecorder) doOpDur(val time.Duration, op func(time.Duration)) { 32 | r.Lock() 33 | op(val) 34 | r.Unlock() 35 | } 36 | 37 | func (r *syncRecorder) doOpTime(val time.Time, op func(time.Time)) { 38 | r.Lock() 39 | op(val) 40 | r.Unlock() 41 | } 42 | 43 | func (r *syncRecorder) doOpBool(val bool, op func(bool)) { 44 | r.Lock() 45 | op(val) 46 | r.Unlock() 47 | } 48 | 49 | func (r *syncRecorder) doOpErr(op func() error) error { 50 | r.Lock() 51 | err := op() 52 | r.Unlock() 53 | return err 54 | } 55 | 56 | func (r *syncRecorder) doOp(op func()) { 57 | r.Lock() 58 | op() 59 | r.Unlock() 60 | } 61 | 62 | func (r *syncRecorder) SetID(id int64) { r.doOpInt(id, r.recorder.SetID) } 63 | func (r *syncRecorder) SetTime(t time.Time) { r.doOpTime(t, r.recorder.SetTime) } 64 | func (r *syncRecorder) SetTotalDuration(val time.Duration) { 65 | r.doOpDur(val, r.recorder.SetTotalDuration) 66 | } 67 | func (r *syncRecorder) SetDuration(val time.Duration) { 68 | r.doOpDur(val, r.recorder.SetDuration) 69 | } 70 | func (r *syncRecorder) IncOperations(val int64) { r.doOpInt(val, r.recorder.IncOperations) } 71 | func (r *syncRecorder) IncIterations(val int64) { r.doOpInt(val, r.recorder.IncIterations) } 72 | func (r *syncRecorder) IncSize(val int64) { r.doOpInt(val, r.recorder.IncSize) } 73 | func (r *syncRecorder) IncError(val int64) { r.doOpInt(val, r.recorder.IncError) } 74 | func (r *syncRecorder) SetState(val int64) { r.doOpInt(val, r.recorder.SetState) } 75 | func (r *syncRecorder) SetWorkers(val int64) { r.doOpInt(val, r.recorder.SetWorkers) } 76 | func (r *syncRecorder) SetFailed(val bool) { r.doOpBool(val, r.recorder.SetFailed) } 77 | func (r *syncRecorder) BeginIteration() { r.doOp(r.recorder.BeginIteration) } 78 | func (r *syncRecorder) EndIteration(val time.Duration) { r.doOpDur(val, r.recorder.EndIteration) } 79 | func (r *syncRecorder) EndTest() error { return r.doOpErr(r.recorder.EndTest) } 80 | func (r *syncRecorder) Reset() { r.doOp(r.recorder.Reset) } 81 | -------------------------------------------------------------------------------- /evergreen.yaml: -------------------------------------------------------------------------------- 1 | ####################################### 2 | # YAML Templates # 3 | ####################################### 4 | variables: 5 | - &run-build 6 | # runs a build operation. The task name in evergreen should 7 | # correspond to a make target for the build operation. 8 | name: test 9 | must_have_test_results: true 10 | commands: 11 | - func: run-make 12 | vars: { target: "${task_name}" } 13 | 14 | ####################################### 15 | # Functions # 16 | ####################################### 17 | functions: 18 | get-project-and-modules: 19 | - command: git.get_project 20 | type: system 21 | params: 22 | directory: ftdc 23 | - command: subprocess.exec 24 | type: setup 25 | params: 26 | working_dir: ftdc 27 | binary: make 28 | args: ["mod-tidy"] 29 | include_expansions_in_env: ["GOROOT"] 30 | parse-results: 31 | command: gotest.parse_files 32 | type: setup 33 | params: 34 | files: 35 | - "ftdc/build/output.*" 36 | run-make: 37 | command: subprocess.exec 38 | type: test 39 | params: 40 | working_dir: ftdc 41 | binary: make 42 | args: ["${make_args|}", "${target}"] 43 | include_expansions_in_env: ["GOROOT", "RACE_DETECTOR", "TEST_TIMEOUT"] 44 | 45 | ####################################### 46 | # Tasks # 47 | ####################################### 48 | tasks: 49 | - <<: *run-build 50 | tags: ["test"] 51 | name: test-ftdc 52 | - <<: *run-build 53 | tags: ["test"] 54 | name: test-events 55 | - <<: *run-build 56 | tags: ["test"] 57 | name: test-hdrhist 58 | - <<: *run-build 59 | tags: ["test"] 60 | name: test-metrics 61 | - <<: *run-build 62 | tags: ["test"] 63 | name: test-util 64 | 65 | - <<: *run-build 66 | tags: ["lint"] 67 | name: lint-ftdc 68 | - <<: *run-build 69 | tags: ["lint"] 70 | name: lint-events 71 | - <<: *run-build 72 | tags: ["lint"] 73 | name: lint-hdrhist 74 | - <<: *run-build 75 | tags: ["lint"] 76 | name: lint-metrics 77 | - <<: *run-build 78 | tags: ["lint"] 79 | name: lint-util 80 | 81 | - name: verify-mod-tidy 82 | commands: 83 | - command: git.get_project 84 | type: system 85 | params: 86 | directory: ftdc 87 | - func: run-make 88 | vars: { target: "${task_name}" } 89 | 90 | task_groups: 91 | - name: lintGroup 92 | tasks: [ ".lint" ] 93 | max_hosts: 2 94 | setup_group: 95 | - func: get-project-and-modules 96 | setup_task: 97 | - func: run-make 98 | vars: { target: "clean-results" } 99 | teardown_task: 100 | - func: parse-results 101 | - name: testGroup 102 | tasks: [ ".test" ] 103 | max_hosts: 2 104 | setup_group_can_fail_task: true 105 | share_processes: true 106 | setup_group: 107 | - func: get-project-and-modules 108 | setup_task: 109 | - func: run-make 110 | vars: { target: "clean-results" } 111 | teardown_task: 112 | - func: parse-results 113 | 114 | ####################################### 115 | # Buildvariants # 116 | ####################################### 117 | buildvariants: 118 | - name: lint 119 | display_name: Lint 120 | expansions: 121 | GOROOT: /opt/golang/go1.24 122 | run_on: 123 | - ubuntu2204-small 124 | tasks: 125 | - lintGroup 126 | - verify-mod-tidy 127 | 128 | - name: ubuntu 129 | display_name: Ubuntu 22.04 130 | expansions: 131 | GOROOT: /opt/golang/go1.24 132 | RACE_DETECTOR: true 133 | run_on: 134 | - ubuntu2204-small 135 | tasks: [ "testGroup" ] 136 | 137 | - name: macos 138 | display_name: macOS 139 | expansions: 140 | GOROOT: /opt/golang/go1.24 141 | run_on: 142 | - macos-1100-arm64 143 | tasks: [ "testGroup" ] 144 | 145 | - name: windows 146 | display_name: Windows 147 | run_on: 148 | - windows-vsCurrent-small 149 | expansions: 150 | GOROOT: C:/golang/go1.24 151 | tasks: [ "testGroup" ] 152 | -------------------------------------------------------------------------------- /ftdc.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | 8 | "github.com/evergreen-ci/birch" 9 | "github.com/evergreen-ci/birch/bsontype" 10 | ) 11 | 12 | // Chunk represents a 'metric chunk' of data in the FTDC. 13 | type Chunk struct { 14 | Metrics []Metric 15 | nPoints int 16 | id time.Time 17 | metadata *birch.Document 18 | reference *birch.Document 19 | } 20 | 21 | func (c *Chunk) GetMetadata() *birch.Document { return c.metadata } 22 | func (c *Chunk) Size() int { return c.nPoints } 23 | func (c *Chunk) Len() int { return len(c.Metrics) } 24 | 25 | // Iterator returns an iterator that you can use to read documents for 26 | // each sample period in the chunk. Documents are returned in collection 27 | // order, with keys flattened and dot-separated fully qualified 28 | // paths. 29 | // 30 | // The documents are constructed from the metrics data lazily. 31 | func (c *Chunk) Iterator(ctx context.Context) Iterator { 32 | sctx, cancel := context.WithCancel(ctx) 33 | return &sampleIterator{ 34 | closer: cancel, 35 | stream: c.streamFlattenedDocuments(sctx), 36 | metadata: c.GetMetadata(), 37 | } 38 | } 39 | 40 | // StructuredIterator returns the contents of the chunk as a sequence 41 | // of documents that (mostly) resemble the original source documents 42 | // (with the non-metrics fields omitted.) The output documents mirror 43 | // the structure of the input documents. 44 | func (c *Chunk) StructuredIterator(ctx context.Context) Iterator { 45 | sctx, cancel := context.WithCancel(ctx) 46 | return &sampleIterator{ 47 | closer: cancel, 48 | stream: c.streamDocuments(sctx), 49 | metadata: c.GetMetadata(), 50 | } 51 | } 52 | 53 | // Metric represents an item in a chunk. 54 | type Metric struct { 55 | // For metrics that were derived from nested BSON documents, 56 | // this preserves the path to the field, in support of being 57 | // able to reconstitute metrics/chunks as a stream of BSON 58 | // documents. 59 | ParentPath []string 60 | 61 | // KeyName is the specific field name of a metric in. It is 62 | // *not* fully qualified with its parent document path, use 63 | // the Key() method to access a value with more appropriate 64 | // user facing context. 65 | KeyName string 66 | 67 | // Values is an array of each value collected for this metric. 68 | // During decoding, this attribute stores delta-encoded 69 | // values, but those are expanded during decoding and should 70 | // never be visible to user. 71 | Values []int64 72 | 73 | // Used during decoding to expand the delta encoded values. In 74 | // a properly decoded value, it should always report 75 | startingValue int64 76 | 77 | originalType bsontype.Type 78 | } 79 | 80 | func (m *Metric) Key() string { 81 | return strings.Join(append(m.ParentPath, m.KeyName), ".") 82 | } 83 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mongodb/ftdc 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/evergreen-ci/birch v0.0.0-20191213201306-f4dae6f450a2 7 | github.com/mongodb/grip v0.0.0-20250224221724-fc8adcb1fe8e 8 | github.com/papertrail/go-tail v0.0.0-20180509224916-973c153b0431 9 | github.com/pkg/errors v0.9.1 10 | github.com/stretchr/testify v1.8.3 11 | go.mongodb.org/mongo-driver/v2 v2.0.0 12 | ) 13 | 14 | require ( 15 | github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect 16 | github.com/PuerkitoBio/rehttp v1.1.0 // indirect 17 | github.com/andygrunwald/go-jira v1.14.0 // indirect 18 | github.com/aws/aws-sdk-go-v2 v1.24.1 // indirect 19 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect 20 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect 21 | github.com/aws/aws-sdk-go-v2/service/ses v1.19.6 // indirect 22 | github.com/aws/smithy-go v1.19.0 // indirect 23 | github.com/cloudflare/circl v1.3.3 // indirect 24 | github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect 25 | github.com/davecgh/go-spew v1.1.1 // indirect 26 | github.com/dghubble/oauth1 v0.7.2 // indirect 27 | github.com/evergreen-ci/utility v0.0.0-20230613214910-daa83783f97c // indirect 28 | github.com/fatih/structs v1.1.0 // indirect 29 | github.com/felixge/httpsnoop v1.0.3 // indirect 30 | github.com/fsnotify/fsnotify v1.5.1 // indirect 31 | github.com/fuyufjh/splunk-hec-go v0.3.4-0.20190414090710-10df423a9f36 // indirect 32 | github.com/go-logr/logr v1.2.4 // indirect 33 | github.com/go-logr/stdr v1.2.2 // indirect 34 | github.com/go-ole/go-ole v1.2.6 // indirect 35 | github.com/golang-jwt/jwt v3.2.1+incompatible // indirect 36 | github.com/golang/protobuf v1.5.2 // indirect 37 | github.com/google/go-github/v53 v53.0.0 // indirect 38 | github.com/google/go-querystring v1.1.0 // indirect 39 | github.com/google/uuid v1.3.0 // indirect 40 | github.com/gorilla/websocket v1.4.2 // indirect 41 | github.com/jmespath/go-jmespath v0.4.0 // indirect 42 | github.com/jpillora/backoff v1.0.0 // indirect 43 | github.com/kr/text v0.2.0 // indirect 44 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 45 | github.com/mattn/go-xmpp v0.0.0-20210723025538-3871461df959 // indirect 46 | github.com/pmezard/go-difflib v1.0.0 // indirect 47 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 48 | github.com/rogpeppe/go-internal v1.14.0 // indirect 49 | github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect 50 | github.com/shirou/gopsutil/v3 v3.23.5 // indirect 51 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 52 | github.com/slack-go/slack v0.12.1 // indirect 53 | github.com/tklauser/go-sysconf v0.3.11 // indirect 54 | github.com/tklauser/numcpus v0.6.0 // indirect 55 | github.com/trivago/tgo v1.0.7 // indirect 56 | github.com/yusufpapurcu/wmi v1.2.3 // indirect 57 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 // indirect 58 | go.opentelemetry.io/otel v1.16.0 // indirect 59 | go.opentelemetry.io/otel/metric v1.16.0 // indirect 60 | go.opentelemetry.io/otel/sdk v1.15.1 // indirect 61 | go.opentelemetry.io/otel/trace v1.16.0 // indirect 62 | golang.org/x/crypto v0.29.0 // indirect 63 | golang.org/x/net v0.21.0 // indirect 64 | golang.org/x/oauth2 v0.8.0 // indirect 65 | golang.org/x/sys v0.27.0 // indirect 66 | google.golang.org/appengine v1.6.7 // indirect 67 | google.golang.org/protobuf v1.28.0 // indirect 68 | gopkg.in/yaml.v2 v2.4.0 // indirect 69 | gopkg.in/yaml.v3 v3.0.1 // indirect 70 | ) 71 | -------------------------------------------------------------------------------- /hdrhist/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Coda Hale 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /hdrhist/snapshot.go: -------------------------------------------------------------------------------- 1 | package hdrhist 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/pkg/errors" 8 | "go.mongodb.org/mongo-driver/v2/bson" 9 | ) 10 | 11 | // A Snapshot is an exported view of a Histogram, useful for serializing them. 12 | // A Histogram can be constructed from it by passing it to Import. 13 | type Snapshot struct { 14 | LowestTrackableValue int64 `bson:"lowest" json:"lowest" yaml:"lowest"` 15 | HighestTrackableValue int64 `bson:"highest" json:"highest" yaml:"highest"` 16 | SignificantFigures int64 `bson:"figures" json:"figures" yaml:"figures"` 17 | Counts []int64 `bson:"counts" json:"counts" yaml:"counts"` 18 | } 19 | 20 | func (h *Histogram) MarshalDocument() (*birch.Document, error) { 21 | return birch.DC.Make(5).Append( 22 | birch.EC.Int64("lowest", h.lowestTrackableValue), 23 | birch.EC.Int64("highest", h.highestTrackableValue), 24 | birch.EC.Int64("figures", h.significantFigures), 25 | birch.EC.SliceInt64("counts", h.counts), 26 | ), nil 27 | } 28 | 29 | func (h *Histogram) MarshalBSON() ([]byte, error) { return birch.MarshalDocumentBSON(h) } 30 | func (h *Histogram) MarshalJSON() ([]byte, error) { return json.Marshal(h.Export()) } 31 | 32 | func (h *Histogram) UnmarshalBSON(in []byte) error { 33 | s := &Snapshot{} 34 | if err := bson.Unmarshal(in, s); err != nil { 35 | return errors.WithStack(err) 36 | } 37 | 38 | *h = *Import(s) 39 | return nil 40 | } 41 | 42 | func (h *Histogram) UnmarshalJSON(in []byte) error { 43 | s := &Snapshot{} 44 | if err := json.Unmarshal(in, s); err != nil { 45 | return errors.WithStack(err) 46 | } 47 | 48 | *h = *Import(s) 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /hdrhist/window.go: -------------------------------------------------------------------------------- 1 | package hdrhist 2 | 3 | // A WindowedHistogram combines histograms to provide windowed statistics. 4 | type WindowedHistogram struct { 5 | idx int 6 | h []Histogram 7 | m *Histogram 8 | 9 | Current *Histogram 10 | } 11 | 12 | // NewWindowed creates a new WindowedHistogram with N underlying histograms with 13 | // the given parameters. 14 | func NewWindowed(n int, minValue, maxValue int64, sigfigs int) *WindowedHistogram { 15 | w := WindowedHistogram{ 16 | idx: -1, 17 | h: make([]Histogram, n), 18 | m: New(minValue, maxValue, sigfigs), 19 | } 20 | 21 | for i := range w.h { 22 | w.h[i] = *New(minValue, maxValue, sigfigs) 23 | } 24 | w.Rotate() 25 | 26 | return &w 27 | } 28 | 29 | // Merge returns a histogram which includes the recorded values from all the 30 | // sections of the window. 31 | func (w *WindowedHistogram) Merge() *Histogram { 32 | w.m.Reset() 33 | for _, h := range w.h { 34 | w.m.Merge(&h) 35 | } 36 | return w.m 37 | } 38 | 39 | // Rotate resets the oldest histogram and rotates it to be used as the current 40 | // histogram. 41 | func (w *WindowedHistogram) Rotate() { 42 | w.idx++ 43 | w.Current = &w.h[w.idx%len(w.h)] 44 | w.Current.Reset() 45 | } 46 | -------------------------------------------------------------------------------- /hdrhist/window_test.go: -------------------------------------------------------------------------------- 1 | package hdrhist_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mongodb/ftdc/hdrhist" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestWindowedHistogram(t *testing.T) { 11 | w := hdrhist.NewWindowed(2, 1, 1000, 3) 12 | 13 | for i := 0; i < 100; i++ { 14 | assert.NoError(t, w.Current.RecordValue(int64(i))) 15 | } 16 | w.Rotate() 17 | 18 | for i := 100; i < 200; i++ { 19 | assert.NoError(t, w.Current.RecordValue(int64(i))) 20 | } 21 | w.Rotate() 22 | 23 | for i := 200; i < 300; i++ { 24 | assert.NoError(t, w.Current.RecordValue(int64(i))) 25 | } 26 | 27 | if v, want := w.Merge().ValueAtQuantile(50), int64(199); v != want { 28 | t.Errorf("Median was %v, but expected %v", v, want) 29 | } 30 | } 31 | 32 | func BenchmarkWindowedHistogramRecordAndRotate(b *testing.B) { 33 | w := hdrhist.NewWindowed(3, 1, 10000000, 3) 34 | b.ReportAllocs() 35 | b.ResetTimer() 36 | 37 | for i := 0; i < b.N; i++ { 38 | if err := w.Current.RecordValue(100); err != nil { 39 | b.Fatal(err) 40 | } 41 | 42 | if i%100000 == 1 { 43 | w.Rotate() 44 | } 45 | } 46 | } 47 | 48 | func BenchmarkWindowedHistogramMerge(b *testing.B) { 49 | w := hdrhist.NewWindowed(3, 1, 10000000, 3) 50 | for i := 0; i < 10000000; i++ { 51 | if err := w.Current.RecordValue(100); err != nil { 52 | b.Fatal(err) 53 | } 54 | 55 | if i%100000 == 1 { 56 | w.Rotate() 57 | } 58 | } 59 | b.ReportAllocs() 60 | b.ResetTimer() 61 | 62 | for i := 0; i < b.N; i++ { 63 | w.Merge() 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /iterator.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/evergreen-ci/birch" 8 | "github.com/mongodb/ftdc/util" 9 | ) 10 | 11 | type Iterator interface { 12 | Next() bool 13 | Document() *birch.Document 14 | Metadata() *birch.Document 15 | Err() error 16 | Close() 17 | } 18 | 19 | // ReadMetrics returns a standard document iterator that reads FTDC 20 | // chunks. The Documents returned by the iterator are flattened. 21 | func ReadMetrics(ctx context.Context, r io.Reader) Iterator { 22 | iterctx, cancel := context.WithCancel(ctx) 23 | iter := &combinedIterator{ 24 | closer: cancel, 25 | chunks: ReadChunks(iterctx, r), 26 | flatten: true, 27 | pipe: make(chan *birch.Document, 100), 28 | catcher: util.NewCatcher(), 29 | } 30 | 31 | go iter.worker(iterctx) 32 | return iter 33 | } 34 | 35 | // ReadStructuredMetrics returns a standard document iterator that reads FTDC 36 | // chunks. The Documents returned by the iterator retain the structure 37 | // of the input documents. 38 | func ReadStructuredMetrics(ctx context.Context, r io.Reader) Iterator { 39 | iterctx, cancel := context.WithCancel(ctx) 40 | iter := &combinedIterator{ 41 | closer: cancel, 42 | chunks: ReadChunks(iterctx, r), 43 | flatten: false, 44 | pipe: make(chan *birch.Document, 100), 45 | catcher: util.NewCatcher(), 46 | } 47 | 48 | go iter.worker(iterctx) 49 | return iter 50 | } 51 | 52 | // ReadMatrix returns a "matrix format" for the data in a chunk. The 53 | // ducments returned by the iterator represent the entire chunk, in 54 | // flattened form, with each field representing a single metric as an 55 | // array of all values for the event. 56 | // 57 | // The matrix documents have full type fidelity, but are not 58 | // substantially less expensive to produce than full iteration. 59 | func ReadMatrix(ctx context.Context, r io.Reader) Iterator { 60 | iterctx, cancel := context.WithCancel(ctx) 61 | iter := &matrixIterator{ 62 | closer: cancel, 63 | chunks: ReadChunks(iterctx, r), 64 | pipe: make(chan *birch.Document, 25), 65 | catcher: util.NewCatcher(), 66 | } 67 | 68 | go iter.worker(iterctx) 69 | return iter 70 | } 71 | 72 | // ReadSeries is similar to the ReadMatrix format, and produces a 73 | // single document per chunk, that contains the flattented keys for 74 | // that chunk, mapped to arrays of all the values of the chunk. 75 | // 76 | // The matrix documents have better type fidelity than raw chunks but 77 | // do not properly collapse the bson timestamp type. To use these 78 | // values produced by the iterator, consider marshaling them directly 79 | // to map[string]interface{} and use a case statement, on the values 80 | // in the map, such as: 81 | // 82 | // switch v.(type) { 83 | // case []int32: 84 | // // ... 85 | // case []int64: 86 | // // ... 87 | // case []bool: 88 | // // ... 89 | // case []time.Time: 90 | // // ... 91 | // case []float64: 92 | // // ... 93 | // } 94 | // 95 | // Although the *birch.Document type does support iteration directly. 96 | func ReadSeries(ctx context.Context, r io.Reader) Iterator { 97 | iterctx, cancel := context.WithCancel(ctx) 98 | iter := &matrixIterator{ 99 | closer: cancel, 100 | chunks: ReadChunks(iterctx, r), 101 | pipe: make(chan *birch.Document, 25), 102 | catcher: util.NewCatcher(), 103 | reflect: true, 104 | } 105 | 106 | go iter.worker(iterctx) 107 | return iter 108 | } 109 | -------------------------------------------------------------------------------- /iterator_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func BenchmarkIterator(b *testing.B) { 15 | ctx, cancel := context.WithCancel(context.Background()) 16 | defer cancel() 17 | 18 | for _, test := range []struct { 19 | Name string 20 | Path string 21 | }{ 22 | { 23 | Name: "PerfMockSmall", 24 | Path: "perf_metrics_small.ftdc", 25 | }, 26 | { 27 | Name: "PerfMock", 28 | Path: "perf_metrics.ftdc", 29 | }, 30 | { 31 | Name: "ServerStatus", 32 | Path: "metrics.ftdc", 33 | }, 34 | } { 35 | b.Run(test.Name, func(b *testing.B) { 36 | file, err := os.Open(test.Path) 37 | require.NoError(b, err) 38 | defer func() { 39 | if err = file.Close(); err != nil { 40 | fmt.Println(err) 41 | } 42 | }() 43 | data, err := ioutil.ReadAll(file) 44 | require.NoError(b, err) 45 | b.ResetTimer() 46 | 47 | b.Run("Chunk", func(b *testing.B) { 48 | b.Run("Resolving", func(b *testing.B) { 49 | iter := ReadChunks(ctx, bytes.NewBuffer(data)) 50 | b.ResetTimer() 51 | for n := 0; n < b.N; n++ { 52 | if !iter.Next() { 53 | break 54 | } 55 | require.NotNil(b, iter.Chunk()) 56 | } 57 | }) 58 | b.Run("Iterating", func(b *testing.B) { 59 | for n := 0; n < b.N; n++ { 60 | iter := ReadChunks(ctx, bytes.NewBuffer(data)) 61 | for iter.Next() { 62 | require.NotNil(b, iter.Chunk()) 63 | } 64 | } 65 | }) 66 | }) 67 | b.Run("Series", func(b *testing.B) { 68 | b.Run("Resolving", func(b *testing.B) { 69 | iter := ReadSeries(ctx, bytes.NewBuffer(data)) 70 | b.ResetTimer() 71 | for n := 0; n < b.N; n++ { 72 | if !iter.Next() { 73 | break 74 | } 75 | require.NotNil(b, iter.Document()) 76 | } 77 | }) 78 | b.Run("Iterating", func(b *testing.B) { 79 | for n := 0; n < b.N; n++ { 80 | iter := ReadSeries(ctx, bytes.NewBuffer(data)) 81 | for iter.Next() { 82 | require.NotNil(b, iter.Document()) 83 | } 84 | } 85 | }) 86 | }) 87 | b.Run("Matrix", func(b *testing.B) { 88 | b.Run("Resolving", func(b *testing.B) { 89 | iter := ReadMatrix(ctx, bytes.NewBuffer(data)) 90 | b.ResetTimer() 91 | for n := 0; n < b.N; n++ { 92 | if !iter.Next() { 93 | break 94 | } 95 | require.NotNil(b, iter.Document()) 96 | } 97 | }) 98 | b.Run("Iterating", func(b *testing.B) { 99 | for n := 0; n < b.N; n++ { 100 | iter := ReadMatrix(ctx, bytes.NewBuffer(data)) 101 | for iter.Next() { 102 | require.NotNil(b, iter.Document()) 103 | } 104 | } 105 | }) 106 | }) 107 | b.Run("Structured", func(b *testing.B) { 108 | b.Run("Resolving", func(b *testing.B) { 109 | iter := ReadStructuredMetrics(ctx, bytes.NewBuffer(data)) 110 | b.ResetTimer() 111 | for n := 0; n < b.N; n++ { 112 | if !iter.Next() { 113 | break 114 | } 115 | require.NotNil(b, iter.Document()) 116 | } 117 | }) 118 | b.Run("Iterating", func(b *testing.B) { 119 | for n := 0; n < b.N; n++ { 120 | iter := ReadStructuredMetrics(ctx, bytes.NewBuffer(data)) 121 | for iter.Next() { 122 | require.NotNil(b, iter.Document()) 123 | } 124 | } 125 | }) 126 | }) 127 | b.Run("Flattened", func(b *testing.B) { 128 | b.Run("Resolving", func(b *testing.B) { 129 | iter := ReadStructuredMetrics(ctx, bytes.NewBuffer(data)) 130 | b.ResetTimer() 131 | for n := 0; n < b.N; n++ { 132 | if !iter.Next() { 133 | break 134 | } 135 | require.NotNil(b, iter.Document()) 136 | } 137 | }) 138 | b.Run("Iterating", func(b *testing.B) { 139 | for n := 0; n < b.N; n++ { 140 | iter := ReadStructuredMetrics(ctx, bytes.NewBuffer(data)) 141 | for iter.Next() { 142 | require.NotNil(b, iter.Document()) 143 | } 144 | } 145 | }) 146 | }) 147 | }) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /iterator_chunk.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/evergreen-ci/birch" 8 | "github.com/mongodb/ftdc/util" 9 | ) 10 | 11 | // ChunkIterator is a simple iterator for reading off of an FTDC data 12 | // source (e.g. file). The iterator processes chunks batches of 13 | // metrics lazily, reading form the io.Reader every time the iterator 14 | // is advanced. 15 | // 16 | // Use the iterator as follows: 17 | // 18 | // iter := ReadChunks(ctx, file) 19 | // 20 | // for iter.Next() { 21 | // chunk := iter.Chunk() 22 | // 23 | // // 24 | // 25 | // } 26 | // 27 | // if err := iter.Err(); err != nil { 28 | // return err 29 | // } 30 | // 31 | // You MUST call the Chunk() method no more than once per iteration. 32 | // 33 | // You shoule check the Err() method when iterator is complete to see 34 | // if there were any issues encountered when decoding chunks. 35 | type ChunkIterator struct { 36 | pipe chan *Chunk 37 | next *Chunk 38 | cancel context.CancelFunc 39 | closed bool 40 | catcher util.Catcher 41 | } 42 | 43 | // ReadChunks creates a ChunkIterator from an underlying FTDC data 44 | // source. 45 | func ReadChunks(ctx context.Context, r io.Reader) *ChunkIterator { 46 | iter := &ChunkIterator{ 47 | catcher: util.NewCatcher(), 48 | pipe: make(chan *Chunk, 2), 49 | } 50 | 51 | ipc := make(chan *birch.Document) 52 | ctx, iter.cancel = context.WithCancel(ctx) 53 | 54 | go func() { 55 | iter.catcher.Add(readDiagnostic(ctx, r, ipc)) 56 | }() 57 | 58 | go func() { 59 | iter.catcher.Add(readChunks(ctx, ipc, iter.pipe)) 60 | }() 61 | 62 | return iter 63 | } 64 | 65 | // Next advances the iterator and returns true if the iterator has a 66 | // chunk that is unprocessed. Use the Chunk() method to access the 67 | // iterator. 68 | func (iter *ChunkIterator) Next() bool { 69 | next, ok := <-iter.pipe 70 | if !ok { 71 | return false 72 | } 73 | 74 | iter.next = next 75 | return true 76 | } 77 | 78 | // Chunk returns a copy of the chunk processed by the iterator. You 79 | // must call Chunk no more than once per iteration. Additional 80 | // accesses to Chunk will panic. 81 | func (iter *ChunkIterator) Chunk() *Chunk { 82 | return iter.next 83 | } 84 | 85 | // Close releases resources of the iterator. Use this method to 86 | // release those resources if you stop iterating before the iterator 87 | // is exhausted. Canceling the context that you used to create the 88 | // iterator has the same effect. 89 | func (iter *ChunkIterator) Close() { iter.cancel(); iter.closed = true } 90 | 91 | // Err returns a non-nil error if the iterator encountered any errors 92 | // during iteration. 93 | func (iter *ChunkIterator) Err() error { return iter.catcher.Resolve() } 94 | -------------------------------------------------------------------------------- /iterator_combined.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/mongodb/ftdc/util" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type combinedIterator struct { 12 | closer context.CancelFunc 13 | chunks *ChunkIterator 14 | sample *sampleIterator 15 | metadata *birch.Document 16 | document *birch.Document 17 | pipe chan *birch.Document 18 | catcher util.Catcher 19 | flatten bool 20 | } 21 | 22 | func (iter *combinedIterator) Close() { 23 | iter.closer() 24 | if iter.sample != nil { 25 | iter.sample.Close() 26 | } 27 | 28 | if iter.chunks != nil { 29 | iter.chunks.Close() 30 | } 31 | } 32 | 33 | func (iter *combinedIterator) Err() error { return iter.catcher.Resolve() } 34 | func (iter *combinedIterator) Metadata() *birch.Document { return iter.metadata } 35 | func (iter *combinedIterator) Document() *birch.Document { return iter.document } 36 | 37 | func (iter *combinedIterator) Next() bool { 38 | doc, ok := <-iter.pipe 39 | if !ok { 40 | return false 41 | } 42 | 43 | iter.document = doc 44 | return true 45 | } 46 | 47 | func (iter *combinedIterator) worker(ctx context.Context) { 48 | defer close(iter.pipe) 49 | var ok bool 50 | 51 | for iter.chunks.Next() { 52 | chunk := iter.chunks.Chunk() 53 | 54 | if iter.flatten { 55 | iter.sample, ok = chunk.Iterator(ctx).(*sampleIterator) 56 | } else { 57 | iter.sample, ok = chunk.StructuredIterator(ctx).(*sampleIterator) 58 | } 59 | if !ok { 60 | iter.catcher.Add(errors.New("programmer error")) 61 | return 62 | } 63 | if iter.metadata != nil { 64 | iter.metadata = chunk.GetMetadata() 65 | } 66 | 67 | for iter.sample.Next() { 68 | select { 69 | case iter.pipe <- iter.sample.Document(): 70 | continue 71 | case <-ctx.Done(): 72 | iter.catcher.Add(errors.New("operation aborted")) 73 | return 74 | } 75 | 76 | } 77 | iter.catcher.Add(iter.sample.Err()) 78 | iter.sample.Close() 79 | } 80 | iter.catcher.Add(iter.chunks.Err()) 81 | } 82 | -------------------------------------------------------------------------------- /iterator_matrix.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "time" 7 | 8 | "github.com/evergreen-ci/birch" 9 | "github.com/evergreen-ci/birch/bsontype" 10 | "github.com/mongodb/ftdc/util" 11 | "github.com/pkg/errors" 12 | "go.mongodb.org/mongo-driver/v2/bson" 13 | ) 14 | 15 | func (c *Chunk) exportMatrix() map[string]interface{} { 16 | out := make(map[string]interface{}) 17 | for _, m := range c.Metrics { 18 | out[m.Key()] = m.getSeries() 19 | } 20 | return out 21 | } 22 | 23 | func (c *Chunk) export() (*birch.Document, error) { 24 | doc := birch.DC.Make(len(c.Metrics)) 25 | sample := 0 26 | 27 | var elem *birch.Element 28 | var err error 29 | 30 | for i := 0; i < len(c.Metrics); i++ { 31 | elem, sample, err = rehydrateMatrix(c.Metrics, sample) 32 | if err == io.EOF { 33 | break 34 | } 35 | if err != nil { 36 | return nil, errors.WithStack(err) 37 | } 38 | 39 | doc.Append(elem) 40 | } 41 | 42 | return doc, nil 43 | } 44 | 45 | func (m *Metric) getSeries() interface{} { 46 | switch m.originalType { 47 | case bsontype.Int64, bsontype.Timestamp: 48 | out := make([]int64, len(m.Values)) 49 | copy(out, m.Values) 50 | return out 51 | case bsontype.Int32: 52 | out := make([]int32, len(m.Values)) 53 | for idx, p := range m.Values { 54 | out[idx] = int32(p) 55 | } 56 | return out 57 | case bsontype.Boolean: 58 | out := make([]bool, len(m.Values)) 59 | for idx, p := range m.Values { 60 | out[idx] = p != 0 61 | } 62 | return out 63 | case bsontype.Double: 64 | out := make([]float64, len(m.Values)) 65 | for idx, p := range m.Values { 66 | out[idx] = restoreFloat(p) 67 | } 68 | return out 69 | case bsontype.DateTime: 70 | out := make([]time.Time, len(m.Values)) 71 | for idx, p := range m.Values { 72 | out[idx] = timeEpocMs(p) 73 | } 74 | return out 75 | default: 76 | return nil 77 | } 78 | } 79 | 80 | type matrixIterator struct { 81 | chunks *ChunkIterator 82 | closer context.CancelFunc 83 | metadata *birch.Document 84 | document *birch.Document 85 | pipe chan *birch.Document 86 | catcher util.Catcher 87 | reflect bool 88 | } 89 | 90 | func (iter *matrixIterator) Close() { 91 | if iter.chunks != nil { 92 | iter.chunks.Close() 93 | } 94 | } 95 | 96 | func (iter *matrixIterator) Err() error { return iter.catcher.Resolve() } 97 | func (iter *matrixIterator) Metadata() *birch.Document { return iter.metadata } 98 | func (iter *matrixIterator) Document() *birch.Document { return iter.document } 99 | func (iter *matrixIterator) Next() bool { 100 | doc, ok := <-iter.pipe 101 | if !ok { 102 | return false 103 | } 104 | 105 | iter.document = doc 106 | return true 107 | } 108 | 109 | func (iter *matrixIterator) worker(ctx context.Context) { 110 | defer func() { iter.catcher.Add(iter.chunks.Err()) }() 111 | defer close(iter.pipe) 112 | 113 | var payload []byte 114 | var doc *birch.Document 115 | var err error 116 | 117 | for iter.chunks.Next() { 118 | chunk := iter.chunks.Chunk() 119 | 120 | if iter.reflect { 121 | payload, err = bson.Marshal(chunk.exportMatrix()) 122 | if err != nil { 123 | iter.catcher.Add(err) 124 | return 125 | } 126 | doc, err = birch.ReadDocument(payload) 127 | if err != nil { 128 | iter.catcher.Add(err) 129 | return 130 | } 131 | } else { 132 | doc, err = chunk.export() 133 | if err != nil { 134 | iter.catcher.Add(err) 135 | return 136 | } 137 | } 138 | 139 | select { 140 | case iter.pipe <- doc: 141 | continue 142 | case <-ctx.Done(): 143 | iter.catcher.Add(errors.New("operation aborted")) 144 | return 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /iterator_sample.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/evergreen-ci/birch" 7 | ) 8 | 9 | // sampleIterator provides an iterator for iterating through the 10 | // results of a FTDC data chunk as BSON documents. 11 | type sampleIterator struct { 12 | closer context.CancelFunc 13 | stream <-chan *birch.Document 14 | sample *birch.Document 15 | metadata *birch.Document 16 | } 17 | 18 | func (c *Chunk) streamFlattenedDocuments(ctx context.Context) <-chan *birch.Document { 19 | out := make(chan *birch.Document, 100) 20 | 21 | go func() { 22 | defer close(out) 23 | for i := 0; i < c.nPoints; i++ { 24 | 25 | doc := birch.DC.Make(len(c.Metrics)) 26 | for _, m := range c.Metrics { 27 | elem, ok := restoreFlat(m.originalType, m.Key(), m.Values[i]) 28 | if !ok { 29 | continue 30 | } 31 | 32 | doc.Append(elem) 33 | } 34 | 35 | select { 36 | case out <- doc: 37 | continue 38 | case <-ctx.Done(): 39 | return 40 | } 41 | } 42 | }() 43 | 44 | return out 45 | } 46 | 47 | func (c *Chunk) streamDocuments(ctx context.Context) <-chan *birch.Document { 48 | out := make(chan *birch.Document, 100) 49 | 50 | go func() { 51 | defer close(out) 52 | 53 | for i := 0; i < c.nPoints; i++ { 54 | doc, _ := restoreDocument(c.reference, i, c.Metrics, 0) 55 | select { 56 | case <-ctx.Done(): 57 | return 58 | case out <- doc: 59 | continue 60 | } 61 | } 62 | }() 63 | 64 | return out 65 | } 66 | 67 | // Close releases all resources associated with the iterator. 68 | func (iter *sampleIterator) Close() { iter.closer() } 69 | func (iter *sampleIterator) Err() error { return nil } 70 | 71 | func (iter *sampleIterator) Metadata() *birch.Document { return iter.metadata } 72 | 73 | // Document returns the current document in the iterator. It is safe 74 | // to call this method more than once, and the result will only be nil 75 | // before the iterator is advanced. 76 | func (iter *sampleIterator) Document() *birch.Document { return iter.sample } 77 | 78 | // Next advances the iterator one document. Returns true when there is 79 | // a document, and false otherwise. 80 | func (iter *sampleIterator) Next() bool { 81 | doc, ok := <-iter.stream 82 | if !ok { 83 | return false 84 | } 85 | 86 | iter.sample = doc 87 | return true 88 | } 89 | -------------------------------------------------------------------------------- /iterator_sample_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestSampleIterator(t *testing.T) { 11 | t.Run("CanceledContextCreator", func(t *testing.T) { 12 | ctx, cancel := context.WithCancel(context.Background()) 13 | cancel() 14 | chunk := &Chunk{ 15 | nPoints: 2, 16 | } 17 | out := chunk.streamDocuments(ctx) 18 | assert.NotNil(t, out) 19 | for { 20 | doc, ok := <-out 21 | if ok { 22 | continue 23 | } 24 | 25 | assert.False(t, ok) 26 | assert.Nil(t, doc) 27 | break 28 | } 29 | 30 | }) 31 | t.Run("CloserOperations", func(t *testing.T) { 32 | iter := &sampleIterator{} 33 | assert.Panics(t, func() { 34 | iter.Close() 35 | }) 36 | counter := 0 37 | iter.closer = func() { counter++ } 38 | assert.NotPanics(t, func() { 39 | iter.Close() 40 | }) 41 | assert.Equal(t, 1, counter) 42 | 43 | }) 44 | 45 | } 46 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | # start project configuration 2 | name := ftdc 3 | buildDir := build 4 | packages := $(name) events hdrhist metrics util 5 | srcFiles := makefile $(shell find . -name "*.go" -not -path "./$(buildDir)/*" -not -name "*_test.go" -not -path "./scripts/*" -not -path "*\#*") 6 | testSrcFiles := makefile $(shell find . -name "*.go" -not -path "./$(buildDir)/*" -not -path "*\#*") 7 | orgPath := github.com/mongodb 8 | projectPath := $(orgPath)/$(name) 9 | # end project configuration 10 | 11 | # start environment setup 12 | gobin := go 13 | ifneq (,$(GOROOT)) 14 | gobin := $(GOROOT)/bin/go 15 | endif 16 | 17 | goCache := $(GOCACHE) 18 | ifeq (,$(goCache)) 19 | goCache := $(abspath $(buildDir)/.cache) 20 | endif 21 | goModCache := $(GOMODCACHE) 22 | ifeq (,$(goModCache)) 23 | goModCache := $(abspath $(buildDir)/.mod-cache) 24 | endif 25 | lintCache := $(GOLANGCI_LINT_CACHE) 26 | ifeq (,$(lintCache)) 27 | lintCache := $(abspath $(buildDir)/.lint-cache) 28 | endif 29 | 30 | ifeq ($(OS),Windows_NT) 31 | gobin := $(shell cygpath $(gobin)) 32 | goCache := $(shell cygpath -m $(goCache)) 33 | goModCache := $(shell cygpath -m $(goModCache)) 34 | lintCache := $(shell cygpath -m $(lintCache)) 35 | export GOROOT := $(shell cygpath -m $(GOROOT)) 36 | endif 37 | 38 | ifneq ($(goCache),$(GOCACHE)) 39 | export GOCACHE := $(goCache) 40 | endif 41 | ifneq ($(goModCache),$(GOMODCACHE)) 42 | export GOMODCACHE := $(goModCache) 43 | endif 44 | ifneq ($(lintCache),$(GOLANGCI_LINT_CACHE)) 45 | export GOLANGCI_LINT_CACHE := $(lintCache) 46 | endif 47 | 48 | ifneq (,$(RACE_DETECTOR)) 49 | # cgo is required for using the race detector. 50 | export CGO_ENABLED := 1 51 | else 52 | export CGO_ENABLED := 0 53 | endif 54 | # end environment setup 55 | 56 | # Ensure the build directory exists, since most targets require it. 57 | $(shell mkdir -p $(buildDir)) 58 | 59 | .DEFAULT_GOAL := compile 60 | 61 | # start test data download targets 62 | metrics.ftdc: 63 | curl -LO "https://ftdc-test-files.s3.amazonaws.com/metrics.ftdc" 64 | perf_metrics.ftdc: 65 | curl -LO "https://ftdc-test-files.s3.amazonaws.com/perf_metrics.ftdc" 66 | genny_metrics.ftdc: 67 | curl -LO "https://ftdc-test-files.s3.amazonaws.com/genny_metrics.ftdc" 68 | $(buildDir)/output.ftdc.test: perf_metrics.ftdc metrics.ftdc genny_metrics.ftdc 69 | # end test data file download targets 70 | 71 | # start lint setup targets 72 | $(buildDir)/golangci-lint: 73 | @curl --retry 10 --retry-max-time 60 -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(buildDir) v1.64.5 >/dev/null 2>&1 74 | $(buildDir)/run-linter: cmd/run-linter/run-linter.go $(buildDir)/golangci-lint 75 | $(gobin) build -o $@ $< 76 | # end lint setup targets 77 | 78 | # start output files 79 | testOutput := $(foreach target,$(packages),$(buildDir)/output.$(target).test) 80 | lintOutput := $(foreach target,$(packages),$(buildDir)/output.$(target).lint) 81 | coverageOutput := $(foreach target,$(packages),$(buildDir)/output.$(target).coverage) 82 | htmlCoverageOutput := $(foreach target,$(packages),$(buildDir)/output.$(target).coverage.html) 83 | .PRECIOUS: $(testOutput) $(lintOutput) $(coverageOutput) $(htmlCoverageOutput) 84 | # end output files 85 | 86 | # start basic development operations 87 | compile: $(srcFiles) 88 | $(gobin) build $(subst $(name),,$(subst -,/,$(foreach pkg,$(packages),./$(pkg)))) 89 | test: $(testOutput) 90 | lint: $(lintOutput) 91 | coverage: $(coverageOutput) 92 | html-coverage: $(htmlCoverageOutput) 93 | phony += compile lint test coverage html-coverage 94 | 95 | # start convenience targets for running tests and coverage tasks on a 96 | # specific package. 97 | test-%: $(buildDir)/output.%.test 98 | 99 | coverage-%: $(buildDir)/output.%.coverage 100 | 101 | html-coverage-%: $(buildDir)/output.%.coverage $(buildDir)/output.%.coverage.html 102 | @grep -s -q -e "^PASS" $(subst coverage,test,$<) 103 | lint-%: $(buildDir)/output.%.lint 104 | @grep -v -s -q "^--- FAIL" $< 105 | # end convenience targets 106 | # end basic development operations 107 | 108 | # start test and coverage artifacts 109 | testArgs := -v 110 | ifneq (,$(RUN_TEST)) 111 | testArgs += -run='$(RUN_TEST)' 112 | endif 113 | ifneq (,$(RUN_COUNT)) 114 | testArgs += -count=$(RUN_COUNT) 115 | endif 116 | ifneq (,$(SKIP_LONG)) 117 | testArgs += -short 118 | endif 119 | ifneq (,$(RACE_DETECTOR)) 120 | testArgs += -race 121 | endif 122 | ifneq (,$(TEST_TIMEOUT)) 123 | testArgs += -timeout=$(TEST_TIMEOUT) 124 | else 125 | testArgs += -timeout=30m 126 | endif 127 | $(buildDir)/output.%.test: .FORCE 128 | $(gobin) test $(testArgs) ./$(if $(subst $(name),,$*),$(subst -,/,$*),) | tee $@ 129 | @grep -s -q -e "^PASS" $@ 130 | $(buildDir)/output.%.coverage: .FORCE 131 | $(gobin) test $(testArgs) ./$(if $(subst $(name),,$*),$(subst -,/,$*),) -covermode=count -coverprofile $@ | tee $(buildDir)/output.$*.test 132 | @-[ -f $@ ] && $(gobin) tool cover -func=$@ | sed 's%$(projectPath)/%%' | column -t 133 | @grep -s -q -e "^PASS" $(subst coverage,test,$@) 134 | $(buildDir)/output.%.coverage.html: $(buildDir)/output.%.coverage 135 | $(gobin) tool cover -html=$< -o $@ 136 | 137 | ifneq (go,$(gobin)) 138 | # We have to handle the PATH specially for linting in CI, because if the PATH has a different version of the Go 139 | # binary in it, the linter won't work properly. 140 | lintEnvVars := PATH="$(shell dirname $(gobin)):$(PATH)" 141 | endif 142 | $(buildDir)/output.%.lint: $(buildDir)/run-linter .FORCE 143 | @$(lintEnvVars) ./$< --output=$@ --lintBin=$(buildDir)/golangci-lint --packages='$*' 144 | # end test and coverage artifacts 145 | 146 | # start module management targets 147 | mod-tidy: 148 | $(gobin) mod tidy 149 | # Check if go.mod and go.sum are clean. If they're clean, then mod tidy should not produce a different result. 150 | verify-mod-tidy: 151 | $(gobin) run cmd/verify-mod-tidy/verify-mod-tidy.go -goBin="$(gobin)" 152 | phony += mod-tidy verify-mod-tidy 153 | # end module management targets 154 | 155 | # start cleanup targets 156 | clean: 157 | rm -rf $(buildDir) 158 | clean-results: 159 | rm -rf $(buildDir)/output.* 160 | phony += clean clean-results 161 | # end cleanup targets 162 | 163 | # configure phony targets 164 | .FORCE: 165 | .PHONY: $(phony) .FORCE 166 | -------------------------------------------------------------------------------- /metrics/json.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "time" 11 | 12 | "github.com/evergreen-ci/birch" 13 | "github.com/mongodb/ftdc" 14 | "github.com/papertrail/go-tail/follower" 15 | "github.com/pkg/errors" 16 | "go.mongodb.org/mongo-driver/v2/bson" 17 | ) 18 | 19 | // CollectJSONOptions specifies options for a JSON2FTDC collector. You 20 | // must specify EITHER an input Source as a reader or a file 21 | // name. 22 | type CollectJSONOptions struct { 23 | OutputFilePrefix string 24 | SampleCount int 25 | FlushInterval time.Duration 26 | InputSource io.Reader `json:"-"` 27 | FileName string 28 | Follow bool 29 | } 30 | 31 | func (opts CollectJSONOptions) validate() error { 32 | bothSpecified := (opts.InputSource == nil && opts.FileName == "") 33 | neitherSpecifed := (opts.InputSource != nil && opts.FileName != "") 34 | 35 | if bothSpecified || neitherSpecifed { 36 | return errors.New("must specify exactly one of input source and filename") 37 | } 38 | 39 | if opts.Follow && opts.FileName == "" { 40 | return errors.New("follow option must not be specified with a file reader") 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func (opts CollectJSONOptions) getSource() (<-chan *birch.Document, <-chan error) { 47 | out := make(chan *birch.Document) 48 | errs := make(chan error, 2) 49 | 50 | switch { 51 | case opts.InputSource != nil: 52 | go func() { 53 | stream := bufio.NewScanner(opts.InputSource) 54 | defer close(errs) 55 | 56 | for stream.Scan() { 57 | doc := &birch.Document{} 58 | err := bson.UnmarshalExtJSON(stream.Bytes(), false, doc) 59 | if err != nil { 60 | errs <- err 61 | return 62 | } 63 | out <- doc 64 | } 65 | }() 66 | case opts.FileName != "" && !opts.Follow: 67 | go func() { 68 | defer close(errs) 69 | f, err := os.Open(opts.FileName) 70 | if err != nil { 71 | errs <- errors.Wrapf(err, "problem opening data file %s", opts.FileName) 72 | return 73 | } 74 | defer func() { errs <- f.Close() }() 75 | stream := bufio.NewScanner(f) 76 | 77 | for stream.Scan() { 78 | doc := &birch.Document{} 79 | err := bson.UnmarshalExtJSON(stream.Bytes(), false, doc) 80 | if err != nil { 81 | errs <- err 82 | return 83 | } 84 | out <- doc 85 | } 86 | }() 87 | case opts.FileName != "" && opts.Follow: 88 | go func() { 89 | defer close(errs) 90 | 91 | tail, err := follower.New(opts.FileName, follower.Config{ 92 | Reopen: true, 93 | }) 94 | if err != nil { 95 | errs <- errors.Wrapf(err, "problem setting up file follower of '%s'", opts.FileName) 96 | return 97 | } 98 | defer func() { 99 | tail.Close() 100 | errs <- tail.Err() 101 | }() 102 | 103 | for line := range tail.Lines() { 104 | doc := birch.NewDocument() 105 | err := bson.UnmarshalExtJSON([]byte(line.String()), false, doc) 106 | if err != nil { 107 | errs <- err 108 | return 109 | } 110 | out <- doc 111 | } 112 | }() 113 | default: 114 | errs <- errors.New("invalid collect options") 115 | close(errs) 116 | } 117 | return out, errs 118 | } 119 | 120 | // CollectJSONStream provides a blocking process that reads new-line 121 | // separated JSON documents from a file and creates FTDC data from 122 | // these sources. 123 | // 124 | // The Options structure allows you to define the collection intervals 125 | // and also specify the source. The collector supports reading 126 | // directly from an arbitrary IO reader, or from a file. The "follow" 127 | // option allows you to watch the end of a file for new JSON 128 | // documents, a la "tail -f". 129 | func CollectJSONStream(ctx context.Context, opts CollectJSONOptions) ([]byte, error) { 130 | if err := opts.validate(); err != nil { 131 | return nil, errors.WithStack(err) 132 | } 133 | 134 | outputCount := 0 135 | collector := ftdc.NewDynamicCollector(opts.SampleCount) 136 | flushTimer := time.NewTimer(opts.FlushInterval) 137 | defer flushTimer.Stop() 138 | 139 | flusher := func() ([]byte, error) { 140 | defer flushTimer.Reset(opts.FlushInterval) 141 | info := collector.Info() 142 | 143 | if info.SampleCount == 0 { 144 | flushTimer.Reset(opts.FlushInterval) 145 | return []byte{}, nil 146 | } 147 | 148 | output, err := collector.Resolve() 149 | defer collector.Reset() 150 | if err != nil { 151 | return nil, errors.Wrap(err, "problem resolving ftdc data") 152 | } 153 | 154 | if opts.OutputFilePrefix == "" { 155 | return output, nil 156 | } else { 157 | fn := fmt.Sprintf("%s.%d", opts.OutputFilePrefix, outputCount) 158 | if err = ioutil.WriteFile(fn, output, 0600); err != nil { 159 | return nil, errors.Wrapf(err, "problem writing data to file %s", fn) 160 | } 161 | } 162 | 163 | outputCount++ 164 | return []byte{}, nil 165 | } 166 | 167 | docs, errs := opts.getSource() 168 | 169 | for { 170 | select { 171 | case <-ctx.Done(): 172 | return nil, errors.New("operation aborted") 173 | case err := <-errs: 174 | if err == nil || errors.Cause(err) == io.EOF { 175 | var output []byte 176 | output, err = flusher() 177 | return output, errors.Wrap(err, "problem flushing results at the end of the file") 178 | } 179 | return nil, errors.WithStack(err) 180 | case doc := <-docs: 181 | if err := collector.Add(doc); err != nil { 182 | return nil, errors.Wrap(err, "problem collecting results") 183 | } 184 | case <-flushTimer.C: 185 | output, err := flusher() 186 | return output, errors.Wrap(err, "problem flushing results") 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /metrics/json_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "testing" 13 | "time" 14 | 15 | "github.com/mongodb/ftdc" 16 | "github.com/mongodb/ftdc/testutil" 17 | "github.com/pkg/errors" 18 | "github.com/stretchr/testify/assert" 19 | "github.com/stretchr/testify/require" 20 | ) 21 | 22 | func TestCollectJSONOptions(t *testing.T) { 23 | for _, test := range []struct { 24 | name string 25 | valid bool 26 | opts CollectJSONOptions 27 | }{ 28 | { 29 | name: "Nil", 30 | valid: false, 31 | }, 32 | { 33 | name: "FileWithIoReader", 34 | valid: false, 35 | opts: CollectJSONOptions{ 36 | FileName: "foo", 37 | InputSource: &bytes.Buffer{}, 38 | }, 39 | }, 40 | { 41 | name: "JustIoReader", 42 | valid: true, 43 | opts: CollectJSONOptions{ 44 | InputSource: &bytes.Buffer{}, 45 | }, 46 | }, 47 | { 48 | name: "JustFile", 49 | valid: true, 50 | opts: CollectJSONOptions{ 51 | FileName: "foo", 52 | }, 53 | }, 54 | { 55 | name: "FileWithFollow", 56 | valid: true, 57 | opts: CollectJSONOptions{ 58 | FileName: "foo", 59 | Follow: true, 60 | }, 61 | }, 62 | { 63 | name: "ReaderWithFollow", 64 | valid: false, 65 | opts: CollectJSONOptions{ 66 | InputSource: &bytes.Buffer{}, 67 | Follow: true, 68 | }, 69 | }, 70 | } { 71 | t.Run(test.name, func(t *testing.T) { 72 | if test.valid { 73 | assert.NoError(t, test.opts.validate()) 74 | } else { 75 | assert.Error(t, test.opts.validate()) 76 | } 77 | }) 78 | } 79 | } 80 | 81 | func makeJSONRandComplex(num int) ([][]byte, error) { 82 | out := [][]byte{} 83 | 84 | for i := 0; i < num; i++ { 85 | doc := testutil.RandComplexDocument(100, 2) 86 | data, err := json.Marshal(doc) 87 | if err != nil { 88 | return nil, errors.WithStack(err) 89 | } 90 | out = append(out, data) 91 | } 92 | 93 | return out, nil 94 | } 95 | 96 | func writeStream(docs [][]byte, writer io.Writer) error { 97 | for _, doc := range docs { 98 | _, err := writer.Write(doc) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | _, err = writer.Write([]byte("\n")) 104 | if err != nil { 105 | return err 106 | } 107 | } 108 | return nil 109 | } 110 | 111 | func TestCollectJSON(t *testing.T) { 112 | buildDir, err := filepath.Abs(filepath.Join("..", "build")) 113 | require.NoError(t, err) 114 | dir, err := ioutil.TempDir(buildDir, "ftdc-") 115 | require.NoError(t, err) 116 | 117 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 118 | defer cancel() 119 | defer func() { 120 | if err = os.RemoveAll(dir); err != nil { 121 | fmt.Println(err) 122 | } 123 | }() 124 | 125 | hundredDocs, err := makeJSONRandComplex(100) 126 | require.NoError(t, err) 127 | 128 | t.Run("SingleReaderIdealCase", func(t *testing.T) { 129 | buf := &bytes.Buffer{} 130 | err = writeStream(hundredDocs, buf) 131 | require.NoError(t, err) 132 | 133 | reader := bytes.NewReader(buf.Bytes()) 134 | 135 | opts := CollectJSONOptions{ 136 | OutputFilePrefix: filepath.Join(dir, fmt.Sprintf("json.%d.%s", 137 | os.Getpid(), 138 | time.Now().Format("2006-01-02.15-04-05"))), 139 | FlushInterval: 100 * time.Millisecond, 140 | SampleCount: 1000, 141 | InputSource: reader, 142 | } 143 | 144 | _, err = CollectJSONStream(ctx, opts) 145 | assert.NoError(t, err) 146 | }) 147 | t.Run("SingleReaderBotchedDocument", func(t *testing.T) { 148 | buf := &bytes.Buffer{} 149 | 150 | var docs [][]byte 151 | docs, err = makeJSONRandComplex(10) 152 | require.NoError(t, err) 153 | 154 | docs[2] = docs[len(docs)-1][1:] // break the last document 155 | 156 | err = writeStream(docs, buf) 157 | require.NoError(t, err) 158 | 159 | reader := bytes.NewReader(buf.Bytes()) 160 | 161 | opts := CollectJSONOptions{ 162 | OutputFilePrefix: filepath.Join(dir, fmt.Sprintf("json.%d.%s", 163 | os.Getpid(), 164 | time.Now().Format("2006-01-02.15-04-05"))), 165 | FlushInterval: 10 * time.Millisecond, 166 | InputSource: reader, 167 | SampleCount: 100, 168 | } 169 | 170 | _, err = CollectJSONStream(ctx, opts) 171 | assert.Error(t, err) 172 | }) 173 | t.Run("ReadFromFile", func(t *testing.T) { 174 | fn := filepath.Join(dir, "json-read-file-one") 175 | var f *os.File 176 | f, err = os.Create(fn) 177 | require.NoError(t, err) 178 | 179 | require.NoError(t, writeStream(hundredDocs, f)) 180 | require.NoError(t, f.Close()) 181 | 182 | opts := CollectJSONOptions{ 183 | OutputFilePrefix: filepath.Join(dir, fmt.Sprintf("json.%d.%s", 184 | os.Getpid(), 185 | time.Now().Format("2006-01-02.15-04-05"))), 186 | FileName: fn, 187 | SampleCount: 100, 188 | } 189 | 190 | _, err = CollectJSONStream(ctx, opts) 191 | assert.NoError(t, err) 192 | }) 193 | t.Run("NoOutputFilePrefix", func(t *testing.T) { 194 | fn := filepath.Join(dir, "json-read-file-two") 195 | var f *os.File 196 | f, err = os.Create(fn) 197 | require.NoError(t, err) 198 | 199 | require.NoError(t, writeStream(hundredDocs, f)) 200 | require.NoError(t, f.Close()) 201 | 202 | opts := CollectJSONOptions{ 203 | FileName: fn, 204 | SampleCount: 100, 205 | FlushInterval: 10 * time.Second, 206 | } 207 | 208 | var output []byte 209 | output, err = CollectJSONStream(ctx, opts) 210 | 211 | iter := ftdc.ReadMetrics(ctx, bytes.NewReader(output)) 212 | i := 0 213 | for iter.Next() { 214 | i++ 215 | } 216 | 217 | assert.Equal(t, 100, i) 218 | assert.NoError(t, err) 219 | }) 220 | t.Run("FollowFile", func(t *testing.T) { 221 | fn := filepath.Join(dir, "json-read-file-three") 222 | var f *os.File 223 | f, err = os.Create(fn) 224 | require.NoError(t, err) 225 | 226 | go func() { 227 | time.Sleep(10 * time.Millisecond) 228 | require.NoError(t, writeStream(hundredDocs, f)) 229 | require.NoError(t, f.Close()) 230 | }() 231 | 232 | ctx, cancel = context.WithTimeout(ctx, 250*time.Millisecond) 233 | defer cancel() 234 | opts := CollectJSONOptions{ 235 | OutputFilePrefix: filepath.Join(dir, fmt.Sprintf("json.%d.%s", 236 | os.Getpid(), 237 | time.Now().Format("2006-01-02.15-04-05"))), 238 | SampleCount: 100, 239 | FlushInterval: 500 * time.Millisecond, 240 | FileName: fn, 241 | Follow: true, 242 | } 243 | 244 | var output []byte 245 | output, err = CollectJSONStream(ctx, opts) 246 | assert.Nil(t, output) 247 | assert.Error(t, err) 248 | assert.Contains(t, err.Error(), "operation aborted") 249 | }) 250 | t.Run("RoundTrip", func(t *testing.T) { 251 | inputs := []map[string]interface{}{ 252 | { 253 | "one": int64(1), 254 | "other": int64(43), 255 | }, 256 | { 257 | "one": int64(33), 258 | "other": int64(41), 259 | }, 260 | { 261 | "one": int64(1), 262 | "other": int64(41), 263 | }, 264 | } 265 | 266 | var ( 267 | doc []byte 268 | docs [][]byte 269 | ) 270 | 271 | for _, in := range inputs { 272 | doc, err = json.Marshal(in) 273 | require.NoError(t, err) 274 | docs = append(docs, doc) 275 | } 276 | require.Len(t, docs, 3) 277 | 278 | buf := &bytes.Buffer{} 279 | 280 | require.NoError(t, writeStream(docs, buf)) 281 | 282 | reader := bytes.NewReader(buf.Bytes()) 283 | 284 | opts := CollectJSONOptions{ 285 | OutputFilePrefix: filepath.Join(dir, "roundtrip"), 286 | FlushInterval: time.Second, 287 | SampleCount: 50, 288 | InputSource: reader, 289 | } 290 | ctx := context.Background() 291 | 292 | output, err := CollectJSONStream(ctx, opts) 293 | assert.Equal(t, []byte{}, output) 294 | assert.NoError(t, err) 295 | _, err = os.Stat(filepath.Join(dir, "roundtrip.0")) 296 | require.False(t, os.IsNotExist(err)) 297 | 298 | fn, err := os.Open(filepath.Join(dir, "roundtrip.0")) 299 | require.NoError(t, err) 300 | 301 | iter := ftdc.ReadMetrics(ctx, fn) 302 | idx := -1 303 | for iter.Next() { 304 | idx++ 305 | 306 | s := iter.Document() 307 | assert.Equal(t, 2, s.Len()) 308 | for k, v := range inputs[idx] { 309 | out := s.Lookup(k) 310 | assert.EqualValues(t, v, out.Interface()) 311 | } 312 | } 313 | require.NoError(t, iter.Err()) 314 | assert.Equal(t, 2, idx) // zero indexed 315 | }) 316 | } 317 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | // Package metrics includes data types used for Golang runtime and 2 | // system metrics collection 3 | package metrics 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "os" 9 | "runtime" 10 | "sort" 11 | "sync" 12 | "time" 13 | 14 | "github.com/evergreen-ci/birch" 15 | "github.com/mongodb/ftdc" 16 | "github.com/mongodb/ftdc/util" 17 | "github.com/mongodb/grip/message" 18 | "github.com/mongodb/grip/recovery" 19 | "github.com/pkg/errors" 20 | "go.mongodb.org/mongo-driver/v2/bson" 21 | ) 22 | 23 | // Runtime provides an aggregated view for 24 | type Runtime struct { 25 | ID int `json:"id" bson:"id"` 26 | Timestamp time.Time `json:"ts" bson:"ts"` 27 | PID int `json:"pid" bson:"pid"` 28 | Golang *message.GoRuntimeInfo `json:"golang,omitempty" bson:"golang,omitempty"` 29 | System *message.SystemInfo `json:"system,omitempty" bson:"system,omitempty"` 30 | Process *message.ProcessInfo `json:"process,omitempty" bson:"process,omitempty"` 31 | } 32 | 33 | // CollectOptions are the settings to provide the behavior of 34 | // the collection process process. 35 | type CollectOptions struct { 36 | FlushInterval time.Duration 37 | CollectionInterval time.Duration 38 | SkipGolang bool 39 | SkipSystem bool 40 | SkipProcess bool 41 | RunParallelCollectors bool 42 | SampleCount int 43 | Collectors Collectors 44 | OutputFilePrefix string 45 | } 46 | 47 | type Collectors []CustomCollector 48 | 49 | func (c Collectors) Len() int { return len(c) } 50 | func (c Collectors) Swap(i, j int) { c[i], c[j] = c[j], c[i] } 51 | func (c Collectors) Less(i, j int) bool { return c[i].Name < c[j].Name } 52 | 53 | type CustomCollector struct { 54 | Name string 55 | Operation func(context.Context) *birch.Document 56 | } 57 | 58 | func (opts *CollectOptions) generate(ctx context.Context, id int) *birch.Document { 59 | pid := os.Getpid() 60 | out := &Runtime{ 61 | ID: id, 62 | PID: pid, 63 | Timestamp: time.Now(), 64 | } 65 | 66 | base := message.Base{} 67 | 68 | if !opts.SkipGolang { 69 | out.Golang = message.CollectGoStatsTotals().(*message.GoRuntimeInfo) 70 | out.Golang.Base = base 71 | } 72 | 73 | if !opts.SkipSystem { 74 | out.System = message.CollectSystemInfo().(*message.SystemInfo) 75 | out.System.Base = base 76 | } 77 | 78 | if !opts.SkipProcess { 79 | out.Process = message.CollectProcessInfo(int32(pid)).(*message.ProcessInfo) 80 | out.Process.Base = base 81 | } 82 | 83 | docb, err := bson.Marshal(out) 84 | if err != nil { 85 | panic(err) 86 | } 87 | 88 | if len(opts.Collectors) == 0 { 89 | return birch.DC.Reader(docb) 90 | } 91 | 92 | doc := birch.DC.Make(len(opts.Collectors) + 1).Append(birch.EC.SubDocument("runtime", birch.DC.Reader(docb))) 93 | if !opts.RunParallelCollectors { 94 | for _, ec := range opts.Collectors { 95 | doc.Append(birch.EC.SubDocument(ec.Name, ec.Operation(ctx))) 96 | } 97 | 98 | return doc 99 | } 100 | 101 | collectors := make(chan CustomCollector, len(opts.Collectors)) 102 | elems := make(chan *birch.Element, len(opts.Collectors)) 103 | num := runtime.NumCPU() 104 | if num > len(opts.Collectors) { 105 | num = len(opts.Collectors) 106 | } 107 | 108 | for _, coll := range opts.Collectors { 109 | collectors <- coll 110 | } 111 | close(collectors) 112 | 113 | wg := &sync.WaitGroup{} 114 | for i := 0; i < num; i++ { 115 | wg.Add(1) 116 | go func() { 117 | defer recovery.LogStackTraceAndContinue("ftdc metrics collector") 118 | defer wg.Done() 119 | 120 | for collector := range collectors { 121 | elems <- birch.EC.SubDocument(collector.Name, collector.Operation(ctx)) 122 | } 123 | }() 124 | } 125 | wg.Wait() 126 | 127 | for elem := range elems { 128 | doc.Append(elem) 129 | } 130 | 131 | return doc.Sorted() 132 | } 133 | 134 | // NewCollectOptions creates a valid, populated collection options 135 | // structure, collecting data every minute, rotating files every 24 136 | // hours. 137 | func NewCollectOptions(prefix string) CollectOptions { 138 | return CollectOptions{ 139 | OutputFilePrefix: prefix, 140 | SampleCount: 300, 141 | FlushInterval: 24 * time.Hour, 142 | CollectionInterval: time.Second, 143 | } 144 | } 145 | 146 | // Validate checks the Collect option settings and ensures that all 147 | // values are reasonable. 148 | func (opts CollectOptions) Validate() error { 149 | catcher := util.NewCatcher() 150 | 151 | sort.Stable(opts.Collectors) 152 | 153 | catcher.NewWhen(opts.FlushInterval < time.Millisecond, 154 | "flush interval must be greater than a millisecond") 155 | catcher.NewWhen(opts.CollectionInterval < time.Millisecond, 156 | "collection interval must be greater than a millisecond") 157 | catcher.NewWhen(opts.CollectionInterval > opts.FlushInterval, 158 | "collection interval must be smaller than flush interval") 159 | catcher.NewWhen(opts.SampleCount < 10, "sample count must be at least 10") 160 | catcher.NewWhen(opts.SkipGolang && opts.SkipProcess && opts.SkipSystem, 161 | "cannot skip all metrics collection, must specify golang, process, or system") 162 | catcher.NewWhen(opts.RunParallelCollectors && len(opts.Collectors) == 0, 163 | "cannot run parallel collectors with no collectors specified") 164 | 165 | return catcher.Resolve() 166 | } 167 | 168 | // CollectRuntime starts a blocking background process that that 169 | // collects metrics about the current process, the go runtime, and the 170 | // underlying system. 171 | func CollectRuntime(ctx context.Context, opts CollectOptions) error { 172 | if err := opts.Validate(); err != nil { 173 | return err 174 | } 175 | 176 | outputCount := 0 177 | collectCount := 0 178 | 179 | file, err := os.Create(fmt.Sprintf("%s.%d", opts.OutputFilePrefix, outputCount)) 180 | if err != nil { 181 | return errors.Wrap(err, "problem creating initial file") 182 | } 183 | 184 | collector := ftdc.NewStreamingCollector(opts.SampleCount, file) 185 | collectTimer := time.NewTimer(0) 186 | flushTimer := time.NewTimer(opts.FlushInterval) 187 | defer collectTimer.Stop() 188 | defer flushTimer.Stop() 189 | 190 | flusher := func() error { 191 | info := collector.Info() 192 | if info.SampleCount == 0 { 193 | return nil 194 | } 195 | 196 | if err = ftdc.FlushCollector(collector, file); err != nil { 197 | return errors.WithStack(err) 198 | } 199 | 200 | if err = file.Close(); err != nil { 201 | return errors.WithStack(err) 202 | } 203 | 204 | outputCount++ 205 | 206 | file, err = os.Create(fmt.Sprintf("%s.%d", opts.OutputFilePrefix, outputCount)) 207 | if err != nil { 208 | return errors.Wrap(err, "problem creating subsequent file") 209 | } 210 | 211 | collector = ftdc.NewStreamingCollector(opts.SampleCount, file) 212 | return nil 213 | } 214 | 215 | for { 216 | select { 217 | case <-ctx.Done(): 218 | return errors.WithStack(flusher()) 219 | case <-collectTimer.C: 220 | if err := collector.Add(opts.generate(ctx, collectCount)); err != nil { 221 | return errors.Wrap(err, "problem collecting results") 222 | } 223 | collectCount++ 224 | collectTimer.Reset(opts.CollectionInterval) 225 | case <-flushTimer.C: 226 | if err := flusher(); err != nil { 227 | return errors.WithStack(err) 228 | } 229 | flushTimer.Reset(opts.FlushInterval) 230 | } 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | "github.com/mongodb/ftdc" 15 | "github.com/stretchr/testify/assert" 16 | "github.com/stretchr/testify/require" 17 | ) 18 | 19 | func GetDirectoryOfFile() string { 20 | _, file, _, _ := runtime.Caller(1) 21 | 22 | return filepath.Dir(file) 23 | } 24 | 25 | func TestCollectRuntime(t *testing.T) { 26 | dir, err := ioutil.TempDir(filepath.Join(filepath.Dir(GetDirectoryOfFile()), "build"), "ftdc-") 27 | require.NoError(t, err) 28 | 29 | defer func() { 30 | if err = os.RemoveAll(dir); err != nil { 31 | fmt.Println(err) 32 | } 33 | }() 34 | 35 | t.Run("CollectData", func(t *testing.T) { 36 | opts := CollectOptions{ 37 | OutputFilePrefix: filepath.Join(dir, fmt.Sprintf("sysinfo.%d.%s", 38 | os.Getpid(), 39 | time.Now().Format("2006-01-02.15-04-05"))), 40 | SampleCount: 10, 41 | FlushInterval: time.Second, 42 | CollectionInterval: time.Millisecond, 43 | SkipProcess: true, 44 | SkipSystem: true, 45 | } 46 | var cancel context.CancelFunc 47 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 48 | defer cancel() 49 | 50 | err = CollectRuntime(ctx, opts) 51 | require.NoError(t, err) 52 | }) 53 | t.Run("ReadData", func(t *testing.T) { 54 | ctx, cancel := context.WithCancel(context.Background()) 55 | defer cancel() 56 | 57 | var files []os.FileInfo 58 | files, err = ioutil.ReadDir(dir) 59 | require.NoError(t, err) 60 | assert.True(t, len(files) >= 1) 61 | 62 | total := 0 63 | for idx, info := range files { 64 | t.Run(fmt.Sprintf("FileNo.%d", idx), func(t *testing.T) { 65 | path := filepath.Join(dir, info.Name()) 66 | var f *os.File 67 | f, err = os.Open(path) 68 | require.NoError(t, err) 69 | defer func() { assert.NoError(t, f.Close()) }() 70 | iter := ftdc.ReadMetrics(ctx, f) 71 | counter := 0 72 | for iter.Next() { 73 | counter++ 74 | doc := iter.Document() 75 | assert.NotNil(t, doc) 76 | require.Equal(t, 3, doc.Len()) 77 | } 78 | require.NoError(t, iter.Err()) 79 | total += counter 80 | }) 81 | } 82 | }) 83 | t.Run("CollectAllData", func(t *testing.T) { 84 | if strings.Contains(os.Getenv("EVR_TASK_ID"), "race") { 85 | t.Skip("evergreen environment inconsistent") 86 | } 87 | // this test runs without the skips, which are 88 | // expected to be less reliable in different environment 89 | opts := CollectOptions{ 90 | OutputFilePrefix: filepath.Join(dir, fmt.Sprintf("complete.%d.%s", 91 | os.Getpid(), 92 | time.Now().Format("2006-01-02.15-04-05"))), 93 | SampleCount: 100, 94 | FlushInterval: time.Second, 95 | CollectionInterval: time.Millisecond, 96 | } 97 | var cancel context.CancelFunc 98 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 99 | defer cancel() 100 | 101 | err = CollectRuntime(ctx, opts) 102 | require.NoError(t, err) 103 | }) 104 | } 105 | -------------------------------------------------------------------------------- /read.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "compress/zlib" 7 | "context" 8 | "encoding/binary" 9 | "io" 10 | 11 | "github.com/evergreen-ci/birch" 12 | "github.com/pkg/errors" 13 | ) 14 | 15 | func readDiagnostic(ctx context.Context, f io.Reader, ch chan<- *birch.Document) error { 16 | defer close(ch) 17 | buf := bufio.NewReader(f) 18 | for { 19 | doc, err := readBufBSON(buf) 20 | if err != nil { 21 | if err == io.EOF { 22 | err = nil 23 | } 24 | return err 25 | } 26 | select { 27 | case ch <- doc: 28 | continue 29 | case <-ctx.Done(): 30 | return nil 31 | } 32 | } 33 | } 34 | 35 | func readChunks(ctx context.Context, ch <-chan *birch.Document, o chan<- *Chunk) error { 36 | defer close(o) 37 | 38 | var metadata *birch.Document 39 | 40 | for doc := range ch { 41 | // the FTDC streams typically have onetime-per-file 42 | // metadata that includes information that doesn't 43 | // change (like process parameters, and machine 44 | // info. This implementation entirely ignores that.) 45 | docType := doc.Lookup("type") 46 | 47 | if isNum(0, docType) { 48 | metadata = doc 49 | continue 50 | } else if !isNum(1, docType) { 51 | continue 52 | } 53 | 54 | id, _ := doc.Lookup("_id").TimeOK() 55 | 56 | // get the data field which holds the metrics chunk 57 | zelem := doc.LookupElement("data") 58 | if zelem == nil { 59 | return errors.New("data is not populated") 60 | } 61 | _, zBytes := zelem.Value().Binary() 62 | 63 | // the metrics chunk, after the first 4 bytes, is zlib 64 | // compressed, so we make a reader for that. data 65 | z, err := zlib.NewReader(bytes.NewBuffer(zBytes[4:])) 66 | if err != nil { 67 | return errors.Wrap(err, "problem building zlib reader") 68 | } 69 | buf := bufio.NewReader(z) 70 | 71 | // the metrics chunk, which is *not* bson, first 72 | // contains a bson document which begins the 73 | // sample. This has the field and we use use it to 74 | // create a slice of Metrics for each series. The 75 | // deltas are not populated. 76 | refDoc, metrics, err := readBufMetrics(buf) 77 | if err != nil { 78 | return errors.Wrap(err, "problem reading metrics") 79 | } 80 | 81 | // now go back and read the first few bytes 82 | // (uncompressed) which tell us how many metrics are 83 | // in each sample (e.g. the fields in the document) 84 | // and how many events are collected in each series. 85 | bl := make([]byte, 8) 86 | _, err = io.ReadAtLeast(buf, bl, 8) 87 | if err != nil { 88 | return err 89 | } 90 | nmetrics := int(binary.LittleEndian.Uint32(bl[:4])) 91 | ndeltas := int(binary.LittleEndian.Uint32(bl[4:])) 92 | 93 | // if the number of metrics that we see from the 94 | // source document (metrics) and the number the file 95 | // reports don't equal, it's probably corrupt. 96 | if nmetrics != len(metrics) { 97 | return errors.Errorf("metrics mismatch, file likely corrupt Expected %d, got %d", nmetrics, len(metrics)) 98 | } 99 | 100 | // now go back and populate the delta numbers 101 | var nzeroes uint64 102 | for i, v := range metrics { 103 | metrics[i].startingValue = v.startingValue 104 | metrics[i].Values = make([]int64, ndeltas) 105 | 106 | for j := 0; j < ndeltas; j++ { 107 | var delta uint64 108 | if nzeroes != 0 { 109 | delta = 0 110 | nzeroes-- 111 | } else { 112 | delta, err = binary.ReadUvarint(buf) 113 | if err != nil { 114 | return errors.Wrap(err, "reached unexpected end of encoded integer") 115 | } 116 | if delta == 0 { 117 | nzeroes, err = binary.ReadUvarint(buf) 118 | if err != nil { 119 | return err 120 | } 121 | } 122 | } 123 | metrics[i].Values[j] = int64(delta) 124 | } 125 | metrics[i].Values = undelta(v.startingValue, metrics[i].Values) 126 | } 127 | select { 128 | case o <- &Chunk{ 129 | Metrics: metrics, 130 | nPoints: ndeltas + 1, // this accounts for the reference document 131 | id: id, 132 | metadata: metadata, 133 | reference: refDoc, 134 | }: 135 | case <-ctx.Done(): 136 | return nil 137 | } 138 | } 139 | return nil 140 | } 141 | 142 | func readBufBSON(buf *bufio.Reader) (*birch.Document, error) { 143 | doc := &birch.Document{} 144 | 145 | if _, err := doc.ReadFrom(buf); err != nil { 146 | return nil, err 147 | } 148 | 149 | return doc, nil 150 | } 151 | 152 | func readBufMetrics(buf *bufio.Reader) (*birch.Document, []Metric, error) { 153 | doc, err := readBufBSON(buf) 154 | if err != nil { 155 | return nil, nil, errors.Wrap(err, "problem reading reference doc") 156 | } 157 | 158 | return doc, metricForDocument([]string{}, doc), nil 159 | } 160 | -------------------------------------------------------------------------------- /t2.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log" 7 | "math" 8 | 9 | "github.com/evergreen-ci/birch" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // GennyOutputMetadata aids in the genny output translation process. 14 | // It stores the actor operation Name, the ftdc file's chunk Iter and 15 | // operation StartTime and EndTime. These values must be set prior to 16 | // calling TranslateGenny. Stores prev values to allow for ftdc files 17 | // to resume from where it previously recorded a window. These values 18 | // are updated during the translation process upon finding a window. 19 | type GennyOutputMetadata struct { 20 | Name string 21 | Iter *ChunkIterator 22 | StartTime int64 23 | EndTime int64 24 | prevIdx int 25 | prevSample []*birch.Element 26 | prevSecond int64 27 | } 28 | 29 | const ( 30 | second_ms int64 = 1000 31 | max_samples int = 300 32 | ) 33 | 34 | // TranslateGenny exports the contents of a stream of genny ts.ftdc 35 | // chunks into cedar ftdc which is readable using t2. Translates 36 | // cumulative event driven metrics into metrics of one-second granularity. 37 | func TranslateGenny(ctx context.Context, gennyOutputSlice []*GennyOutputMetadata, output io.Writer) error { 38 | collector := NewStreamingCollector(max_samples, output) 39 | workloadStartSec := int64(math.MaxInt64) 40 | workloadEndSec := int64(0) 41 | 42 | // Determine when the whole genny workload starts and ends. 43 | for _, gennyOut := range gennyOutputSlice { 44 | workloadStartSec = min(workloadStartSec, gennyOut.StartTime) 45 | workloadEndSec = max(workloadEndSec, gennyOut.EndTime) 46 | if gennyOut.prevSample == nil { 47 | gennyOut.prevSample = createZeroedMetrics() 48 | } 49 | } 50 | 51 | // Iterate through the whole workload duration. 52 | for timeSecond := workloadStartSec; timeSecond < workloadEndSec; timeSecond++ { 53 | if err := ctx.Err(); err != nil { 54 | return err 55 | } 56 | var workloadDoc []*birch.Element 57 | startTime := birch.EC.DateTime("start", timeSecond*second_ms) 58 | workloadDoc = append(workloadDoc, startTime) 59 | 60 | // Append prevSample to workloadDoc if the file has ended, we don't find the next window, 61 | // or if the operation hasn't started at the current time in seconds. 62 | for _, gennyOut := range gennyOutputSlice { 63 | if timeSecond >= gennyOut.prevSecond { 64 | if elems := translateAtNextWindow(gennyOut); elems != nil { 65 | workloadDoc = append(workloadDoc, birch.EC.SubDocument(gennyOut.Name, birch.NewDocument(elems...))) 66 | } else { 67 | workloadDoc = append(workloadDoc, birch.EC.SubDocument(gennyOut.Name, birch.NewDocument(gennyOut.prevSample...))) 68 | } 69 | } else { 70 | workloadDoc = append(workloadDoc, birch.EC.SubDocument(gennyOut.Name, birch.NewDocument(gennyOut.prevSample...))) 71 | } 72 | } 73 | 74 | if len(workloadDoc) > 1 { 75 | cedarElems := birch.NewDocument(workloadDoc...) 76 | cedarDoc := birch.EC.SubDocument("cedar", cedarElems) 77 | if err := collector.Add(birch.NewDocument(cedarDoc)); err != nil { 78 | log.Fatal(err) 79 | } 80 | } 81 | } 82 | 83 | return errors.Wrap(FlushCollector(collector, output), "flushing collector") 84 | } 85 | 86 | // GetGennyTime determines the StartTime and EndTime of a genny workload file 87 | // by passing through all of its chunks. 88 | func GetGennyTime(ctx context.Context, gennyOutputMetadata GennyOutputMetadata) GennyOutputMetadata { 89 | iter := gennyOutputMetadata.Iter 90 | 91 | var endTime int64 92 | for iter.Next() { 93 | if gennyOutputMetadata.StartTime == 0 { 94 | gennyOutputMetadata.StartTime = int64(math.Ceil(float64(iter.Chunk().Metrics[0].Values[0]) / float64(second_ms))) 95 | } 96 | iter.Chunk().GetMetadata() 97 | timestamp := iter.Chunk().Metrics[0].Values 98 | endTime = max(endTime, timestamp[len(timestamp)-1]) 99 | } 100 | gennyOutputMetadata.EndTime = int64(math.Ceil(float64(endTime) / float64(second_ms))) 101 | iter.Close() 102 | 103 | return gennyOutputMetadata 104 | } 105 | 106 | // translateAtNextWindow iterates through the chunks until we find the end 107 | // of the window, i.e., a change in second. Updates GennyOutputMetadata prev values. 108 | func translateAtNextWindow(gennyOutput *GennyOutputMetadata) []*birch.Element { 109 | var elems []*birch.Element 110 | 111 | iter := gennyOutput.Iter 112 | 113 | if iter.Chunk() == nil { 114 | iter.Next() 115 | } 116 | 117 | for elems == nil { 118 | chunk := iter.Chunk() 119 | 120 | // The 0th position in Metrics is always timestamp. 121 | metrics := chunk.Metrics 122 | tsMetric := metrics[0] 123 | for i := gennyOutput.prevIdx; i < len(tsMetric.Values); i++ { 124 | ts := tsMetric.Values[i] 125 | currSecond := int64(math.Ceil(float64(ts) / float64(second_ms))) 126 | 127 | // If we've iterated to the next second, record the values in this sample. 128 | if currSecond != gennyOutput.prevSecond { 129 | elems = translateMetrics(i, metrics) 130 | gennyOutput.prevIdx = i 131 | gennyOutput.prevSecond = currSecond 132 | gennyOutput.prevSample = elems 133 | break 134 | } 135 | } 136 | 137 | // If the end of window isn't found, try the next chunk. 138 | // If the file has ended, returns nil. 139 | if elems == nil { 140 | if iter.Next() { 141 | gennyOutput.prevIdx = 0 142 | } else { 143 | break 144 | } 145 | } 146 | } 147 | return elems 148 | } 149 | 150 | // translateMetrics takes the current chunk and index translate the corresponding metrics. 151 | func translateMetrics(idx int, metrics []Metric) []*birch.Element { 152 | var elems []*birch.Element 153 | for _, metric := range metrics { 154 | switch name := metric.Key(); name { 155 | case "counters.n": 156 | elems = append(elems, birch.EC.Int64("n", metric.Values[idx])) 157 | case "counters.ops": 158 | elems = append(elems, birch.EC.Int64("ops", metric.Values[idx])) 159 | case "counters.size": 160 | elems = append(elems, birch.EC.Int64("size", metric.Values[idx])) 161 | case "counters.errors": 162 | elems = append(elems, birch.EC.Int64("errors", metric.Values[idx])) 163 | case "timers.dur": 164 | elems = append(elems, birch.EC.Int64("dur", metric.Values[idx])) 165 | case "timers.total": 166 | elems = append(elems, birch.EC.Int64("total", metric.Values[idx])) 167 | case "gauges.workers": 168 | elems = append(elems, birch.EC.Int64("workers", metric.Values[idx])) 169 | case "gauges.failed": 170 | elems = append(elems, birch.EC.Int64("failed", metric.Values[idx])) 171 | default: 172 | break 173 | } 174 | } 175 | return elems 176 | } 177 | 178 | // createZeroedMetrics generates a sample of 0s for samples preceding a genny output StartTime. 179 | func createZeroedMetrics() []*birch.Element { 180 | var elems []*birch.Element 181 | 182 | elems = append(elems, birch.EC.Int64("n", 0)) 183 | elems = append(elems, birch.EC.Int64("ops", 0)) 184 | elems = append(elems, birch.EC.Int64("size", 0)) 185 | elems = append(elems, birch.EC.Int64("errors", 0)) 186 | elems = append(elems, birch.EC.Int64("dur", 0)) 187 | elems = append(elems, birch.EC.Int64("total", 0)) 188 | elems = append(elems, birch.EC.Int64("workers", 0)) 189 | elems = append(elems, birch.EC.Int64("failed", 0)) 190 | 191 | return elems 192 | } 193 | 194 | func min(a, b int64) int64 { 195 | if a < b { 196 | return a 197 | } 198 | return b 199 | } 200 | 201 | func max(a, b int64) int64 { 202 | if a > b { 203 | return a 204 | } 205 | return b 206 | } 207 | -------------------------------------------------------------------------------- /t2_test.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "math/rand" 9 | "os" 10 | "testing" 11 | "time" 12 | 13 | "github.com/stretchr/testify/require" 14 | 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | func TestTranslateGennyIntegration(t *testing.T) { 19 | for _, test := range []struct { 20 | name string 21 | path string 22 | skipSlow bool 23 | skipAll bool 24 | expectedNum int 25 | expectedChunks int 26 | expectedMetrics int 27 | reportInterval int 28 | docLen int 29 | }{ 30 | { 31 | name: "GennyMock", 32 | path: "genny_metrics.ftdc", 33 | docLen: 4, 34 | expectedNum: 9, 35 | expectedChunks: 1, 36 | expectedMetrics: 300, 37 | reportInterval: 1000, 38 | skipSlow: true, 39 | }, 40 | } { 41 | t.Run(test.name, func(t *testing.T) { 42 | file, err := os.Open(test.path) 43 | require.NoError(t, err) 44 | defer func() { printError(file.Close()) }() 45 | ctx, cancel := context.WithCancel(context.Background()) 46 | defer cancel() 47 | data, err := ioutil.ReadAll(file) 48 | require.NoError(t, err) 49 | 50 | var mockSlice []*GennyOutputMetadata 51 | var mockGennyMetadata GennyOutputMetadata 52 | mockGennyMetadata.StartTime = 1625003933 53 | mockGennyMetadata.EndTime = 1625004233 54 | mockGennyMetadata.Name = "test" 55 | 56 | t.Run("Translate", func(t *testing.T) { 57 | 58 | //setup metadata 59 | iter := ReadChunks(ctx, bytes.NewBuffer(data)) 60 | mockGennyMetadata.Iter = iter 61 | 62 | startAt := time.Now() 63 | out := &bytes.Buffer{} 64 | err := TranslateGenny(ctx, append(mockSlice, &mockGennyMetadata), out) 65 | require.NoError(t, err) 66 | 67 | // verify output 68 | iter = ReadChunks(ctx, out) 69 | counter := 0 70 | num := 0 71 | lastChunk := false 72 | for iter.Next() { 73 | c := iter.Chunk() 74 | counter++ 75 | if num == 0 { 76 | num = len(c.Metrics) 77 | require.Equal(t, test.expectedNum, num) 78 | } 79 | metric := c.Metrics[rand.Intn(num)] 80 | require.True(t, len(metric.Values) > 0) 81 | 82 | assert.Equal(t, metric.startingValue, metric.Values[0], "key=%s", metric.Key()) 83 | 84 | // only check length of values if it's not the last chunk 85 | if len(metric.Values) < test.expectedMetrics { 86 | require.Equal(t, false, lastChunk) 87 | lastChunk = true 88 | } 89 | 90 | if !lastChunk { 91 | assert.Len(t, metric.Values, test.expectedMetrics, "%d: %d", len(metric.Values), test.expectedMetrics) 92 | } 93 | } 94 | 95 | assert.NoError(t, iter.Err()) 96 | assert.Equal(t, test.expectedNum, num) 97 | assert.Equal(t, test.expectedChunks, counter) 98 | fmt.Println(testMessage{ 99 | "series": num, 100 | "iters": counter, 101 | "dur_secs": time.Since(startAt).Seconds(), 102 | }) 103 | }) 104 | }) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /testutil/util.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math/rand" 7 | "strconv" 8 | 9 | "github.com/evergreen-ci/birch" 10 | "github.com/evergreen-ci/birch/bsontype" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | func CreateEventRecord(count, duration, size, workers int64) *birch.Document { 15 | return birch.NewDocument( 16 | birch.EC.Int64("count", count), 17 | birch.EC.Int64("duration", duration), 18 | birch.EC.Int64("size", size), 19 | birch.EC.Int64("workers", workers), 20 | ) 21 | } 22 | 23 | func RandFlatDocument(numKeys int) *birch.Document { 24 | doc := birch.NewDocument() 25 | for i := 0; i < numKeys; i++ { 26 | doc.Append(birch.EC.Int64(fmt.Sprint(i), rand.Int63n(int64(numKeys)*1))) 27 | } 28 | 29 | return doc 30 | } 31 | 32 | func RandFlatDocumentWithFloats(numKeys int) *birch.Document { 33 | doc := birch.NewDocument() 34 | for i := 0; i < numKeys; i++ { 35 | doc.Append(birch.EC.Double(fmt.Sprintf("%d_float", i), rand.Float64())) 36 | doc.Append(birch.EC.Int64(fmt.Sprintf("%d_long", i), rand.Int63())) 37 | } 38 | return doc 39 | } 40 | 41 | func RandComplexDocument(numKeys, otherNum int) *birch.Document { 42 | doc := birch.NewDocument() 43 | 44 | for i := 0; i < numKeys; i++ { 45 | doc.Append(birch.EC.Int64(fmt.Sprintln(numKeys, otherNum), rand.Int63n(int64(numKeys)*1))) 46 | doc.Append(birch.EC.Double(fmt.Sprintln("float", numKeys, otherNum), rand.Float64())) 47 | 48 | if otherNum%5 == 0 { 49 | ar := birch.NewArray() 50 | for ii := int64(0); i < otherNum; i++ { 51 | ar.Append(birch.VC.Int64(rand.Int63n(1 + ii*int64(numKeys)))) 52 | } 53 | doc.Append(birch.EC.Array(fmt.Sprintln("first", numKeys, otherNum), ar)) 54 | } 55 | 56 | if otherNum%3 == 0 { 57 | doc.Append(birch.EC.SubDocument(fmt.Sprintln("second", numKeys, otherNum), RandFlatDocument(otherNum))) 58 | } 59 | 60 | if otherNum%12 == 0 { 61 | doc.Append(birch.EC.SubDocument(fmt.Sprintln("third", numKeys, otherNum), RandComplexDocument(otherNum, 10))) 62 | } 63 | } 64 | 65 | return doc 66 | } 67 | 68 | func IsMetricsDocument(key string, doc *birch.Document) ([]string, int) { 69 | iter := doc.Iterator() 70 | keys := []string{} 71 | seen := 0 72 | for iter.Next() { 73 | elem := iter.Element() 74 | k, num := IsMetricsValue(fmt.Sprintf("%s/%s", key, elem.Key()), elem.Value()) 75 | if num > 0 { 76 | seen += num 77 | keys = append(keys, k...) 78 | } 79 | } 80 | 81 | return keys, seen 82 | } 83 | 84 | func IsMetricsArray(key string, array *birch.Array) ([]string, int) { 85 | idx := 0 86 | numKeys := 0 87 | keys := []string{} 88 | iter := array.Iterator() 89 | for iter.Next() { 90 | ks, num := IsMetricsValue(key+strconv.Itoa(idx), iter.Value()) 91 | 92 | if num > 0 { 93 | numKeys += num 94 | keys = append(keys, ks...) 95 | } 96 | 97 | idx++ 98 | } 99 | 100 | return keys, numKeys 101 | } 102 | 103 | func IsMetricsValue(key string, val *birch.Value) ([]string, int) { 104 | switch val.Type() { 105 | case bsontype.ObjectID: 106 | return nil, 0 107 | case bsontype.String: 108 | return nil, 0 109 | case bsontype.Decimal128: 110 | return nil, 0 111 | case bsontype.Array: 112 | return IsMetricsArray(key, val.MutableArray()) 113 | case bsontype.EmbeddedDocument: 114 | return IsMetricsDocument(key, val.MutableDocument()) 115 | case bsontype.Boolean: 116 | return []string{key}, 1 117 | case bsontype.Double: 118 | return []string{key}, 1 119 | case bsontype.Int32: 120 | return []string{key}, 1 121 | case bsontype.Int64: 122 | return []string{key}, 1 123 | case bsontype.DateTime: 124 | return []string{key}, 1 125 | case bsontype.Timestamp: 126 | return []string{key}, 2 127 | default: 128 | return nil, 0 129 | } 130 | } 131 | 132 | type NoopWRiter struct { 133 | bytes.Buffer 134 | } 135 | 136 | func (n *NoopWRiter) Write(in []byte) (int, error) { return n.Buffer.Write(in) } 137 | func (n *NoopWRiter) Close() error { return nil } 138 | 139 | type ErrorWriter struct { 140 | bytes.Buffer 141 | } 142 | 143 | func (n *ErrorWriter) Write(in []byte) (int, error) { return 0, errors.New("foo") } 144 | func (n *ErrorWriter) Close() error { return errors.New("close") } 145 | 146 | type marshaler struct { 147 | doc *birch.Document 148 | } 149 | 150 | func (m *marshaler) MarshalBSON() ([]byte, error) { 151 | if m.doc == nil { 152 | return nil, errors.New("empty") 153 | } 154 | return m.doc.MarshalBSON() 155 | } 156 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "encoding/binary" 7 | "math" 8 | "sort" 9 | "time" 10 | 11 | "github.com/evergreen-ci/birch" 12 | "github.com/evergreen-ci/birch/bsontype" 13 | "github.com/pkg/errors" 14 | "go.mongodb.org/mongo-driver/v2/bson" 15 | ) 16 | 17 | func readDocument(in interface{}) (*birch.Document, error) { 18 | switch doc := in.(type) { 19 | case *birch.Document: 20 | return doc, nil 21 | case birch.DocumentMarshaler: 22 | return doc.MarshalDocument() 23 | case []byte: 24 | return birch.ReadDocument(doc) 25 | case birch.Marshaler: 26 | data, err := doc.MarshalBSON() 27 | if err != nil { 28 | return nil, errors.Wrap(err, "problem with unmarshaler") 29 | } 30 | return birch.ReadDocument(data) 31 | case map[string]interface{}, map[string]int, map[string]int64, map[string]uint, map[string]uint64: 32 | elems := birch.DC.Interface(doc).Elements() 33 | sort.Stable(elems) 34 | return birch.DC.Elements(elems...), nil 35 | case map[string]string: 36 | return nil, errors.New("cannot use string maps for metrics documents") 37 | default: 38 | data, err := bson.Marshal(in) 39 | if err != nil { 40 | return nil, errors.Wrap(err, "problem with fallback marshaling") 41 | } 42 | return birch.ReadDocument(data) 43 | } 44 | } 45 | 46 | func getOffset(count, sample, metric int) int { return metric*count + sample } 47 | 48 | func undelta(value int64, deltas []int64) []int64 { 49 | out := make([]int64, len(deltas)+1) 50 | out[0] = value 51 | for idx, delta := range deltas { 52 | out[idx+1] = out[idx] + delta 53 | } 54 | return out 55 | } 56 | 57 | func encodeSizeValue(val uint32) []byte { 58 | tmp := make([]byte, 4) 59 | 60 | binary.LittleEndian.PutUint32(tmp, val) 61 | 62 | return tmp 63 | } 64 | 65 | func encodeValue(val int64) []byte { 66 | tmp := make([]byte, binary.MaxVarintLen64) 67 | num := binary.PutUvarint(tmp, uint64(val)) 68 | return tmp[:num] 69 | } 70 | 71 | func compressBuffer(input []byte) ([]byte, error) { 72 | buf := bytes.NewBuffer([]byte{}) 73 | zbuf := zlib.NewWriter(buf) 74 | 75 | _, err := buf.Write(encodeSizeValue(uint32(len(input)))) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | _, err = zbuf.Write(input) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | err = zbuf.Close() 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return buf.Bytes(), nil 91 | } 92 | 93 | func normalizeFloat(in float64) int64 { return int64(math.Float64bits(in)) } 94 | func restoreFloat(in int64) float64 { return math.Float64frombits(uint64(in)) } 95 | func epochMs(t time.Time) int64 { return t.UnixNano() / 1000000 } 96 | func timeEpocMs(in int64) time.Time { return time.Unix(in/1000, in%1000*1000000) } 97 | 98 | func isNum(num int, val *birch.Value) bool { 99 | if val == nil { 100 | return false 101 | } 102 | 103 | switch val.Type() { 104 | case bsontype.Int32: 105 | return val.Int32() == int32(num) 106 | case bsontype.Int64: 107 | return val.Int64() == int64(num) 108 | case bsontype.Double: 109 | return val.Double() == float64(num) 110 | default: 111 | return false 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /util/catcher.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | // Catcher is an interface for an error collector for use when 11 | // implementing continue-on-error semantics in concurrent 12 | // operations. There are three different Catcher implementations 13 | // provided by this package that differ *only* in terms of the 14 | // string format returned by String() (and also the format of the 15 | // error returned by Resolve().) 16 | // 17 | // If you do not use github.com/pkg/errors to attach 18 | // errors, the implementations are usually functionally 19 | // equivalent. The Extended variant formats the errors using the "%+v" 20 | // (which returns a full stack trace with pkg/errors,) the Simple 21 | // variant uses %s (which includes all the wrapped context,) and the 22 | // basic catcher calls error.Error() (which should be equvalent to %s 23 | // for most error implementations.) 24 | // 25 | // This interface is lifted from mongodb/grip with one implementation 26 | // to avoid having a dependency between FTDC and grip. 27 | type Catcher interface { 28 | Add(error) 29 | AddWhen(bool, error) 30 | Extend([]error) 31 | ExtendWhen(bool, []error) 32 | Len() int 33 | HasErrors() bool 34 | String() string 35 | Resolve() error 36 | Errors() []error 37 | 38 | New(string) 39 | NewWhen(bool, string) 40 | Errorf(string, ...interface{}) 41 | ErrorfWhen(bool, string, ...interface{}) 42 | 43 | Wrap(error, string) 44 | Wrapf(error, string, ...interface{}) 45 | 46 | Check(func() error) 47 | CheckWhen(bool, func() error) 48 | } 49 | 50 | type basicCatcher struct { 51 | errs []error 52 | mutex sync.RWMutex 53 | } 54 | 55 | // NewCatcher constructs the basic implementation of the Catcher interface 56 | func NewCatcher() Catcher { 57 | return &basicCatcher{} 58 | } 59 | 60 | func (c *basicCatcher) String() string { 61 | c.mutex.RLock() 62 | defer c.mutex.RUnlock() 63 | 64 | output := make([]string, len(c.errs)) 65 | 66 | for idx, err := range c.errs { 67 | output[idx] = err.Error() 68 | } 69 | 70 | return strings.Join(output, "\n") 71 | } 72 | 73 | // Add takes an error object and, if it's non-nil, adds it to the 74 | // internal collection of errors. 75 | func (c *basicCatcher) Add(err error) { 76 | if err == nil { 77 | return 78 | } 79 | 80 | c.mutex.Lock() 81 | defer c.mutex.Unlock() 82 | 83 | c.errs = append(c.errs, err) 84 | } 85 | 86 | // Len returns the number of errors stored in the collector. 87 | func (c *basicCatcher) Len() int { 88 | c.mutex.RLock() 89 | defer c.mutex.RUnlock() 90 | 91 | return len(c.errs) 92 | } 93 | 94 | // HasErrors returns true if the collector has ingested errors, and 95 | // false otherwise. 96 | func (c *basicCatcher) HasErrors() bool { 97 | c.mutex.RLock() 98 | defer c.mutex.RUnlock() 99 | 100 | return len(c.errs) > 0 101 | } 102 | 103 | // Extend adds all non-nil errors, passed as arguments to the catcher. 104 | func (c *basicCatcher) Extend(errs []error) { 105 | if len(errs) == 0 { 106 | return 107 | } 108 | 109 | c.mutex.Lock() 110 | defer c.mutex.Unlock() 111 | 112 | for _, err := range errs { 113 | if err == nil { 114 | continue 115 | } 116 | 117 | c.errs = append(c.errs, err) 118 | } 119 | } 120 | 121 | func (c *basicCatcher) Errorf(form string, args ...interface{}) { 122 | if form == "" { 123 | return 124 | } else if len(args) == 0 { 125 | c.New(form) 126 | return 127 | } 128 | c.Add(errors.Errorf(form, args...)) 129 | } 130 | 131 | func (c *basicCatcher) New(e string) { 132 | if e == "" { 133 | return 134 | } 135 | c.Add(errors.New(e)) 136 | } 137 | 138 | func (c *basicCatcher) Wrap(err error, m string) { c.Add(errors.Wrap(err, m)) } 139 | 140 | func (c *basicCatcher) Wrapf(err error, f string, args ...interface{}) { 141 | c.Add(errors.Wrapf(err, f, args...)) 142 | } 143 | 144 | func (c *basicCatcher) AddWhen(cond bool, err error) { 145 | if !cond { 146 | return 147 | } 148 | 149 | c.Add(err) 150 | } 151 | 152 | func (c *basicCatcher) ExtendWhen(cond bool, errs []error) { 153 | if !cond { 154 | return 155 | } 156 | 157 | c.Extend(errs) 158 | } 159 | 160 | func (c *basicCatcher) ErrorfWhen(cond bool, form string, args ...interface{}) { 161 | if !cond { 162 | return 163 | } 164 | 165 | c.Errorf(form, args...) 166 | } 167 | 168 | func (c *basicCatcher) NewWhen(cond bool, e string) { 169 | if !cond { 170 | return 171 | } 172 | 173 | c.New(e) 174 | } 175 | 176 | func (c *basicCatcher) Check(fn func() error) { c.Add(fn()) } 177 | 178 | func (c *basicCatcher) CheckWhen(cond bool, fn func() error) { 179 | if !cond { 180 | return 181 | } 182 | 183 | c.Add(fn()) 184 | } 185 | 186 | func (c *basicCatcher) Errors() []error { 187 | c.mutex.RLock() 188 | defer c.mutex.RUnlock() 189 | 190 | out := make([]error, len(c.errs)) 191 | 192 | copy(out, c.errs) 193 | 194 | return out 195 | } 196 | 197 | // Resolve returns a final error object for the Catcher. If there are 198 | // no errors, it returns nil, and returns an error object with the 199 | // string form of all error objects in the collector. 200 | func (c *basicCatcher) Resolve() error { 201 | if !c.HasErrors() { 202 | return nil 203 | } 204 | 205 | return errors.New(c.String()) 206 | } 207 | -------------------------------------------------------------------------------- /writer.go: -------------------------------------------------------------------------------- 1 | package ftdc 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/evergreen-ci/birch" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | type writerCollector struct { 11 | writer io.WriteCloser 12 | collector *streamingDynamicCollector 13 | } 14 | 15 | func NewWriterCollector(chunkSize int, writer io.WriteCloser) io.WriteCloser { 16 | return &writerCollector{ 17 | writer: writer, 18 | collector: &streamingDynamicCollector{ 19 | output: writer, 20 | streamingCollector: newStreamingCollector(chunkSize, writer), 21 | }, 22 | } 23 | } 24 | 25 | func (w *writerCollector) Write(in []byte) (int, error) { 26 | doc, err := birch.ReadDocument(in) 27 | if err != nil { 28 | return 0, errors.Wrap(err, "problem reading bson document") 29 | } 30 | return len(in), errors.Wrap(w.collector.Add(doc), "problem adding document to collector") 31 | } 32 | 33 | func (w *writerCollector) Close() error { 34 | if err := FlushCollector(w.collector, w.writer); err != nil { 35 | return errors.Wrap(err, "problem flushing documents to collector") 36 | } 37 | 38 | return errors.Wrap(w.writer.Close(), "problem closing underlying writer") 39 | } 40 | --------------------------------------------------------------------------------