├── LICENSE ├── README.md ├── pbench.go └── pbench_test.go /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Ben Burkert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pbench [![GoDoc](https://godoc.org/github.com/benburkert/pbench?status.svg)](https://godoc.org/github.com/benburkert/pbench) 2 | 3 | Percentiles for benchmarks. 4 | 5 | ``` shell 6 | $ benchstat <(for i in $(seq 1 5) ; do go test -run=NONE -bench=. -benchtime=10s -benchmem -cpu=1,2,4,8 github.com/benburkert/pbench ; done) 7 | name time/op 8 | Test/Example 246µs ± 1% 9 | Test/Example-2 114µs ± 2% 10 | Test/Example-4 53.0µs ± 1% 11 | Test/Example-8 25.5µs ± 1% 12 | Test/Example/P50 92.6µs ±20% 13 | Test/Example/P50-2 102µs ± 3% 14 | Test/Example/P50-4 93.4µs ±17% 15 | Test/Example/P50-8 104µs ± 0% 16 | Test/Example/P95 568µs ±99% 17 | Test/Example/P95-2 706µs ±65% 18 | Test/Example/P95-4 1.01ms ± 0% 19 | Test/Example/P95-8 502µs ±100% 20 | Test/Example/P99 4.92ms ±109% 21 | Test/Example/P99-2 4.84ms ±107% 22 | Test/Example/P99-4 4.79ms ±109% 23 | Test/Example/P99-8 6.46ms ±82% 24 | Control/Example 250µs ± 6% 25 | Control/Example-2 113µs ± 1% 26 | Control/Example-4 53.1µs ± 2% 27 | Control/Example-8 25.5µs ± 2% 28 | 29 | name alloc/op 30 | Test/Example 72.0B ± 0% 31 | Test/Example-2 80.0B ± 0% 32 | Test/Example-4 96.0B ± 0% 33 | Test/Example-8 128B ± 0% 34 | Control/Example 64.0B ± 0% 35 | Control/Example-2 64.0B ± 0% 36 | Control/Example-4 64.0B ± 0% 37 | Control/Example-8 64.0B ± 0% 38 | 39 | name allocs/op 40 | Test/Example 1.00 ± 0% 41 | Test/Example-2 1.00 ± 0% 42 | Test/Example-4 1.00 ± 0% 43 | Test/Example-8 1.00 ± 0% 44 | Control/Example 1.00 ± 0% 45 | Control/Example-2 1.00 ± 0% 46 | Control/Example-4 1.00 ± 0% 47 | Control/Example-8 1.00 ± 0% 48 | ``` 49 | 50 | # Example 51 | 52 | ``` go 53 | func BenchmarkPercentiles(tb *testing.B) { 54 | b := pbench.New(tb) 55 | b.ReportPercentile(0.5) 56 | b.ReportPercentile(0.95) 57 | b.ReportPercentile(0.99) 58 | 59 | b.Run("Example", func(b *pbench.B) { 60 | rand.Seed(int64(time.Now().Nanosecond())) 61 | 62 | b.ResetTimer() 63 | 64 | b.RunParallel(func(pb *pbench.PB) { 65 | for pb.Next() { 66 | v := rand.Float64() 67 | switch { 68 | case v <= 0.5: 69 | time.Sleep(10 * time.Microsecond) 70 | case v <= 0.95: 71 | time.Sleep(100 * time.Microsecond) 72 | case v <= 0.99: 73 | time.Sleep(1000 * time.Microsecond) 74 | default: 75 | time.Sleep(10000 * time.Microsecond) 76 | } 77 | } 78 | }) 79 | }) 80 | } 81 | ``` 82 | 83 | -------------------------------------------------------------------------------- /pbench.go: -------------------------------------------------------------------------------- 1 | // Package pbench reports percentiles for parallel benchmarks. 2 | package pbench 3 | 4 | import ( 5 | "fmt" 6 | "reflect" 7 | "runtime" 8 | "sort" 9 | "sync" 10 | "testing" 11 | "time" 12 | 13 | "github.com/gavv/monotime" 14 | ) 15 | 16 | // B wraps a testing.B and adds percentiles. 17 | type B struct { 18 | sync.Mutex 19 | *testing.B 20 | 21 | percs []float64 22 | 23 | pbs []*PB 24 | 25 | subBs map[int]*B 26 | cpus []int 27 | } 28 | 29 | // New initializes a B from a wrapped testing.B. 30 | func New(b *testing.B) *B { 31 | return &B{ 32 | B: b, 33 | percs: []float64{}, 34 | pbs: []*PB{}, 35 | subBs: map[int]*B{}, 36 | cpus: []int{}, 37 | } 38 | } 39 | 40 | // ReportPercentile records and reports a percentile in sub-benchmark results. 41 | func (b *B) ReportPercentile(perc float64) { 42 | b.percs = append(b.percs, perc) 43 | } 44 | 45 | // Run benchmarks f as a subbenchmark with the given name. 46 | func (b *B) Run(name string, f func(b *B)) bool { 47 | defer b.report() 48 | 49 | return b.B.Run(name, func(tb *testing.B) { 50 | subB, cpus := &B{B: tb, percs: b.percs}, runtime.GOMAXPROCS(-1) 51 | b.subBs[cpus] = subB 52 | b.cpus = append(b.cpus, cpus) 53 | f(subB) 54 | }) 55 | } 56 | 57 | func (b *B) report() { 58 | b.Lock() 59 | defer b.Unlock() 60 | 61 | cpus := b.cpus[1:] 62 | for _, perc := range b.percs { 63 | reported := map[int]struct{}{} 64 | for _, i := range cpus { 65 | if _, ok := reported[i]; !ok { 66 | b.subBs[i].reportSub(b, perc, i) 67 | } 68 | reported[i] = struct{}{} 69 | } 70 | } 71 | } 72 | 73 | func (b *B) reportSub(parent *B, perc float64, cpus int) { 74 | var durations durationSlice 75 | for _, pb := range b.pbs { 76 | durations = append(durations, pb.s[:pb.idx]...) 77 | } 78 | sort.Sort(durations) 79 | 80 | v := reflect.ValueOf(b.B).Elem() 81 | name := v.FieldByName("name").String() 82 | n := int(v.FieldByName("result").FieldByName("N").Int()) 83 | 84 | ctx := v.FieldByName("context") 85 | if ctx.IsNil() { 86 | ctx = reflect.ValueOf(parent.B).Elem().FieldByName("context") 87 | } 88 | maxLen := ctx.Elem().FieldByName("maxLen").Int() 89 | 90 | idx := int(float64(len(durations)) * perc) 91 | pvalue := time.Duration(durations[idx]) 92 | result := &testing.BenchmarkResult{ 93 | N: n, 94 | T: pvalue * time.Duration(n), 95 | } 96 | 97 | var cpuList string 98 | if cpus > 1 { 99 | cpuList = fmt.Sprintf("-%d", cpus) 100 | } 101 | 102 | benchName := fmt.Sprintf("%s/P%02.5g%s", name, perc*100, cpuList) 103 | fmt.Printf("%-*s\t%s\n", maxLen, benchName, result) 104 | } 105 | 106 | // RunParallel runs a benchmark in parallel. 107 | func (b *B) RunParallel(body func(*PB)) { 108 | b.B.RunParallel(func(pb *testing.PB) { 109 | body(b.pb(pb)) 110 | }) 111 | } 112 | 113 | func (b *B) pb(inner *testing.PB) *PB { 114 | pb := &PB{ 115 | PB: inner, 116 | s: make(durationSlice, b.N), 117 | } 118 | 119 | b.Lock() 120 | defer b.Unlock() 121 | b.pbs = append(b.pbs, pb) 122 | return pb 123 | } 124 | 125 | // A PB is used by RunParallel for running parallel benchmarks. 126 | type PB struct { 127 | *testing.PB 128 | 129 | s durationSlice 130 | tick time.Duration 131 | idx int 132 | } 133 | 134 | // Next reports whether there are more iterations to execute. 135 | func (pb *PB) Next() bool { 136 | if pb.PB.Next() { 137 | pb.record() 138 | return true 139 | } 140 | return false 141 | } 142 | 143 | func (pb *PB) record() { 144 | if pb.tick == 0 { 145 | pb.tick = monotime.Now() 146 | return 147 | } 148 | 149 | now := monotime.Now() 150 | pb.s[pb.idx] = now - pb.tick 151 | pb.idx++ 152 | pb.tick = now 153 | } 154 | 155 | type durationSlice []time.Duration 156 | 157 | func (p durationSlice) Len() int { return len(p) } 158 | func (p durationSlice) Less(i, j int) bool { return p[i] < p[j] } 159 | func (p durationSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 160 | -------------------------------------------------------------------------------- /pbench_test.go: -------------------------------------------------------------------------------- 1 | package pbench 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func BenchmarkTest(tb *testing.B) { 10 | b := New(tb) 11 | b.ReportPercentile(0.5) 12 | b.ReportPercentile(0.95) 13 | b.ReportPercentile(0.99) 14 | 15 | b.Run("Example", func(b *B) { 16 | rand.Seed(int64(time.Now().Nanosecond())) 17 | 18 | b.ResetTimer() 19 | 20 | b.RunParallel(func(pb *PB) { 21 | for pb.Next() { 22 | v := rand.Float64() 23 | switch { 24 | case v <= 0.5: 25 | time.Sleep(10 * time.Microsecond) 26 | case v <= 0.95: 27 | time.Sleep(100 * time.Microsecond) 28 | case v <= 0.99: 29 | time.Sleep(1000 * time.Microsecond) 30 | default: 31 | time.Sleep(10000 * time.Microsecond) 32 | } 33 | } 34 | }) 35 | }) 36 | } 37 | 38 | func BenchmarkControl(b *testing.B) { 39 | b.Run("Example", func(b *testing.B) { 40 | rand.Seed(int64(time.Now().Nanosecond())) 41 | 42 | b.ResetTimer() 43 | 44 | b.RunParallel(func(pb *testing.PB) { 45 | for pb.Next() { 46 | v := rand.Float64() 47 | switch { 48 | case v <= 0.5: 49 | time.Sleep(10 * time.Microsecond) 50 | case v <= 0.95: 51 | time.Sleep(100 * time.Microsecond) 52 | case v <= 0.99: 53 | time.Sleep(1000 * time.Microsecond) 54 | default: 55 | time.Sleep(10000 * time.Microsecond) 56 | } 57 | } 58 | }) 59 | }) 60 | } 61 | --------------------------------------------------------------------------------