├── go.mod ├── LICENSE ├── main.go └── README.md /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Quasilyte/go-benchrun 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Iskander (Alex) Sharipov / Quasilyte 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "os" 11 | "os/exec" 12 | "strings" 13 | ) 14 | 15 | func main() { 16 | flag.Usage = func() { 17 | lines := []string{ 18 | "Usage: go-benchrun [flags...] oldBench newBench [go test args...]", 19 | "* oldBench is a pattern for `old` benchmark (w/o `Benchmark` prefix)", 20 | "* newBench is a pattern for `new` benchmark (w/o `Benchmark` prefix)", 21 | "", 22 | "Example:", 23 | "\t# compare BenchmarkOld and BenchmarkNew from foopkg package with -count=10", 24 | "\t$ go-benchrun Old New -v -count=10 foopkg", 25 | "", 26 | "Flags and defaults:", 27 | } 28 | for _, l := range lines { 29 | fmt.Fprintln(flag.CommandLine.Output(), l) 30 | } 31 | flag.PrintDefaults() 32 | } 33 | 34 | oldFile := flag.String("oldFile", "./old.txt", 35 | `old benchmark results destination file`) 36 | newFile := flag.String("newFile", "./new.txt", 37 | `new benchmark results destination file`) 38 | 39 | flag.Parse() 40 | 41 | oldBench := flag.Arg(0) 42 | newBench := flag.Arg(1) 43 | if oldBench == "" { 44 | log.Fatal("empty first positional arg (old bench name)") 45 | } 46 | if newBench == "" { 47 | log.Fatal("empty second positional arg (new bench name)") 48 | } 49 | 50 | // The "Benchmark" prefix is added implicitly. 51 | // But only if it doesn't look like a sub-bench selector. 52 | if !strings.Contains(oldBench, "/") { 53 | oldBench = "Benchmark" + oldBench 54 | } 55 | if !strings.Contains(newBench, "/") { 56 | newBench = "Benchmark" + newBench 57 | } 58 | 59 | testArgs := flag.Args()[2:] 60 | fmt.Println(" Running old benchmarks:") 61 | runBenchmarks(*oldFile, oldBench, "", testArgs) 62 | fmt.Println(" Running new benchmarks:") 63 | runBenchmarks(*newFile, newBench, oldBench, testArgs) 64 | fmt.Println(" Benchstat results:") 65 | runBenchstat(*oldFile, *newFile) 66 | } 67 | 68 | func runBenchmarks(dstFile, selector, rename string, args []string) { 69 | testArgs := []string{ 70 | "test", 71 | "-bench", selector, 72 | } 73 | testArgs = append(testArgs, args...) 74 | var output bytes.Buffer 75 | cmd := exec.Command("go", testArgs...) 76 | cmd.Stdout = io.MultiWriter(&output, os.Stdout) 77 | cmd.Stderr = io.MultiWriter(&output, os.Stderr) 78 | err := cmd.Run() 79 | out := output.Bytes() 80 | if err != nil { 81 | log.Fatalf("%q: run go test: %v: %s", selector, err, out) 82 | } 83 | if rename != "" { 84 | out = bytes.Replace(out, []byte(selector), []byte(rename), -1) 85 | } 86 | if err := ioutil.WriteFile(dstFile, out, 0666); err != nil { 87 | log.Fatalf("%q: write results: %v", selector, err) 88 | } 89 | } 90 | 91 | func runBenchstat(file1, file2 string) { 92 | out, err := exec.Command("benchstat", "-geomean", file1, file2).CombinedOutput() 93 | if err != nil { 94 | log.Fatalf("run benchstat: %v: %s", err, out) 95 | } 96 | fmt.Print(string(out)) 97 | } 98 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-benchrun 2 | 3 | Convenience wrapper around "go test" + [benchstat](https://godoc.org/golang.org/x/perf/cmd/benchstat). 4 | 5 | Run benchmarking in 1 simple command. 6 | 7 | ## Installation & Quick start 8 | 9 | This install `go-benchrun` binary under your `$GOPATH/bin`: 10 | 11 | ```bash 12 | go get github.com/Quasilyte/go-benchrun 13 | ``` 14 | 15 | If `$GOPATH/bin` is under your system `$PATH`, `go-benchrun` command should be available after that.
16 | This should print the help message: 17 | 18 | ```bash 19 | $ go-benchrun --help 20 | Usage: go-benchrun [flags...] oldBench newBench [go test args...] 21 | * oldBench is a pattern for `old` benchmark (w/o `Benchmark` prefix) 22 | * newBench is a pattern for `new` benchmark (w/o `Benchmark` prefix) 23 | 24 | Example: 25 | # compare BenchmarkOld and BenchmarkNew from foopkg package with -count=10 26 | $ go-benchrun Old New -v -count=10 foopkg 27 | 28 | Flags and defaults: 29 | -newFile string 30 | new benchmark results destination file (default "./new.txt") 31 | -oldFile string 32 | old benchmark results destination file (default "./old.txt") 33 | ``` 34 | 35 | See "Workflow" section for more usage info". 36 | 37 | ## Workflow 38 | 39 | Without `go-benchrun`, your workflow is either of these two: 40 | 41 | 1. Rely on VSC. 42 | * Store old benchmark results (run go test). 43 | * Apply optimizations. 44 | * Run benchmarks again with optimized code. 45 | * Compare results with `benchstat`. 46 | * If you need to switch between implementations, you use stash and/or branches. 47 | 2. Rely on renaming. 48 | * Use one branch, two different benchmarks. 49 | * Collect results from both benchmarks. 50 | * Before running `benchstat`, rename benchmarks, so their name matches. 51 | 52 | `go-benchrun` automates (2) scheme for you. 53 | 54 | 1. First, it runs `-bench=oldBench` and saves results to `oldFile`. 55 | 2. Then it runs `-bench=newBench` and saves results to `newFile`. 56 | 3. After that, it renames `newBench` from `newFile` to `oldBench`. 57 | 4. Finally, it runs `benchstat -geomean oldFile newFile`. 58 | 59 | For example, lets say that you have this test file with benchmarks: 60 | 61 | ```go 62 | package benchmark 63 | 64 | import ( 65 | "testing" 66 | ) 67 | 68 | //go:noinline 69 | func emptySliceLit() []int { 70 | return []int{} 71 | } 72 | 73 | //go:noinline 74 | func makeEmptySlice() []int { 75 | return make([]int, 0) 76 | } 77 | 78 | func BenchmarkEmptySliceLit(b *testing.B) { 79 | for i := 0; i < b.N; i++ { 80 | _ = emptySliceLit() 81 | } 82 | } 83 | 84 | func BenchmarkMakeEmptySlice(b *testing.B) { 85 | for i := 0; i < b.N; i++ { 86 | _ = makeEmptySlice() 87 | } 88 | } 89 | ``` 90 | 91 | In order to compare `BenchmarkEmptySliceLit` and `BenchmarkMakeEmptySlice` you do: 92 | 93 | ```bash 94 | $ go-benchrun EmptySliceLit MakeEmptySlice -v -count=5 . 95 | Running old benchmarks: 96 | goos: linux 97 | goarch: amd64 98 | BenchmarkEmptySliceLit-8 300000000 5.79 ns/op 99 | BenchmarkEmptySliceLit-8 300000000 5.66 ns/op 100 | BenchmarkEmptySliceLit-8 300000000 5.70 ns/op 101 | BenchmarkEmptySliceLit-8 300000000 5.75 ns/op 102 | BenchmarkEmptySliceLit-8 300000000 5.84 ns/op 103 | PASS 104 | ok _/home/quasilyte/CODE/go/bench 11.595s 105 | Running new benchmarks: 106 | goos: linux 107 | goarch: amd64 108 | BenchmarkMakeEmptySlice-8 200000000 6.77 ns/op 109 | BenchmarkMakeEmptySlice-8 200000000 6.52 ns/op 110 | BenchmarkMakeEmptySlice-8 200000000 6.30 ns/op 111 | BenchmarkMakeEmptySlice-8 300000000 5.89 ns/op 112 | BenchmarkMakeEmptySlice-8 200000000 7.03 ns/op 113 | PASS 114 | ok _/home/quasilyte/CODE/go/bench 10.375s 115 | Benchstat results: 116 | name old time/op new time/op delta 117 | EmptySliceLit-8 5.75ns ± 2% 6.50ns ± 9% +13.12% (p=0.008 n=5+5) 118 | ``` 119 | 120 | To skip unit tests, specify `-run` flag for `go test`, as usual. 121 | --------------------------------------------------------------------------------