├── .gitignore ├── LICENSE ├── README.md ├── benchdb ├── benchdb.go └── benchdb_test.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | *~ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, yhat 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of gobenchdb nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # benchdb 2 | Stores go test bench data in a database 3 | 4 | [![GoDoc](https://godoc.org/github.com/yhat/gobenchdb?status.svg)](https://godoc.org/github.com/yhat/gobenchdb) 5 | 6 | benchdb is a command line tool for running and storing go benchmark data in a database. 7 | It runs the `go test -bench` command in the current working directory and parses the output 8 | using the [parse package](https://godoc.org/golang.org/x/tools/benchmark/parse). The parsed 9 | data is then written to a sql database of your choice. 10 | 11 | Writing benchmark tests in go is simple. The `go test -bench` command is great, but what we needed was a simple tool that organizes the benchmarking data it produces across multiple benchmarking test suites and as the source code changes over time. 12 | 13 | # Installation 14 | 15 | If you have the go tools installed on your machine, gobenchdb can be installed using `go get`. 16 | 17 | ``` 18 | go get github.com/yhat/benchdb 19 | ``` 20 | 21 | Direct downloads of compiled binaries are available at the [releases page](https://github.com/yhat/benchdb/releases). 22 | 23 | # Basic Usage 24 | 25 | benchdb supports postgres as a sql database backend. 26 | 27 | ``` 28 | Usage: benchdb [options...] 29 | 30 | Options: 31 | -conn sql database connection string 32 | -table sql table name 33 | -test.bench run only those benchmarks matching the regular expression 34 | ``` 35 | 36 | # Example 37 | 38 | You can cd to any package directory that has defined benchmark tests and run benchdb. Lets 39 | run a few benchmarks from the golang crypto package and store them in a database! 40 | 41 | ``` 42 | cd $GOPATH/golang.org/src/golang.org/x/crypto/ssh 43 | ``` 44 | 45 | benchdb writes to stdout and a database associated with your connection string. Here is 46 | what you get when you run the command with a connection string and a sql database table 47 | name. 48 | 49 | ``` 50 | $ benchdb -conn="postgres://yhat:foopass@/benchmarks" -table="mytable" 51 | PASS 52 | BenchmarkEndToEnd 100 10195771 ns/op 102.84 MB/s 1286656 B/op 78 allocs/op 53 | BenchmarkMarshalKexInitMsg 200000 8956 ns/op 4040 B/op 7 allocs/op 54 | BenchmarkUnmarshalKexInitMsg 100000 22160 ns/op 5392 B/op 43 allocs/op 55 | BenchmarkMarshalKexDHInitMsg 1000000 1165 ns/op 248 B/op 8 allocs/op 56 | BenchmarkUnmarshalKexDHInitMsg 2000000 658 ns/op 96 B/op 4 allocs/op 57 | ok golang.org/x/crypto/ssh 8.595s 58 | ``` 59 | 60 | Lets look at our database. 61 | 62 | ``` 63 | benchmarks=> select * from mytable where latest_sha = 'c57d4a7'; 64 | id | batch_id | latest_sha | datetime | name | n | ns_op | allocated_bytes_op | allocs_op 65 | ----+----------------------------------+------------+----------------------------+-----------------------+---------+----------+--------------------+----------- 66 | 48 | e1ae21896edb38420d767cace4957efe | c57d4a7 | 2015-07-01 15:39:32.00976 | EndToEnd | 100 | 10257787 | 1286660 | 78 67 | 49 | e1ae21896edb38420d767cace4957efe | c57d4a7 | 2015-07-01 15:39:32.105 | MarshalKexInitMsg | 200000 | 9099 | 4040 | 7 68 | 50 | e1ae21896edb38420d767cace4957efe | c57d4a7 | 2015-07-01 15:39:32.189808 | UnmarshalKexInitMsg | 100000 | 22147 | 5392 | 143 69 | 51 | e1ae21896edb38420d767cace4957efe | c57d4a7 | 2015-07-01 15:39:32.270077 | MarshalKexDHInitMsg | 1000000 | 1161 | 248 | 8 70 | 52 | e1ae21896edb38420d767cace4957efe | c57d4a7 | 2015-07-01 15:39:32.350378 | UnmarshalKexDHInitMsg | 2000000 | 660 | 96 | 4 71 | 72 | ``` 73 | 74 | Each benchdb run is assigned a unique batch_id and the first 7 characters of the latest git sha for HEAD. This way you can group by a batch_id and identify separate benchmark test runs. 75 | 76 | Thats it! 77 | 78 | # Database Schema 79 | 80 | benchdb assumes a table schema of the form. 81 | 82 | ```sql 83 | # postgres 84 | CREATE TABLE IF NOT EXISTS benchmarks ( 85 | id serial primary key, 86 | batch_id varchar(50), 87 | latest_sha varchar(50), 88 | datetime timestamp without time zone, 89 | name varchar(50), 90 | n integer, 91 | ns_op double precision, 92 | allocated_bytes_op integer, 93 | allocs_op integer); 94 | ``` 95 | 96 | # Extending benchdb 97 | 98 | If you want to use benchdb but you wish to use a database besides postgres, you can implent the [BenchDB](https://godoc.org/github.com/yhat/benchdb/benchdb#BenchDB) interface in the [benchdb package](https://godoc.org/github.com/yhat/benchdb/benchdb). 99 | -------------------------------------------------------------------------------- /benchdb/benchdb.go: -------------------------------------------------------------------------------- 1 | package benchdb 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "database/sql" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "os" 11 | "os/exec" 12 | "strings" 13 | "time" 14 | 15 | _ "github.com/lib/pq" 16 | 17 | "golang.org/x/tools/benchmark/parse" 18 | ) 19 | 20 | // A BenchDB manges the execution of benchmark tests using go test and 21 | // writing a parse.Set to a database. 22 | // 23 | // Implementations of BenchDB for different databases are allowed by way 24 | // of the WriteSet method. 25 | type BenchDB interface { 26 | // Run executes go test bench for benchmarks matching a regex defined in 27 | // BenchDBConfigthe current directory. By default it does not run unit tests 28 | // by way of setting test.run to XXX in the call to go test. It also parses the 29 | // benchSet and calls WriteSet to write the benchmark data to a database. It 30 | // returns any error encountered. 31 | Run() error 32 | 33 | // WriteSet is responsible for opening a postgres database connection and writing 34 | // a parsed benchSet to a db table. It closes the connection, returns the number of 35 | // benchmark tests written, and any error encountered. 36 | WriteSet(parse.Set) (int, error) 37 | } 38 | 39 | // BenchDBConfig represents configuration data for BenchDB. 40 | type BenchDBConfig struct { 41 | Regex string // regex to run tests 42 | ShaLen int // number of latest git sha characters 43 | } 44 | 45 | // BenchPSQL represents a BenchDB that writes benchmarks to a postgres 46 | // database. 47 | type BenchPSQL struct { 48 | Config *BenchDBConfig // configuration for go test 49 | Driver string // database driver name 50 | ConnStr string // sql connection string 51 | TableName string // database table name 52 | 53 | dbConn *sql.DB 54 | } 55 | 56 | // Run runs go test benchmarks matching regex in the current directory and writes 57 | // benchmark data to a PSQL database by calling WriteSet. It returns any error 58 | // encountered. 59 | func (benchdb *BenchPSQL) Run() error { 60 | // Exec a subprocess for go test bench and write 61 | // to both stdout and a byte buffer. 62 | cmd := exec.Command("go", "test", "-bench", benchdb.Config.Regex, 63 | "-test.run", "XXX", "-benchmem") 64 | var out bytes.Buffer 65 | cmd.Stdout = io.MultiWriter(os.Stdout, &out) 66 | cmd.Stderr = io.Writer(os.Stderr) 67 | err := cmd.Run() 68 | if err != nil { 69 | return fmt.Errorf("command failed: %v", err) 70 | } 71 | 72 | benchSet, err := parse.ParseSet(&out) 73 | if err != nil { 74 | return fmt.Errorf("failed to parse benchmark data: %v", err) 75 | } 76 | 77 | // Writes parse set to sql database. 78 | _, err = benchdb.WriteSet(benchSet) 79 | if err != nil { 80 | return fmt.Errorf("failed to write benchSet to db: %v", err) 81 | } 82 | return nil 83 | } 84 | 85 | // WriteSet is responsible for opening a postgres database connection and writing 86 | // a parsed benchSet to a db table. It closes the connection, returns the number of 87 | // benchmark tests written, and any error encountered. 88 | // 89 | // A new sql transaction is created and committed per Benchmark in benchSet. This way if a 90 | // db failure occurs all data from the benchSet is not lost. 91 | func (benchdb *BenchPSQL) WriteSet(benchSet parse.Set) (int, error) { 92 | sqlDB, err := sql.Open(benchdb.Driver, benchdb.ConnStr) 93 | if err != nil { 94 | return 0, fmt.Errorf("could not connect to db: %v", err) 95 | } 96 | defer sqlDB.Close() 97 | benchdb.dbConn = sqlDB 98 | 99 | batchId, err := uuid() 100 | if err != nil { 101 | return 0, fmt.Errorf("could not generate batch id: %v\n", err) 102 | } 103 | 104 | cnt := 0 105 | for _, b := range benchSet { 106 | n := len(b) 107 | for i := 0; i < n; i++ { 108 | val := b[i] 109 | err := benchdb.saveBenchmark(batchId, *val) 110 | if err != nil { 111 | return 0, fmt.Errorf("failed to save benchmark: %v", err) 112 | } 113 | cnt++ 114 | } 115 | } 116 | return cnt, nil 117 | } 118 | 119 | func (benchdb *BenchPSQL) saveBenchmark(batchId string, b parse.Benchmark) error { 120 | // Create a transaction per Benchmark. 121 | tx, err := benchdb.dbConn.Begin() 122 | if err != nil { 123 | return err 124 | } 125 | defer tx.Rollback() 126 | 127 | sha, err := latestGitSha(benchdb.Config.ShaLen) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | q := fmt.Sprintf(` 133 | INSERT INTO %s 134 | (batch_id, latest_sha, datetime, name, n, ns_op, allocated_bytes_op, allocs_op) 135 | VALUES 136 | ($1, $2, $3, $4, $5, $6, $7, $8) 137 | `, benchdb.TableName) 138 | 139 | // Strips leading Benchmark string in Benchmark.Name 140 | name := strings.TrimPrefix(strings.TrimSpace(b.Name), "Benchmark") 141 | ts := time.Now().UTC() 142 | 143 | _, err = tx.Exec(q, 144 | batchId, sha, ts, name, b.N, b.NsPerOp, b.AllocedBytesPerOp, b.AllocsPerOp) 145 | if err != nil { 146 | return err 147 | } 148 | return tx.Commit() 149 | } 150 | 151 | func latestGitSha(n int) (string, error) { 152 | out, err := exec.Command("git", "rev-parse", "HEAD").Output() 153 | if err != nil { 154 | return "", fmt.Errorf("failed to get latest git sha: %v\n", err) 155 | } 156 | return string(out[:n]), nil 157 | } 158 | 159 | func uuid() (string, error) { 160 | b := make([]byte, 16) 161 | if _, err := io.ReadFull(rand.Reader, b); err != nil { 162 | return "", err 163 | } 164 | return hex.EncodeToString(b), nil 165 | } 166 | -------------------------------------------------------------------------------- /benchdb/benchdb_test.go: -------------------------------------------------------------------------------- 1 | package benchdb 2 | 3 | import ( 4 | "math/rand" 5 | "sort" 6 | "testing" 7 | ) 8 | 9 | func TestRunNoDBConn(t *testing.T) { 10 | c := "postgres://foo@localhost:5432/benchmark" 11 | table := "footable" 12 | tregex := "BenchmarkMySort1K" 13 | nsha := 7 14 | 15 | err := (&BenchPSQL{ 16 | Config: &BenchDBConfig{ 17 | Regex: tregex, 18 | ShaLen: nsha, 19 | }, 20 | Driver: "postgres", 21 | ConnStr: c, 22 | TableName: table, 23 | }).Run() 24 | if err == nil { 25 | t.Errorf("expected failure due to no db connection") 26 | } 27 | } 28 | 29 | func mySort(data sort.Interface, a, b int) { 30 | sort.Sort(data) 31 | } 32 | 33 | func BenchmarkMySort1K(b *testing.B) { 34 | b.StopTimer() 35 | for i := 0; i < b.N; i++ { 36 | data := make([]int, 1000) 37 | for i := 0; i < len(data); i++ { 38 | data[i] = rand.Int() 39 | } 40 | b.StartTimer() 41 | mySort(sort.IntSlice(data), 0, len(data)) 42 | b.StopTimer() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /* 2 | benchdb is a command line tool for running and storing go benchmark data in a 3 | database. It runs the go test bench command in the current working directory 4 | and parses the output using the parse package. The parsed data is then written to 5 | sql database of your choice. 6 | 7 | Usage: 8 | benchdb [options...] 9 | 10 | Options: 11 | -conn postgres database connection string 12 | -table postgres table name 13 | -test.bench run only those benchmarks matching the regular expression 14 | 15 | benchdb assumes a schema is created in a database 16 | 17 | CREATE TABLE IF NOT EXISTS benchmarks ( 18 | id serial primary key, 19 | batch_id varchar(50), 20 | latest_sha varchar(50), 21 | datetime timestamp without time zone, 22 | name varchar(50), 23 | n integer, 24 | ns_op double precision, 25 | allocated_bytes_op integer, 26 | allocs_op integer 27 | ); 28 | 29 | */ 30 | package main 31 | 32 | import ( 33 | "flag" 34 | "fmt" 35 | "os" 36 | 37 | "github.com/yhat/benchdb/benchdb" 38 | ) 39 | 40 | var ( 41 | conn = flag.String("conn", "", "postgres database connection string") 42 | table = flag.String("table", "", "postgres table name") 43 | testBench = flag.String("test.bench", ".", 44 | "run only those benchmarks matching the regular expression") 45 | ) 46 | 47 | const Postgres = "postgres" 48 | 49 | var nsha = 7 50 | 51 | var usage = `Usage: benchdb [options...] 52 | 53 | Options: 54 | -conn postgres database connection string 55 | -table postgres table name 56 | -test.bench run only those benchmarks matching the regular expression` 57 | 58 | func main() { 59 | // Parse command line args 60 | flag.Parse() 61 | c := *conn 62 | t := *table 63 | tregex := *testBench 64 | 65 | if c == "" { 66 | UsageExit("database conn must be specified") 67 | 68 | } 69 | if t == "" { 70 | UsageExit("database table must be specified") 71 | } 72 | 73 | // Initalize a BenchPSQL to and run benchmarks. 74 | err := (&benchdb.BenchPSQL{ 75 | Config: &benchdb.BenchDBConfig{ 76 | Regex: tregex, 77 | ShaLen: nsha, 78 | }, 79 | Driver: Postgres, 80 | ConnStr: c, 81 | TableName: t, 82 | }).Run() 83 | if err != nil { 84 | fmt.Fprintf(os.Stderr, err.Error()) 85 | os.Exit(1) 86 | } 87 | } 88 | 89 | func UsageExit(message string) { 90 | if message != "" { 91 | fmt.Fprintf(os.Stderr, message) 92 | fmt.Fprintf(os.Stderr, "\n") 93 | } 94 | fmt.Fprintf(os.Stderr, usage) 95 | fmt.Fprintf(os.Stderr, "\n") 96 | os.Exit(1) 97 | } 98 | --------------------------------------------------------------------------------