├── dashs.go ├── dumpssa.go ├── dumpssa_test.go ├── go.mod ├── main.go └── readme.md /dashs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "crypto/sha256" 7 | "fmt" 8 | "hash" 9 | "io" 10 | "log" 11 | "os" 12 | "os/exec" 13 | "path/filepath" 14 | "strconv" 15 | "strings" 16 | ) 17 | 18 | func compareFunctions(platform string, before, after commit) { 19 | await, ascan := streamDashS(platform, before) 20 | bwait, bscan := streamDashS(platform, after) 21 | compareFuncReaders(ascan, bscan, before.sha, after.sha) 22 | await() 23 | bwait() 24 | } 25 | 26 | func compareFuncReaders(a, b io.Reader, aHash, bHash string) { 27 | sizesBuf := new(bytes.Buffer) 28 | sizes := newFilesizes(sizesBuf) 29 | 30 | aChan := make(chan *pkgScanner) 31 | go scanDashS(a, []byte(aHash), aChan) 32 | bChan := make(chan *pkgScanner) 33 | go scanDashS(b, []byte(bHash), bChan) 34 | aPkgs := make(map[string]*pkgScanner) 35 | bPkgs := make(map[string]*pkgScanner) 36 | for { 37 | if aChan == nil && bChan == nil { 38 | // all done! 39 | break 40 | } 41 | var aPkg, bPkg *pkgScanner 42 | select { 43 | case aPkg = <-aChan: 44 | if aPkg == nil { 45 | // aChan is done, disable it 46 | aChan = nil 47 | continue 48 | } 49 | var ok bool 50 | bPkg, ok = bPkgs[aPkg.Name] 51 | if !ok { 52 | // we don't have the bPkg for this one, store and wait for later 53 | aPkgs[aPkg.Name] = aPkg 54 | continue 55 | } 56 | // we're going to process this entry, so delete it 57 | delete(bPkgs, aPkg.Name) 58 | case bPkg = <-bChan: 59 | if bPkg == nil { 60 | // bChan is done, disable it 61 | bChan = nil 62 | continue 63 | } 64 | var ok bool 65 | aPkg, ok = aPkgs[bPkg.Name] 66 | if !ok { 67 | // we don't have the aPkg for this one, store and wait for later 68 | bPkgs[bPkg.Name] = bPkg 69 | continue 70 | } 71 | // we're going to process this entry, so delete it 72 | delete(aPkgs, bPkg.Name) 73 | } 74 | 75 | pkg := aPkg.Name 76 | 77 | needsHeader := true 78 | printHeader := func() { 79 | if !needsHeader { 80 | return 81 | } 82 | fmt.Printf("\n%s%s%s%s\n", ansiFgYellow, ansiBold, pkg, ansiReset) 83 | needsHeader = false 84 | } 85 | 86 | var aTot, bTot int 87 | for name, asf := range aPkg.Funcs { 88 | aTot += asf.textsize 89 | bsf, ok := bPkg.Funcs[name] 90 | if !ok { 91 | if *flagFn != "stats" { 92 | printHeader() 93 | fmt.Println("deleted", cleanFuncName(name)) 94 | } 95 | continue 96 | } 97 | delete(bPkg.Funcs, name) 98 | bTot += bsf.textsize 99 | if bytes.Equal(asf.bodyhash, bsf.bodyhash) { 100 | continue 101 | } 102 | // TODO: option to show these 103 | if asf.textsize == bsf.textsize { 104 | if *flagFn == "all" { 105 | printHeader() 106 | fmt.Print(ansiFgBlue) 107 | fmt.Println(name, "changed") 108 | fmt.Print(ansiReset) 109 | } 110 | // TODO: option for this? 111 | // diff.Text("a", "b", asf.body, bsf.body, os.Stdout) 112 | continue 113 | } 114 | color := "" 115 | show := true 116 | if asf.textsize < bsf.textsize { 117 | if *flagFn == "smaller" || *flagFn == "stats" { 118 | show = false 119 | } 120 | color = ansiFgRed 121 | } else { 122 | if *flagFn == "bigger" || *flagFn == "stats" { 123 | show = false 124 | } 125 | color = ansiFgGreen 126 | } 127 | if show { 128 | printHeader() 129 | fmt.Print(color) 130 | pct := 100 * (float64(bsf.textsize)/float64(asf.textsize) - 1) 131 | fmt.Printf("%s %d -> %d (%+0.2f%%)\n", cleanFuncName(name), asf.textsize, bsf.textsize, pct) 132 | fmt.Print(ansiReset) 133 | } 134 | } 135 | for name, bsf := range bPkg.Funcs { 136 | if *flagFn != "stats" { 137 | printHeader() 138 | fmt.Println("inserted", cleanFuncName(name)) 139 | } 140 | bTot += bsf.textsize 141 | } 142 | sizes.add(pkg+".s", int64(aTot), int64(bTot)) 143 | // TODO: option to print these 144 | // printHeader() 145 | // if aTot == bTot { 146 | // fmt.Print(ansiFgBlue) 147 | // } else if aTot < bTot { 148 | // fmt.Print(ansiFgRed) 149 | // } else { 150 | // fmt.Print(ansiFgGreen) 151 | // } 152 | // // TODO: instead, save totals and print at end 153 | // fmt.Printf("%sTOTAL %d -> %d%s\n", ansiBold, aTot, bTot, ansiReset) 154 | } 155 | sizes.flush("text size") 156 | fmt.Println() 157 | io.Copy(os.Stdout, sizesBuf) 158 | for pkg := range aPkgs { 159 | log.Printf("package %s was deleted", pkg) 160 | } 161 | for pkg := range bPkgs { 162 | log.Printf("package %s was added", pkg) 163 | } 164 | } 165 | 166 | func cleanFuncName(name string) string { 167 | name = strings.TrimPrefix(name, `"".`) 168 | name = strings.TrimPrefix(name, `type.`) 169 | return name 170 | } 171 | 172 | func scanDashS(r io.Reader, sha []byte, c chan<- *pkgScanner) { 173 | // Lazy: attach a fake package to the end 174 | // to flush out the final package being processed. 175 | rr := io.MultiReader(r, strings.NewReader("\n# EOF\n")) 176 | scan := bufio.NewScanner(rr) 177 | var pkgscan *pkgScanner 178 | for scan.Scan() { 179 | // Look for "# pkgname". 180 | b := scan.Bytes() 181 | if len(b) == 0 { 182 | continue 183 | } 184 | if len(b) >= 2 && b[0] == '#' && b[1] == ' ' { 185 | // Found new package. 186 | // If we were working on a package, flush and emit it. 187 | if pkgscan != nil { 188 | pkgscan.flush() 189 | c <- pkgscan 190 | } 191 | pkgscan = &pkgScanner{ 192 | Name: string(b[2:]), 193 | Funcs: make(map[string]stextFunc), 194 | Hash: sha256.New(), 195 | } 196 | continue 197 | } 198 | // Not a new package. Pass the line on to the current WIP package. 199 | if pkgscan != nil { 200 | // TODO: bytes.ReplaceAll allocates; modify in-place instead 201 | b = bytes.ReplaceAll(b, sha, []byte("SHA")) 202 | pkgscan.ProcessLine(b) 203 | } 204 | } 205 | check(scan.Err()) 206 | close(c) 207 | } 208 | 209 | type pkgScanner struct { 210 | Name string 211 | Funcs map[string]stextFunc 212 | // transient state 213 | Hash hash.Hash 214 | stext string 215 | } 216 | 217 | func (s *pkgScanner) ProcessLine(b []byte) { 218 | if b[0] != '\t' { 219 | s.flush() 220 | s.stext = string(b) 221 | return 222 | } 223 | s.Hash.Write(b) 224 | s.Hash.Write([]byte{'\n'}) 225 | } 226 | 227 | func (s *pkgScanner) flush() { 228 | if s.stext != "" && strings.Contains(s.stext, " STEXT ") { 229 | name, size := extractNameAndSize(s.stext) 230 | s.Funcs[name] = stextFunc{ 231 | textsize: size, 232 | bodyhash: s.Hash.Sum(nil), 233 | } 234 | } 235 | s.Hash.Reset() 236 | s.stext = "" 237 | } 238 | 239 | type stextFunc struct { 240 | textsize int // length in instructions of the function 241 | bodyhash []byte // hash of -S output for the function 242 | body string 243 | } 244 | 245 | func extractNameAndSize(stext string) (string, int) { 246 | i := strings.IndexByte(stext, ' ') 247 | name := stext[:i] 248 | stext = stext[i:] 249 | i = strings.Index(stext, " size=") 250 | stext = stext[i+len(" size="):] 251 | i = strings.Index(stext, " ") 252 | stext = stext[:i] 253 | n, err := strconv.Atoi(stext) 254 | check(err) 255 | return name, n 256 | } 257 | 258 | func streamDashS(platform string, c commit) (wait func(), r io.Reader) { 259 | cmdgo := filepath.Join(c.dir, "bin", "go") 260 | cmd := exec.Command(cmdgo, "build", "-gcflags=all=-S -dwarf=false", "std", "cmd") 261 | goos, goarch := parsePlatform(platform) 262 | cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch) 263 | pipe, err := cmd.StderrPipe() 264 | check(err) 265 | err = cmd.Start() 266 | check(err) 267 | wait = func() { 268 | err := cmd.Wait() 269 | check(err) 270 | } 271 | return wait, pipe 272 | } 273 | -------------------------------------------------------------------------------- /dumpssa.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | ) 13 | 14 | func dumpSSA(platform string, before, after commit, fnname string) { 15 | fmt.Printf("dumping SSA for %v:\n", fnname) 16 | // split fnname into pkg+fnname, if necessary 17 | pkg, fnname := splitPkgFnname(fnname) 18 | if pkg == "" { 19 | log.Fatalf("must specify package for %v", fnname) 20 | } 21 | 22 | // make fnname into an easier to deal with filename 23 | filename := strings.ReplaceAll(fnname, "(", "_") 24 | filename = strings.ReplaceAll(filename, ")", "_") 25 | filename = strings.ReplaceAll(filename, ":", "_") 26 | filename = strings.ReplaceAll(filename, "*", ".") 27 | filename = strings.ReplaceAll(filename, "\"", "_") 28 | filename = strings.ReplaceAll(filename, "[", "_") 29 | filename = strings.ReplaceAll(filename, "]", "_") 30 | 31 | for _, c := range []commit{before, after} { 32 | cmdgo := filepath.Join(c.dir, "bin", "go") 33 | args := []string{"build"} 34 | if pkg != "" { 35 | args = append(args, pkg) 36 | } else { 37 | args = append(args, "std", "cmd") 38 | } 39 | cmd := exec.Command(cmdgo, args...) 40 | goos, goarch := parsePlatform(platform) 41 | cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch, "GOSSAFUNC="+fnname) 42 | cmd.Dir = filepath.Join(c.dir, "src") 43 | out, err := cmd.CombinedOutput() 44 | if err != nil { 45 | fmt.Printf("%v:\n%s\n", cmd, out) 46 | log.Fatal(err) 47 | } 48 | 49 | scan := bufio.NewScanner(bytes.NewReader(out)) 50 | for scan.Scan() { 51 | s := scan.Text() 52 | if len(s) == 0 { 53 | continue 54 | } 55 | const dumpedSSATo = "dumped SSA to " 56 | if strings.HasPrefix(s, dumpedSSATo) { 57 | path := s[len(dumpedSSATo):] 58 | if !strings.HasSuffix(path, "ssa.html") { 59 | panic("wrote ssa to non-ssa.html file") 60 | } 61 | if !filepath.IsAbs(path) { 62 | path = filepath.Join(c.dir, "src", path) 63 | } 64 | src := path 65 | prefix := "" 66 | if platform != "" { 67 | prefix = fmt.Sprintf("%s_%s_", goos, goarch) 68 | } 69 | dst := strings.TrimSuffix(path, "ssa.html") + prefix + filename + ".html" 70 | err = os.Rename(src, dst) 71 | check(err) 72 | fmt.Println(dst) 73 | } 74 | } 75 | check(scan.Err()) 76 | } 77 | fmt.Println() 78 | } 79 | 80 | func splitPkgFnname(in string) (pkg, fnname string) { 81 | fnname = in 82 | if slash := strings.LastIndex(fnname, "/"); slash >= 0 { 83 | pkg = fnname[:slash] 84 | fnname = fnname[slash:] 85 | } 86 | if !strings.ContainsAny(fnname, "()*") { 87 | if dot := strings.Index(fnname, "."); dot >= 0 { 88 | pkg += fnname[:dot] 89 | fnname = fnname[dot+1:] 90 | } 91 | } 92 | return pkg, fnname 93 | } 94 | -------------------------------------------------------------------------------- /dumpssa_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestSplitPkgFnname(t *testing.T) { 6 | cases := []struct { 7 | in, pkg, fn string 8 | }{ 9 | {"a/b.c", "a/b", "c"}, 10 | {"(*scanner).digits:*", "", "(*scanner).digits:*"}, 11 | } 12 | 13 | for _, test := range cases { 14 | pkg, fn := splitPkgFnname(test.in) 15 | if pkg != test.pkg || fn != test.fn { 16 | t.Errorf("splitPkgFnname(%q)=%q, %q, want %q, %q", test.in, pkg, fn, test.pkg, test.fn) 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/josharian/compilecmp 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "math" 12 | "net/http" 13 | "os" 14 | "os/exec" 15 | "os/user" 16 | "path/filepath" 17 | "runtime" 18 | "strings" 19 | "sync" 20 | "text/tabwriter" 21 | "time" 22 | ) 23 | 24 | const debug = false // print commands as they are run 25 | 26 | var ( 27 | flagRun = flag.String("run", "", "run benchmarks matching regex") 28 | flagAll = flag.Bool("all", false, "run all benchmarks, not just short ones") 29 | flagCPU = flag.Bool("cpu", false, "run only CPU tests, not alloc tests") 30 | flagObj = flag.Bool("obj", false, "report object file sizes") 31 | flagPkg = flag.String("pkg", "", "benchmark compilation of `pkg`") 32 | flagCount = flag.Int("n", 0, "iterations") 33 | flagEach = flag.Bool("each", false, "run for every commit between before and after") 34 | flagCL = flag.Int("cl", 0, "run benchmark on CL number") 35 | flagFn = flag.String("fn", "", "find changed functions: all, changed, smaller, bigger, stats, or help") 36 | flagDumpSSA = flag.String("dumpssa", "", "dump SSA html for named functions (use like GOSSAFUNC)") 37 | flagAllBash = flag.Bool("allbash", false, "run all.bash for each commit") 38 | 39 | flagFlags = flag.String("flags", "", "compiler flags for both before and after") 40 | flagBeforeFlags = flag.String("beforeflags", "", "compiler flags for before") 41 | flagAfterFlags = flag.String("afterflags", "", "compiler flags for after") 42 | flagPlatforms = flag.String("platforms", "", "comma-separated list of platforms to compile for; all=all platforms, arch=one platform per arch") 43 | ) 44 | 45 | var cwd string 46 | 47 | func main() { 48 | flag.Parse() 49 | log.SetFlags(0) 50 | 51 | cleanCache() 52 | 53 | // Make a temp dir to use for the GOCACHE. 54 | // See golang.org/issue/29561. 55 | dir, err := ioutil.TempDir("", "compilecmp-gocache-") 56 | check(err) 57 | if debug { 58 | fmt.Printf("GOCACHE=%s\n", dir) 59 | } 60 | defer os.RemoveAll(dir) 61 | os.Setenv("GOCACHE", dir) 62 | 63 | cwd, err = os.Getwd() 64 | if err != nil { 65 | log.Fatalf("could not get current working dir: %v", err) 66 | } 67 | beforeRef := "master" 68 | afterRef := "HEAD" 69 | if *flagCL != 0 { 70 | if flag.NArg() > 0 { 71 | log.Fatal("-cl NNN is incompatible with ref arguments") 72 | } 73 | clHead, parent, err := clHeadAndParent(*flagCL) 74 | if err != nil { 75 | log.Fatalf("failed to get CL %d information: %v", *flagCL, err) 76 | } 77 | if parent == "" { 78 | log.Fatal("CL does not have parent") 79 | } 80 | beforeRef = parent 81 | afterRef = clHead 82 | } 83 | switch flag.NArg() { 84 | case 0: 85 | case 1: 86 | beforeRef = flag.Arg(0) 87 | case 2: 88 | beforeRef = flag.Arg(0) 89 | afterRef = flag.Arg(1) 90 | default: 91 | log.Fatal("usage: compilecmp [before-git-ref] [after-git-ref]") 92 | } 93 | // Resolve immediately, for two reasons: 94 | // catch ref problems early, 95 | // and lock in stone the resolution in case the user changes branches 96 | resolve(beforeRef) 97 | resolve(afterRef) 98 | 99 | switch *flagFn { 100 | case "", "all", "changed", "smaller", "bigger", "stats": 101 | case "help": 102 | fallthrough 103 | default: 104 | fmt.Fprintln(os.Stdout, ` 105 | all: print all functions whose contents have changed, regardless of whether their text size changed 106 | changed: print only functions whose text size has changed 107 | smaller: print only functions whose text size has gotten smaller 108 | bigger: print only functions whose text size has gotten bigger 109 | stats: print only the summary (per package function size total) 110 | help: print this message and exit 111 | `[1:]) 112 | os.Exit(2) 113 | } 114 | 115 | // Clean up unused worktrees to avoid error under the following circumstances: 116 | // * run compilecmp ref1 ref2 117 | // * rm -r ~/.compilecmp 118 | // * run compilecmp ref1 ref2 119 | // git gets confused because it thinks ref1 and ref2 have worktrees. 120 | // Pruning fixes that. 121 | if _, err := git("worktree", "prune"); err != nil { 122 | log.Fatalf("could not prune worktrees: %v", err) 123 | } 124 | 125 | compare(beforeRef, afterRef) 126 | if !*flagEach { 127 | return 128 | } 129 | 130 | list, err := git("rev-list", afterRef, beforeRef+".."+afterRef) 131 | check(err) 132 | revs := strings.Fields(string(list)) 133 | for i := len(revs); i > 0; i-- { 134 | before := beforeRef 135 | if i < len(revs) { 136 | before = revs[i] 137 | } 138 | after := revs[i-1] 139 | fmt.Println("---") 140 | compare(before, after) 141 | } 142 | } 143 | 144 | func combineFlags(x, y string) string { 145 | x = strings.TrimSpace(x) 146 | y = strings.TrimSpace(y) 147 | switch { 148 | case x == "": 149 | return y 150 | case y == "": 151 | return x 152 | } 153 | return x + " " + y 154 | } 155 | 156 | func printcommit(ref string) { 157 | sha := resolve(ref) 158 | if !strings.HasPrefix(ref, sha) { 159 | fmt.Printf("%s (%s): %s\n", ref, sha, commitmessage(sha)) 160 | } else { 161 | // TODO: try rev-parse to get a "pretty" name for this ref. 162 | fmt.Printf("%s: %s\n", sha, commitmessage(sha)) 163 | } 164 | } 165 | 166 | func allPlatforms() []string { 167 | cmd := exec.Command("go", "tool", "dist", "list") 168 | out, err := cmd.CombinedOutput() 169 | // TODO: maybe I should run this in the before/after 170 | // repos, since I know they should have functional tools... 171 | if err != nil { 172 | log.Fatalf("failed to run 'go tool dist list': %v", err) 173 | } 174 | out = bytes.TrimSpace(out) 175 | return strings.Split(string(out), "\n") 176 | } 177 | 178 | func compare(beforeRef, afterRef string) { 179 | var platforms []string 180 | switch *flagPlatforms { 181 | case "all": 182 | platforms = allPlatforms() 183 | case "arch": 184 | // one platform per architecture 185 | // in practice, right now, this means linux/* and js/wasm 186 | all := allPlatforms() 187 | for _, platform := range all { 188 | goos, goarch := parsePlatform(platform) 189 | if goos == "linux" || goarch == "wasm" { 190 | platforms = append(platforms, platform) 191 | } 192 | } 193 | default: 194 | platforms = strings.Split(*flagPlatforms, ",") 195 | } 196 | for _, platform := range platforms { 197 | comparePlatform(platform, beforeRef, afterRef) 198 | } 199 | } 200 | 201 | func comparePlatform(platform, beforeRef, afterRef string) { 202 | fmt.Printf("compilecmp %s -> %s\n", beforeRef, afterRef) 203 | printcommit(beforeRef) 204 | printcommit(afterRef) 205 | 206 | if platform != "" { 207 | fmt.Printf("platform: %s\n", platform) 208 | } 209 | 210 | beforeFlags := combineFlags(*flagFlags, *flagBeforeFlags) 211 | if beforeFlags != "" { 212 | fmt.Printf("before flags: %s\n", beforeFlags) 213 | } 214 | afterFlags := combineFlags(*flagFlags, *flagAfterFlags) 215 | if afterFlags != "" { 216 | fmt.Printf("after flags: %s\n", afterFlags) 217 | } 218 | 219 | before := worktree(beforeRef) 220 | after := worktree(afterRef) 221 | if debug { 222 | fmt.Printf("before GOROOT: %s\n", before.dir) 223 | fmt.Printf("after GOROOT: %s\n", after.dir) 224 | } 225 | 226 | if *flagCount > 0 { 227 | fmt.Println() 228 | fmt.Println("benchstat", before.tmp.Name(), after.tmp.Name()) 229 | e := ETA{start: time.Now(), n: *flagCount} 230 | e.update(0) 231 | for i := 0; i < *flagCount+1; i++ { 232 | record := i != 0 // don't record the first run 233 | if record { 234 | e.update(i - 1) 235 | } 236 | before.bench(platform, beforeFlags, record, after.dir) 237 | after.bench(platform, afterFlags, record, after.dir) 238 | if record { 239 | e.update(i) 240 | } 241 | } 242 | fmt.Println() 243 | } 244 | check(before.tmp.Close()) 245 | check(after.tmp.Close()) 246 | if *flagCount > 0 { 247 | cmd := exec.Command("benchstat", before.tmp.Name(), after.tmp.Name()) 248 | out, err := cmd.CombinedOutput() 249 | check(err) 250 | fmt.Println(string(out)) 251 | fmt.Println() 252 | } 253 | fmt.Println() 254 | if platform != "" { 255 | before.cmdgo(platform, "install", "std", "cmd") 256 | after.cmdgo(platform, "install", "std", "cmd") 257 | } 258 | compareBinaries(platform, before, after) 259 | fmt.Println() 260 | if *flagObj { 261 | compareObjectFiles(platform, before, after) 262 | fmt.Println() 263 | } 264 | if *flagFn != "" { 265 | compareFunctions(platform, before, after) 266 | fmt.Println() 267 | } 268 | if *flagDumpSSA != "" { 269 | dumpSSA(platform, before, after, *flagDumpSSA) 270 | } 271 | // todo: notification? 272 | 273 | // Clean the go cache; see golang.org/issue/29561. 274 | after.cmdgo("", "clean", "-cache") 275 | } 276 | 277 | const ( 278 | ansiBold = "\u001b[1m" 279 | ansiDim = "\u001b[2m" 280 | ansiFgRed = "\u001b[31m" 281 | ansiFgGreen = "\u001b[32m" 282 | ansiFgYellow = "\u001b[33m" 283 | ansiFgBlue = "\u001b[36m" 284 | ansiFgWhite = "\u001b[37m" 285 | ansiReset = "\u001b[0m" 286 | ) 287 | 288 | type filesizes struct { 289 | totbefore int64 290 | totafter int64 291 | haschange bool 292 | out io.Writer 293 | w *tabwriter.Writer 294 | } 295 | 296 | func newFilesizes(out io.Writer) *filesizes { 297 | w := tabwriter.NewWriter(out, 8, 8, 1, ' ', 0) 298 | fmt.Fprintln(w, "file\tbefore\tafter\tΔ\t%\t") 299 | sizes := new(filesizes) 300 | sizes.w = w 301 | sizes.out = out 302 | return sizes 303 | } 304 | 305 | func (s *filesizes) add(name string, beforeSize, afterSize int64) { 306 | if beforeSize == 0 || afterSize == 0 { 307 | return 308 | } 309 | s.totbefore += beforeSize 310 | s.totafter += afterSize 311 | if beforeSize == afterSize { 312 | return 313 | } 314 | s.haschange = true 315 | fmt.Fprintf(s.w, "%s\t%d\t%d\t%+d\t%+0.3f%%\t\n", name, beforeSize, afterSize, afterSize-beforeSize, 100*float64(afterSize)/float64(beforeSize)-100) 316 | } 317 | 318 | func (s *filesizes) flush(desc string) { 319 | if s.haschange { 320 | fmt.Fprintf(s.w, "%s\t%d\t%d\t%+d\t%+0.3f%%\t\n", "total", s.totbefore, s.totafter, s.totafter-s.totbefore, 100*float64(s.totafter)/float64(s.totbefore)-100) 321 | s.w.Flush() 322 | return 323 | } 324 | fmt.Fprintf(s.out, "no %s size changes\n", desc) 325 | } 326 | 327 | func compareBinaries(platform string, before, after commit) { 328 | sizes := newFilesizes(os.Stdout) 329 | // TODO: use glob instead of hard-coding 330 | goos, goarch := parsePlatform(platform) 331 | dirs := []string{"pkg/tool/" + goos + "_" + goarch} 332 | if platform != "" { 333 | dirs = append(dirs, "bin") 334 | } 335 | for _, dir := range dirs { 336 | for _, base := range []string{"go", "addr2line", "api", "asm", "buildid", "cgo", "compile", "cover", "dist", "doc", "fix", "link", "nm", "objdump", "pack", "pprof", "test2json", "trace", "vet"} { 337 | path := filepath.FromSlash(dir + "/" + base) 338 | beforeSize := filesize(filepath.Join(before.dir, path)) 339 | afterSize := filesize(filepath.Join(after.dir, filepath.FromSlash(path))) 340 | name := filepath.Base(path) 341 | sizes.add(name, beforeSize, afterSize) 342 | } 343 | } 344 | sizes.flush("binary") 345 | } 346 | 347 | func compareObjectFiles(platform string, before, after commit) { 348 | platformPath := strings.ReplaceAll(platform, "/", "_") 349 | pkg := filepath.Join(before.dir, "pkg") 350 | if platformPath != "" { 351 | pkg = filepath.Join(pkg, platformPath) 352 | } 353 | var files []string 354 | err := filepath.Walk(pkg, func(path string, info os.FileInfo, err error) error { 355 | if err != nil { 356 | return err 357 | } 358 | if info.IsDir() || !strings.HasSuffix(path, ".a") || !strings.HasPrefix(path, pkg) { 359 | return nil 360 | } 361 | files = append(files, path) 362 | return nil 363 | }) 364 | check(err) 365 | sizes := newFilesizes(os.Stdout) 366 | for _, beforePath := range files { 367 | suff := beforePath[len(pkg):] 368 | afterPath := filepath.Join(after.dir, "pkg") 369 | if platformPath != "" { 370 | afterPath = filepath.Join(afterPath, platformPath) 371 | } 372 | afterPath = filepath.Join(afterPath, suff) 373 | beforeSize := filesize(beforePath) 374 | afterSize := filesize(afterPath) 375 | // suff is of the form /arch/. Remove that. 376 | suff = filepath.ToSlash(suff) 377 | suff = suff[1:] // remove leading slash 378 | suff = suff[strings.IndexByte(suff, '/')+1:] // remove next slash 379 | sizes.add(suff, beforeSize, afterSize) 380 | } 381 | sizes.flush("object file") 382 | } 383 | 384 | func filesize(path string) int64 { 385 | fi, err := os.Stat(path) 386 | if err != nil { 387 | return 0 388 | } 389 | return fi.Size() 390 | } 391 | 392 | func check(err error) { 393 | if err != nil { 394 | log.Panic(err) 395 | } 396 | } 397 | 398 | func parsePlatform(platform string) (goos, goarch string) { 399 | if platform == "" { 400 | return runtime.GOOS, runtime.GOARCH 401 | } 402 | f := strings.Split(platform, "/") 403 | if len(f) != 2 { 404 | panic("bad platform: " + platform) 405 | } 406 | return f[0], f[1] 407 | } 408 | 409 | type commit struct { 410 | ref string 411 | sha string 412 | dir string 413 | tmp *os.File 414 | } 415 | 416 | func (c *commit) cmdgo(platform string, args ...string) []byte { 417 | cmdgo := filepath.Join(c.dir, "bin", "go") 418 | cmd := exec.Command(cmdgo, args...) 419 | goos, goarch := parsePlatform(platform) 420 | cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch) 421 | cmd.Dir = filepath.Join(c.dir, "src") 422 | out, err := cmd.CombinedOutput() 423 | check(err) 424 | return out 425 | } 426 | 427 | func (c *commit) bench(platform, compilerflags string, record bool, goroot string) { 428 | var args []string 429 | if !*flagAll { 430 | args = append(args, "-short") 431 | } 432 | if !*flagCPU { 433 | args = append(args, "-alloc") 434 | } 435 | if *flagObj { 436 | args = append(args, "-obj") 437 | } 438 | if *flagPkg != "" { 439 | args = append(args, "-pkg", *flagPkg) 440 | } 441 | if *flagRun != "" { 442 | args = append(args, "-run", *flagRun) 443 | } 444 | if strings.TrimSpace(compilerflags) != "" { 445 | args = append(args, "-compileflags", compilerflags) 446 | } 447 | args = append(args, "-go="+filepath.Join(c.dir, "bin", "go")) 448 | cmd := exec.Command("compilebench", args...) 449 | path := "PATH=" + filepath.Join(c.dir, "bin") 450 | if sz, err := exec.LookPath("size"); err == nil { 451 | path += ":" + filepath.Dir(sz) 452 | } 453 | goos, goarch := parsePlatform(platform) 454 | cmd.Env = append(os.Environ(), path, "GOOS="+goos, "GOARCH="+goarch) 455 | cmd.Dir = c.dir 456 | if record { 457 | cmd.Stdout = c.tmp 458 | } 459 | err := cmd.Run() 460 | check(err) 461 | } 462 | 463 | func git(args ...string) ([]byte, error) { 464 | cmd := exec.Command("git", args...) 465 | cmd.Dir = cwd 466 | out, err := cmd.CombinedOutput() 467 | return bytes.TrimSpace(out), err 468 | } 469 | 470 | var ( 471 | resolveMu sync.Mutex 472 | resolved = map[string]string{} // ref -> sha 473 | ) 474 | 475 | func resolve(ref string) string { 476 | resolveMu.Lock() 477 | defer resolveMu.Unlock() 478 | if sha, ok := resolved[ref]; ok { 479 | return sha 480 | } 481 | // Resolve ref to a sha1. 482 | out, err := git("rev-parse", "--short", ref) 483 | if err != nil { 484 | log.Fatalf("could not resolve ref %q: %v", ref, err) 485 | } 486 | sha := string(out) 487 | resolved[ref] = sha 488 | return sha 489 | } 490 | 491 | func worktree(ref string) commit { 492 | u, err := user.Current() 493 | check(err) 494 | sha := resolve(ref) 495 | dest := filepath.Join(u.HomeDir, ".compilecmp", sha) 496 | if !exists(dest) { 497 | if debug { 498 | fmt.Printf("cp <%s> %s\n", ref, dest) 499 | } 500 | if _, err := git("worktree", "add", "--detach", dest, ref); err != nil { 501 | log.Fatalf("could not create worktree for %q (%q): %v", ref, sha, err) 502 | } 503 | } 504 | var commands []string 505 | cmdgo := filepath.Join(dest, "bin", "go") 506 | switch { 507 | case *flagAllBash: 508 | // If requested, run all.bash. 509 | commands = append(commands, filepath.Join(dest, "src", "all.bash")) 510 | case exists(cmdgo): 511 | // cmd/go exists, presumably from a previous run. 512 | // Make sure everything is built, just in case a prior make.bash got interrupted. 513 | commands = append(commands, cmdgo+" install std cmd") 514 | default: 515 | // No cmd/go. Probably a new installation. Run make.bash. 516 | commands = append(commands, filepath.Join(dest, "src", "make.bash")) 517 | } 518 | for _, command := range commands { 519 | args := strings.Split(command, " ") 520 | cmd := exec.Command(args[0], args[1:]...) 521 | cmd.Dir = filepath.Join(dest, "src") 522 | if debug { 523 | fmt.Println(command) 524 | } 525 | out, err := cmd.CombinedOutput() 526 | if err != nil { 527 | log.Fatalf("%s\n%v", out, err) 528 | } 529 | } 530 | // These deletions are best effort. 531 | // See https://github.com/golang/go/issues/31851 for context. 532 | os.RemoveAll(filepath.Join(dest, "pkg", "obj")) 533 | os.RemoveAll(filepath.Join(dest, "pkg", "bootstrap")) 534 | tmp, err := ioutil.TempFile("", "") 535 | check(err) 536 | return commit{ref: ref, sha: sha, dir: dest, tmp: tmp} 537 | } 538 | 539 | func exists(path string) bool { 540 | // can stat? it exists. good enough. 541 | _, err := os.Stat(path) 542 | return err == nil 543 | } 544 | 545 | func commitmessage(ref string) []byte { 546 | b, err := git("log", "--format=%s", "-n", "1", ref) 547 | check(err) 548 | return b 549 | } 550 | 551 | // clHeadAndParent fetches the given CL to local, returns the CL HEAD and its parents commits. 552 | func clHeadAndParent(cl int) (string, string, error) { 553 | clUrlFormat := "https://go-review.googlesource.com/changes/%d/?o=CURRENT_REVISION&o=ALL_COMMITS" 554 | resp, err := http.Get(fmt.Sprintf(clUrlFormat, cl)) 555 | if err != nil { 556 | return "", "", err 557 | } 558 | 559 | // Work around https://code.google.com/p/gerrit/issues/detail?id=3540 560 | body, err := ioutil.ReadAll(resp.Body) 561 | if err != nil { 562 | return "", "", err 563 | } 564 | body = bytes.TrimPrefix(body, []byte(")]}'")) 565 | 566 | var parse struct { 567 | CurrentRevision string `json:"current_revision"` 568 | Revisions map[string]struct { 569 | Fetch struct { 570 | HTTP struct { 571 | URL string 572 | Ref string 573 | } 574 | } 575 | Commit struct { 576 | Parents []struct { 577 | Commit string 578 | } 579 | } 580 | } 581 | } 582 | 583 | if err := json.Unmarshal(body, &parse); err != nil { 584 | return "", "", err 585 | } 586 | parent := "" 587 | if len(parse.Revisions[parse.CurrentRevision].Commit.Parents) > 0 { 588 | parent = parse.Revisions[parse.CurrentRevision].Commit.Parents[0].Commit 589 | } 590 | 591 | ref := parse.Revisions[parse.CurrentRevision].Fetch.HTTP 592 | 593 | if _, err := git("fetch", ref.URL, ref.Ref); err != nil { 594 | return "", "", err 595 | } 596 | return parse.CurrentRevision, parent, nil 597 | } 598 | 599 | func cleanCache() { 600 | u, err := user.Current() 601 | check(err) 602 | root := filepath.Join(u.HomeDir, ".compilecmp") 603 | err = os.MkdirAll(root, 0755) 604 | check(err) 605 | f, err := os.Open(root) 606 | check(err) 607 | defer f.Close() 608 | fis, err := f.Readdir(-1) 609 | check(err) 610 | 611 | // Look through ~/.compilecmp for any shas 612 | // that are no longer contained in any branch, and delete them. 613 | // This is the most common way to end up accumulating 614 | // lots of junk in .compilecmp. 615 | var wg sync.WaitGroup 616 | gate := make(chan bool, 10) // gate concurrent calls 617 | for _, fi := range fis { 618 | if !fi.IsDir() { 619 | continue 620 | } 621 | wg.Add(1) 622 | go func(sha string) { 623 | defer wg.Done() 624 | gate <- true 625 | defer func() { <-gate }() 626 | wt := filepath.Join(root, sha) 627 | cmd := exec.Command("git", "branch", "--contains", sha) 628 | cmd.Dir = wt 629 | out, err := cmd.CombinedOutput() 630 | okToDelete := false 631 | if err != nil { 632 | if strings.Contains(string(out), "not a git repository") { 633 | // partially initialized repo; nuke it 634 | okToDelete = true 635 | } else { 636 | log.Fatalf("%s\n%s$ %s: %v", out, wt, cmd, err) 637 | } 638 | } 639 | s := strings.TrimSpace(string(out)) 640 | lines := strings.Split(s, "\n") 641 | if okToDelete || len(lines) == 0 || 642 | (len(lines) == 1 && lines[0] == "* (no branch)") { 643 | // OK to delete 644 | err := os.RemoveAll(wt) 645 | if err != nil { 646 | log.Printf("failed to remove unreachable worktree %s: %v", wt, err) 647 | } 648 | } 649 | }(fi.Name()) 650 | } 651 | wg.Wait() 652 | 653 | // TODO: also look for very old versions? 654 | // We could do this by always touching (say) GOROOT/VERSION 655 | // every time we use a worktree, and then looking at last mtime. 656 | } 657 | 658 | type ETA struct { 659 | start time.Time 660 | n int 661 | } 662 | 663 | func (e *ETA) update(i int) { 664 | elapsed := time.Since(e.start) 665 | eta := "??" 666 | remain := "??" 667 | if i > 0 { 668 | avg := elapsed / time.Duration(i) 669 | r := (time.Duration(e.n - i)) * avg 670 | r /= time.Second 671 | r *= time.Second 672 | remain = fmt.Sprint(r) 673 | eta = time.Now().Add(r).Round(time.Second).Format(time.Kitchen) 674 | } 675 | digits := int(math.Ceil(math.Log10(float64(e.n + 1)))) 676 | fmt.Printf("\rcompleted %[1]*d of %d, estimated time remaining %v (ETA %v) ", digits, i, e.n, remain, eta) 677 | } 678 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | compilecmp is a bit of a Swiss Army knife for Go compiler developers. It compares the compiler at different git commits. It can compare compilation time and memory usage, the sizes of the generated binaries, the sizes of the generated object files, and the generated code. 2 | 3 | Important caveats: 4 | 5 | - compilecmp measures the toolchain (GOROOT) it was compiled with 6 | - compilecmp (unlike toolstash) uses the toolchain on itself, so if you (say) add a bunch of code to text/template, compilecmp will report that text/template got slower to compile; since the compiler itself is one of the subject packages, it can be ambiguous why a performance change for those entries occurred 7 | - it is not safe to run multiple compilecmps concurrently 8 | - on startup, compilecmp deletes git-unreachable entries from its cache, which can be slow, because GOROOTs are large 9 | 10 | # Specifying commits 11 | 12 | compilecmp accepts up to two git refs as arguments. 13 | 14 | - If no refs are provided, it assumes you are measuring from master to HEAD. 15 | 16 | ``` 17 | $ compilecmp # compares master to head 18 | ``` 19 | 20 | - If one ref is provided, compilecmp treats it as the before commit. 21 | 22 | ``` 23 | $ compilecmp head~1 # compares current commit to its parent 24 | ``` 25 | 26 | - If two refs are provided, they are treated as the before and after commits. 27 | 28 | ``` 29 | $ compilecmp go1.13 speedy # compares tagged go1.13 release to branch speedy 30 | ``` 31 | 32 | There is one exception: instead of providing any commits, you may provide -cl. This downloads a CL from gerrit and compares it to its parent. 33 | 34 | ``` 35 | $ compilecmp -cl 12345 # measures the performance impact of CL 12345 36 | ``` 37 | 38 | compilecmp can also easily measure a series of commits, usually in a branch, using `-each`. 39 | 40 | ``` 41 | $ compilecmp -each master head # compares master to head, and then every individual commit between master and head to its parent 42 | ``` 43 | 44 | # Number of runs 45 | 46 | Some compiler outputs are always the same, like the generated code, object files, and binaries. 47 | 48 | Others require multiple runs to measure accurately, such as allocations and CPU time. The `-n` flag lets you run multiple iterations. (Without `-n`, compilecmp does not report any data on allocations or CPU time.) 49 | 50 | ``` 51 | $ compilecmp -n 5 # run five iterations 52 | ``` 53 | 54 | `-n 5` is a good number for measuring allocations. 55 | 56 | `-n 50` is a good number for measuring CPU time. `-n 100` is better, particularly if you are trying to detect small changes. This takes a long time! compilecmp will print an ETA. Be sure to quit all other running apps, including backups (like Time Machine). I also suggest using `-cpu` in this case, to suppress memory profiling, which yields more consistent results. 57 | 58 | When `n > 0`, compilecmp also does a uncounted warmup run at the beginning. 59 | 60 | When you specify a number of runs, compilecmp defaults to running all benchmarks. 61 | 62 | # Compare files sizes 63 | 64 | By default, compilecmp prints the sizes of executables such as cmd/addr2line. 65 | 66 | `-obj` adds object files sizes. Beware that object sizes aren’t always correlated to compilation quality! There’s lots of other stuff in there: dwarf, pclntab, export information, etc. 67 | 68 | # Comparing generated code 69 | 70 | compilecmp can also compare the generated code, function by function. (This part is still in flux a bit.) 71 | 72 | - `-fn=all`: print all functions whose contents have changed 73 | - `-fn=changed`: print all functions whose text size has changed 74 | - `-fn=smaller`: print all functions whose text size has gotten smaller 75 | - `-fn=bigger`: print all functions whose text size has gotten bigger 76 | - `-fn=stats`: print only the summary (per package total function text size) 77 | 78 | # Dumping SSA 79 | 80 | If you've identified a function of interest, you might want to compare 81 | the ssa.html output: 82 | 83 | ``` 84 | $ compilecmp -dumpssa "(*decoder).processDHT" 85 | ``` 86 | 87 | This will print the path to before and after SSA html files. 88 | 89 | # Platform 90 | 91 | compilecmp compiles for the host platform by default. To compile for other platforms, use `-platforms`. 92 | 93 | ``` 94 | $ compilecmp -platforms=darwin/amd64,linux/arm # compare compilation for two platforms 95 | $ compilecmp -platforms=all # compare compilation for all platforms 96 | $ compilecmp -platforms=arch # compare compilation for one platform per architecture 97 | ``` 98 | 99 | # Limiting the set of benchmarks 100 | 101 | By default, compilecmp uses the benchmarks from `compilebench`. To run just a subset of those: 102 | 103 | ``` 104 | $ compilecmp -run Unicode # runs only the unicode compiler benchmark 105 | ``` 106 | 107 | You can also run benchmarks for any package, not necessarily only those in `compilebench`, by using `-pkg`. 108 | 109 | ``` 110 | $ compilebench -pkg github.com/pkg/diff # test speed/allocs when compiling this package 111 | ``` 112 | 113 | # Extra compiler flags 114 | 115 | compilecmp can pass extra compiler flags. To see how much using `-race` slows down the compiler: 116 | 117 | ``` 118 | $ compilecmp -afterflags=-race 119 | ``` 120 | 121 | `-beforeflags` passes flags to only the "before" commit. `flags` adds flags to both `-beforeflags` and `-afterflags`. 122 | --------------------------------------------------------------------------------