├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE.txt ├── README.md ├── bench ├── README.md ├── analysis.ipynb ├── block_bufchan_linux_x86_64.csv ├── block_darwin_x86_64.csv ├── block_linux_x86_64.csv ├── block_linux_x86_64.png ├── csv.go ├── main.go ├── run.sh ├── workload_chan.go └── workload_mutex.go ├── block-bias.md ├── block.md ├── cpu.md ├── datadog.excalidraw ├── datadog.md ├── datadog.png ├── delve └── stackannotate.star ├── examples ├── block-bias │ ├── block.pb.gz │ ├── go.mod │ ├── go.sum │ └── main.go ├── block-net │ ├── README.md │ ├── block-net.png │ ├── block.pb.gz │ ├── go.mod │ ├── go.sum │ └── main.go ├── block-sample │ ├── go.mod │ ├── go.sum │ └── main.go ├── block-vs-mutex │ └── main.go ├── block │ ├── main │ └── main.go ├── callers-func-length │ ├── README.md │ ├── go.mod │ ├── main.go │ └── main_test.go ├── cpu │ ├── benchmark.cpu.pb.gz │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── main_test.go │ ├── pprof.samples.cpu.001.pb.gz │ ├── pprof.samples.cpu.001.pprof.txt │ └── pprof.samples.cpu.001.protoc.txt ├── datadog │ ├── go.mod │ ├── go.sum │ └── main.go ├── delve-trace │ ├── go.mod │ └── main.go ├── frame-order │ ├── frames.txt │ ├── go.mod │ └── main.go ├── goroutine │ ├── 1.net.http.pprof.goroutine.debug0.pb.gz │ ├── 1.net.http.pprof.goroutine.debug1.txt │ ├── 1.net.http.pprof.goroutine.debug2.txt │ ├── 1.pprof.lookup.goroutine.debug0.pb.gz │ ├── 1.pprof.lookup.goroutine.debug1.txt │ ├── 1.pprof.lookup.goroutine.debug2.txt │ ├── 1.runtime.goroutineprofile.json │ ├── 1.runtime.stack.txt │ ├── 2.net.http.pprof.goroutine.debug0.pb.gz │ ├── 2.net.http.pprof.goroutine.debug1.txt │ ├── 2.net.http.pprof.goroutine.debug2.txt │ ├── 2.pprof.lookup.goroutine.debug0.pb.gz │ ├── 2.pprof.lookup.goroutine.debug1.txt │ ├── 2.pprof.lookup.goroutine.debug2.txt │ ├── 2.runtime.goroutineprofile.json │ ├── 2.runtime.stack.txt │ ├── go.mod │ ├── main.go │ └── main_test.go ├── guide │ ├── main │ ├── main.go │ ├── main2.go │ ├── main3.go │ └── trace.bin ├── memory │ ├── go.mod │ ├── go.sum │ └── main.go ├── pclnttab │ ├── darwin.go │ ├── go.mod │ ├── linux.go │ └── main.go ├── presentation │ ├── main.go │ ├── main.simple.txt │ ├── main.txt │ └── stack.txt ├── pseudo-code │ ├── pseudo-block.txt │ └── pseudo-heap.txt ├── runtime-stack │ ├── go.mod │ ├── main.go │ ├── main_test.go │ └── runtime-stack.txt ├── stack-unwind-overhead │ ├── README.md │ ├── generated.go │ ├── go.mod │ ├── main.go │ └── main_test.go ├── stackannotate │ └── main.go ├── stuck-deadlock │ ├── go.mod │ ├── go.sum │ └── main.go ├── stuck-producer-consumer │ ├── go.mod │ ├── go.sum │ └── main.go └── threadcreate │ └── main.go ├── flame-abc.png ├── flame-abc.txt ├── go-binary.png ├── goroutine-matrix.png ├── goroutine-stack.png ├── goroutine.md ├── guide.excalidraw ├── guide ├── Makefile ├── README.md ├── cpu-max-stack-depth.go ├── cpu-max-stack-depth.pprof ├── cpu-profiler-labels.go ├── cpu-profiler-labels.png ├── cpu-profiler-labels.pprof ├── cpu-rate.go ├── cpu-rate.pprof ├── cpu-utilization.go ├── cpu-utilization.pprof ├── guide.excalidraw ├── heap-gc.1.png ├── heap-gc.2.png ├── heap-gc.gif ├── heap-simple.png ├── heap.go ├── heap.png ├── http-get.go ├── memory-profiler.go ├── memory-profiler.pprof ├── profiler-venn.png ├── scheduler-complete.png ├── scheduler.1.png ├── scheduler.2.png ├── scheduler.3.png ├── scheduler.4.png ├── scheduler.gif ├── stack.gif ├── stack.go ├── stack1.png ├── stack2.png ├── stack3.png ├── time.1.png ├── time.2.png ├── time.3.png ├── timeline.png └── trace.out ├── heap.md ├── hello.trace ├── mutex.md ├── pprof.md ├── profile.png ├── profile.proto ├── profilers.excalidraw ├── scheduler.png ├── sim ├── block_sampling.ipynb ├── block_sampling_biased.png ├── block_sampling_debiased.png └── block_sampling_hist.png ├── stack-traces.excalidraw └── stack-traces.md /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | jobs: 10 | check-links: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@main 15 | - name: Markup Link Checker (mlc) 16 | uses: becheran/mlc@v0.15.4 17 | with: 18 | args: --no-web-links 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | venv 3 | *.pb.gz 4 | .ipynb_checkpoints 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-profiler-notes 2 | 3 | Hey there 👋🏻, I'm [felixge](https://github.com/felixge) and I've just started a new job at [Datadog](https://www.datadoghq.com/) to work on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. 4 | 5 | I found that Go has a lot of profilers and there are many tools for looking at the data, but that there is very little information on what any of it means. So in order to make sure that I know what I'm talking about, I've started to research the existing profilers and how they work. This repository is my attempt to summarize my findings in the hope that it might be useful to others. 6 | 7 | - [The Busy Developers's Guide to Go Profiling, Tracing and Observability](./guide/README.md) 8 | - [pprof tool & format](./pprof.md): Describes the pprof tool and the binary data format for storing profiles. 9 | - [Stack Traces](./stack-traces.md): An in-depth look at stack traces in Go which are a load bearing subsystem for profiling. 10 | - [Goroutine Profiling](./goroutine.md): Allows you to get a list of all active goroutines and what they're currently doing. 11 | - [Block Profiling](./block.md): Understand how much time your code spends waiting on channels and locks. 12 | - CPU Profiling (🚧 coming soon!) 13 | - Heap Profiling (🚧 coming soon!) 14 | - Mutex Profiling (🚧 coming soon!) 15 | - Wallclock Profiling (🚧 coming soon!) 16 | 17 | ## External Links 18 | 19 | - Go Docs 20 | - [Diagnostics](https://golang.org/doc/diagnostics.html): Has a very good overview over the available profiling and tracing facilities but doesn't go into a lot of depth. 21 | - [runtime/pprof](https://golang.org/pkg/runtime/pprof/#Profile): Lists the available profiles and has a little more explanation about what kind of data they produce. 22 | - [runtime](https://golang.org/pkg/runtime/): Has documentation on the various control knobs and pprof facilities, e.g. `MemProfileRate`. 23 | - [net/http/pprof](https://golang.org/src/net/http/pprof/pprof.go): Not a lot of docs, but diving into the code from there shows how the various profilers can be started/stopped on demand. 24 | - JBD 25 | - [Profiler labels in Go](https://rakyll.org/profiler-labels/): An introduction to using pprof labels and how they allow you to add additional context to your profiles. 26 | - [Custom pprof profiles](https://rakyll.org/custom-profiles/): Example for using custom profiles, shows tracking open/close events of a blob store and how to figure out how many blobs are open at a given time. 27 | - [Mutex profile](https://rakyll.org/mutexprofile/): Brief intro to the mutex profile. 28 | - [Using Instruments to profile Go programs](https://rakyll.org/instruments/): How to use the macOS Instruments app (I think it's built on dtrace) to profile Go programs. Not clear what the benfits are, if any. 29 | - [Profiling Go programs with pprof](https://jvns.ca/blog/2017/09/24/profiling-go-with-pprof/) by Julia Evans: A nice tour with a focus on heap profiling and the pprof output format. 30 | 31 | Got great links to recommend? Open an issue or PR, I'd happy to add your suggestions : ). 32 | 33 | ## Stargazers over time 34 | 35 | [![Stargazers over time](https://starchart.cc/DataDog/go-profiler-notes.svg)](https://starchart.cc/DataDog/go-profiler-notes) 36 | 37 | ## License 38 | 39 | The markdown files in this repository are licensed under the [CC BY-SA 4.0 license](https://creativecommons.org/licenses/by-sa/4.0/). 40 | 41 | ## Disclaimers 42 | 43 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ). 44 | 45 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! 46 | -------------------------------------------------------------------------------- /bench/README.md: -------------------------------------------------------------------------------- 1 | # Go Profiler Overheads 2 | 3 | This page is documenting the benchmark methodology used to analyze the performance overhead of the various go profilers. The results are discussed in the documents for each individual profiler. 4 | 5 | Benchmarking is done by invoking the Go program included in this directory. You can look at [run.sh](./run.sh) to see the current arguments that are being used, but here is an example for block profiling with two workloads and various profiling rates: 6 | 7 | ``` 8 | go run . \ 9 | -workloads mutex,chan \ 10 | -ops 100000 \ 11 | -blockprofilerates 0,1,10,100,1000,10000,100000,1000000 \ 12 | -runs 20 \ 13 | -depths 16 \ 14 | > "result.csv" 15 | ``` 16 | 17 | The benchmark works by spawning a new child process for the given number of `-runs` and every unique combination of parameters. The child reports the results to the parent process which then combines all the results in a CSV file. The hope is that using a new child process for every config/run eliminates scheduler, GC and other runtime state building up as a source of errors. 18 | 19 | Workloads are defined in the [workloads_chan.go](./workload_chan.go) and [workloads_mutex.go](./workload_mutex.go) files. For now the workloads are designed to be **pathological**, i.e. they try to show the worst performance impact the profiler might have on applications that are not doing anything useful other than stressing the profiler. The numbers are not intended to scare you away from profiling in production, but to guide you towards universally **safe profiling rates** as a starting point. 20 | 21 | The CSV files are visualized using the [analysis.ipynb](./analysis.ipynb) notebook that's included in this directory. 22 | 23 | For now the data is only collected from my local MacBook Pro machine (using docker for mac), but more realistic environments will be included in the future. But it's probably a good setup for finding pathological scenarios : ). 24 | 25 | ## Disclaimers 26 | 27 | I work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go (you should check it out) and they generously allowed me to do all this research and publish it. 28 | 29 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! -------------------------------------------------------------------------------- /bench/block_bufchan_linux_x86_64.csv: -------------------------------------------------------------------------------- 1 | workload,ops,goroutines,depth,bufsize,blockprofilerate,run,ms 2 | chan,100000,12,16,128,0,1,26.111602 3 | chan,100000,12,16,128,0,2,24.330099 4 | chan,100000,12,16,128,0,3,25.564551 5 | chan,100000,12,16,128,0,4,21.197282 6 | chan,100000,12,16,128,0,5,25.208467 7 | chan,100000,12,16,128,0,6,24.942660 8 | chan,100000,12,16,128,0,7,25.745805 9 | chan,100000,12,16,128,0,8,25.952320 10 | chan,100000,12,16,128,0,9,25.502639 11 | chan,100000,12,16,128,0,10,25.505424 12 | chan,100000,12,16,128,0,11,19.021869 13 | chan,100000,12,16,128,0,12,24.093446 14 | chan,100000,12,16,128,0,13,25.470600 15 | chan,100000,12,16,128,0,14,24.750474 16 | chan,100000,12,16,128,0,15,25.318373 17 | chan,100000,12,16,128,0,16,24.801701 18 | chan,100000,12,16,128,0,17,25.410303 19 | chan,100000,12,16,128,0,18,23.951828 20 | chan,100000,12,16,128,0,19,25.501909 21 | chan,100000,12,16,128,0,20,25.479501 22 | chan,100000,12,16,128,1,1,28.818918 23 | chan,100000,12,16,128,1,2,27.955490 24 | chan,100000,12,16,128,1,3,27.657836 25 | chan,100000,12,16,128,1,4,28.679312 26 | chan,100000,12,16,128,1,5,28.240668 27 | chan,100000,12,16,128,1,6,27.620967 28 | chan,100000,12,16,128,1,7,25.435637 29 | chan,100000,12,16,128,1,8,28.245240 30 | chan,100000,12,16,128,1,9,24.528563 31 | chan,100000,12,16,128,1,10,27.792067 32 | chan,100000,12,16,128,1,11,28.284682 33 | chan,100000,12,16,128,1,12,28.398926 34 | chan,100000,12,16,128,1,13,28.064085 35 | chan,100000,12,16,128,1,14,27.722551 36 | chan,100000,12,16,128,1,15,27.945585 37 | chan,100000,12,16,128,1,16,27.936740 38 | chan,100000,12,16,128,1,17,27.591487 39 | chan,100000,12,16,128,1,18,28.128410 40 | chan,100000,12,16,128,1,19,27.597430 41 | chan,100000,12,16,128,1,20,27.567145 42 | chan,100000,12,16,128,10,1,27.021942 43 | chan,100000,12,16,128,10,2,27.381591 44 | chan,100000,12,16,128,10,3,27.646931 45 | chan,100000,12,16,128,10,4,26.951227 46 | chan,100000,12,16,128,10,5,28.278806 47 | chan,100000,12,16,128,10,6,27.964904 48 | chan,100000,12,16,128,10,7,28.244240 49 | chan,100000,12,16,128,10,8,27.615860 50 | chan,100000,12,16,128,10,9,28.215444 51 | chan,100000,12,16,128,10,10,27.167347 52 | chan,100000,12,16,128,10,11,27.595361 53 | chan,100000,12,16,128,10,12,27.618797 54 | chan,100000,12,16,128,10,13,27.547148 55 | chan,100000,12,16,128,10,14,27.300065 56 | chan,100000,12,16,128,10,15,27.808355 57 | chan,100000,12,16,128,10,16,27.218818 58 | chan,100000,12,16,128,10,17,27.017832 59 | chan,100000,12,16,128,10,18,27.885908 60 | chan,100000,12,16,128,10,19,26.738916 61 | chan,100000,12,16,128,10,20,27.247155 62 | chan,100000,12,16,128,100,1,27.034945 63 | chan,100000,12,16,128,100,2,28.234691 64 | chan,100000,12,16,128,100,3,27.598459 65 | chan,100000,12,16,128,100,4,27.291299 66 | chan,100000,12,16,128,100,5,27.678947 67 | chan,100000,12,16,128,100,6,27.056659 68 | chan,100000,12,16,128,100,7,28.539225 69 | chan,100000,12,16,128,100,8,27.786019 70 | chan,100000,12,16,128,100,9,27.309631 71 | chan,100000,12,16,128,100,10,27.551574 72 | chan,100000,12,16,128,100,11,27.652476 73 | chan,100000,12,16,128,100,12,27.467752 74 | chan,100000,12,16,128,100,13,24.288803 75 | chan,100000,12,16,128,100,14,27.395383 76 | chan,100000,12,16,128,100,15,24.060403 77 | chan,100000,12,16,128,100,16,28.417372 78 | chan,100000,12,16,128,100,17,27.703919 79 | chan,100000,12,16,128,100,18,27.478132 80 | chan,100000,12,16,128,100,19,23.975185 81 | chan,100000,12,16,128,100,20,27.867348 82 | chan,100000,12,16,128,1000,1,27.918440 83 | chan,100000,12,16,128,1000,2,27.861850 84 | chan,100000,12,16,128,1000,3,26.594190 85 | chan,100000,12,16,128,1000,4,27.599727 86 | chan,100000,12,16,128,1000,5,27.658375 87 | chan,100000,12,16,128,1000,6,27.025214 88 | chan,100000,12,16,128,1000,7,27.803807 89 | chan,100000,12,16,128,1000,8,27.876512 90 | chan,100000,12,16,128,1000,9,27.634843 91 | chan,100000,12,16,128,1000,10,27.179596 92 | chan,100000,12,16,128,1000,11,27.866468 93 | chan,100000,12,16,128,1000,12,27.517947 94 | chan,100000,12,16,128,1000,13,27.595231 95 | chan,100000,12,16,128,1000,14,28.281941 96 | chan,100000,12,16,128,1000,15,27.365247 97 | chan,100000,12,16,128,1000,16,27.375731 98 | chan,100000,12,16,128,1000,17,27.552872 99 | chan,100000,12,16,128,1000,18,27.886964 100 | chan,100000,12,16,128,1000,19,27.468668 101 | chan,100000,12,16,128,1000,20,27.491871 102 | chan,100000,12,16,128,10000,1,27.505759 103 | chan,100000,12,16,128,10000,2,28.449661 104 | chan,100000,12,16,128,10000,3,27.502533 105 | chan,100000,12,16,128,10000,4,28.213946 106 | chan,100000,12,16,128,10000,5,28.175616 107 | chan,100000,12,16,128,10000,6,28.522483 108 | chan,100000,12,16,128,10000,7,23.045822 109 | chan,100000,12,16,128,10000,8,27.745446 110 | chan,100000,12,16,128,10000,9,28.315042 111 | chan,100000,12,16,128,10000,10,27.612150 112 | chan,100000,12,16,128,10000,11,27.724441 113 | chan,100000,12,16,128,10000,12,28.278531 114 | chan,100000,12,16,128,10000,13,27.564547 115 | chan,100000,12,16,128,10000,14,27.545289 116 | chan,100000,12,16,128,10000,15,28.498775 117 | chan,100000,12,16,128,10000,16,28.194233 118 | chan,100000,12,16,128,10000,17,28.042158 119 | chan,100000,12,16,128,10000,18,27.733760 120 | chan,100000,12,16,128,10000,19,23.489180 121 | chan,100000,12,16,128,10000,20,27.867968 122 | chan,100000,12,16,128,100000,1,28.688822 123 | chan,100000,12,16,128,100000,2,28.172282 124 | chan,100000,12,16,128,100000,3,28.431106 125 | chan,100000,12,16,128,100000,4,28.655976 126 | chan,100000,12,16,128,100000,5,23.034884 127 | chan,100000,12,16,128,100000,6,28.475143 128 | chan,100000,12,16,128,100000,7,27.949890 129 | chan,100000,12,16,128,100000,8,28.711835 130 | chan,100000,12,16,128,100000,9,28.264331 131 | chan,100000,12,16,128,100000,10,28.390331 132 | chan,100000,12,16,128,100000,11,27.687716 133 | chan,100000,12,16,128,100000,12,28.300156 134 | chan,100000,12,16,128,100000,13,28.252632 135 | chan,100000,12,16,128,100000,14,28.841992 136 | chan,100000,12,16,128,100000,15,29.123085 137 | chan,100000,12,16,128,100000,16,24.058373 138 | chan,100000,12,16,128,100000,17,28.251988 139 | chan,100000,12,16,128,100000,18,23.060083 140 | chan,100000,12,16,128,100000,19,28.803184 141 | chan,100000,12,16,128,100000,20,28.249340 142 | chan,100000,12,16,128,1000000,1,23.341275 143 | chan,100000,12,16,128,1000000,2,27.960480 144 | chan,100000,12,16,128,1000000,3,27.965699 145 | chan,100000,12,16,128,1000000,4,28.260114 146 | chan,100000,12,16,128,1000000,5,22.787201 147 | chan,100000,12,16,128,1000000,6,28.314806 148 | chan,100000,12,16,128,1000000,7,27.769908 149 | chan,100000,12,16,128,1000000,8,27.955299 150 | chan,100000,12,16,128,1000000,9,28.364142 151 | chan,100000,12,16,128,1000000,10,28.056026 152 | chan,100000,12,16,128,1000000,11,28.292778 153 | chan,100000,12,16,128,1000000,12,28.040546 154 | chan,100000,12,16,128,1000000,13,28.281895 155 | chan,100000,12,16,128,1000000,14,28.111810 156 | chan,100000,12,16,128,1000000,15,27.549470 157 | chan,100000,12,16,128,1000000,16,26.898413 158 | chan,100000,12,16,128,1000000,17,29.058733 159 | chan,100000,12,16,128,1000000,18,27.354946 160 | chan,100000,12,16,128,1000000,19,28.469475 161 | chan,100000,12,16,128,1000000,20,28.109963 162 | -------------------------------------------------------------------------------- /bench/block_darwin_x86_64.csv: -------------------------------------------------------------------------------- 1 | workload,ops,goroutines,depth,blockprofilerate,run,ms 2 | mutex,100000,12,16,0,1,42.438360 3 | mutex,100000,12,16,0,2,56.538516 4 | mutex,100000,12,16,0,3,52.666698 5 | mutex,100000,12,16,0,4,50.588831 6 | mutex,100000,12,16,0,5,42.993016 7 | mutex,100000,12,16,0,6,47.262495 8 | mutex,100000,12,16,0,7,42.614335 9 | mutex,100000,12,16,0,8,50.589187 10 | mutex,100000,12,16,0,9,53.232434 11 | mutex,100000,12,16,0,10,64.352130 12 | mutex,100000,12,16,0,11,52.387284 13 | mutex,100000,12,16,0,12,65.223335 14 | mutex,100000,12,16,0,13,50.572637 15 | mutex,100000,12,16,0,14,49.732913 16 | mutex,100000,12,16,0,15,46.789851 17 | mutex,100000,12,16,0,16,42.016991 18 | mutex,100000,12,16,0,17,50.339128 19 | mutex,100000,12,16,0,18,52.286547 20 | mutex,100000,12,16,0,19,50.750858 21 | mutex,100000,12,16,0,20,62.743093 22 | mutex,100000,12,16,1,1,43.440872 23 | mutex,100000,12,16,1,2,49.936582 24 | mutex,100000,12,16,1,3,50.284105 25 | mutex,100000,12,16,1,4,58.646662 26 | mutex,100000,12,16,1,5,63.336982 27 | mutex,100000,12,16,1,6,63.688515 28 | mutex,100000,12,16,1,7,63.627604 29 | mutex,100000,12,16,1,8,39.704634 30 | mutex,100000,12,16,1,9,43.121722 31 | mutex,100000,12,16,1,10,46.115743 32 | mutex,100000,12,16,1,11,50.286809 33 | mutex,100000,12,16,1,12,47.840831 34 | mutex,100000,12,16,1,13,62.910580 35 | mutex,100000,12,16,1,14,51.057369 36 | mutex,100000,12,16,1,15,54.046803 37 | mutex,100000,12,16,1,16,48.035078 38 | -------------------------------------------------------------------------------- /bench/block_linux_x86_64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/bench/block_linux_x86_64.png -------------------------------------------------------------------------------- /bench/csv.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type Record struct { 9 | Blockprofilerate int 10 | Bufsize int 11 | Depth int 12 | Duration time.Duration 13 | Goroutines int 14 | Ops int 15 | Run int 16 | Workload string 17 | } 18 | 19 | type Column struct { 20 | Name string 21 | MarshalValue func(*Record) (string, error) 22 | } 23 | 24 | var Columns = []Column{ 25 | {"workload", func(r *Record) (string, error) { 26 | return fmt.Sprintf("%s", r.Workload), nil 27 | }}, 28 | {"ops", func(r *Record) (string, error) { 29 | return fmt.Sprintf("%d", r.Ops), nil 30 | }}, 31 | {"goroutines", func(r *Record) (string, error) { 32 | return fmt.Sprintf("%d", r.Goroutines), nil 33 | }}, 34 | {"depth", func(r *Record) (string, error) { 35 | return fmt.Sprintf("%d", r.Depth), nil 36 | }}, 37 | {"bufsize", func(r *Record) (string, error) { 38 | return fmt.Sprintf("%d", r.Bufsize), nil 39 | }}, 40 | {"blockprofilerate", func(r *Record) (string, error) { 41 | return fmt.Sprintf("%d", r.Blockprofilerate), nil 42 | }}, 43 | {"run", func(r *Record) (string, error) { 44 | return fmt.Sprintf("%d", r.Run), nil 45 | }}, 46 | {"ms", func(r *Record) (string, error) { 47 | return fmt.Sprintf("%f", r.Duration.Seconds()*1000), nil 48 | }}, 49 | } 50 | 51 | func (r *Record) MarshalRecord() ([]string, error) { 52 | record := make([]string, len(Columns)) 53 | for i, col := range Columns { 54 | val, err := col.MarshalValue(r) 55 | if err != nil { 56 | return nil, err 57 | } 58 | record[i] = val 59 | } 60 | return record, nil 61 | } 62 | 63 | func Headers() []string { 64 | headers := make([]string, len(Columns)) 65 | for i, col := range Columns { 66 | headers[i] = col.Name 67 | } 68 | return headers 69 | } 70 | -------------------------------------------------------------------------------- /bench/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "runtime" 11 | "runtime/pprof" 12 | "strconv" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | func main() { 18 | if err := run(); err != nil { 19 | fmt.Fprintln(os.Stderr, err) 20 | os.Exit(1) 21 | } 22 | } 23 | 24 | func run() error { 25 | if os.Getenv("WORKER") == "" { 26 | return leader() 27 | } else { 28 | return worker() 29 | } 30 | } 31 | 32 | func leader() error { 33 | var ( 34 | blockprofilerates = flagIntSlice("blockprofilerates", []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}, "The runtime.SetBlockProfileRate() values to benchmark.") 35 | bufsizes = flagIntSlice("bufsizes", []int{0, 64}, "The buffer sizes to use for channel operations (not applicable to all workloads).") 36 | depths = flagIntSlice("depths", []int{2, 4, 8, 16, 32}, "The different frame depths values to use for each workload.") 37 | goroutines = flagIntSlice("goroutines", []int{runtime.NumCPU()}, "The number of goroutine values to use for each workloads.") 38 | ops = flag.Int("ops", 1000, "The number of operations to perform for each workload.") 39 | runs = flag.Int("runs", 3, "The number of times to repeat the same benchmark to understand variance.") 40 | workloads = flagStringSlice("workloads", []string{"mutex", "chan"}, "The workloads to benchmark.") 41 | ) 42 | flag.Parse() 43 | 44 | cw := csv.NewWriter(os.Stdout) 45 | cw.Write(Headers()) 46 | cw.Flush() 47 | 48 | for _, workload := range *workloads { 49 | for _, goroutine := range *goroutines { 50 | for _, blockprofilerate := range *blockprofilerates { 51 | for _, depth := range *depths { 52 | for _, bufsize := range *bufsizes { 53 | for run := 1; run <= *runs; run++ { 54 | cmd := exec.Command(os.Args[0], 55 | "-run", fmt.Sprintf("%d", run), 56 | "-blockprofilerate", fmt.Sprintf("%d", blockprofilerate), 57 | "-ops", fmt.Sprintf("%d", *ops), 58 | "-goroutines", fmt.Sprintf("%d", goroutine), 59 | "-depth", fmt.Sprintf("%d", depth), 60 | "-bufsize", fmt.Sprintf("%d", bufsize), 61 | "-workload", workload, 62 | ) 63 | 64 | buf := &bytes.Buffer{} 65 | cmd.Stdout = buf 66 | cmd.Stderr = os.Stderr 67 | cmd.Env = append(cmd.Env, "WORKER=yeah") 68 | 69 | if err := cmd.Run(); err != nil { 70 | return err 71 | } 72 | 73 | buf.WriteTo(os.Stdout) 74 | } 75 | } 76 | } 77 | } 78 | } 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func worker() error { 85 | var ( 86 | blockprofilerate = flag.Int("blockprofilerate", 1, "The block profile rate to use.") 87 | bufsize = flag.Int("bufsize", 0, "The buffer size to use for channel operations (not applicable to all workloads).") 88 | depth = flag.Int("depth", 16, "The stack depth at which to perform blocking events.") 89 | goroutines = flag.Int("goroutines", runtime.NumCPU(), "The number of goroutines to utilize.") 90 | ops = flag.Int("ops", 100000, "The number of operations to perform.") 91 | out = flag.String("blockprofile", "", "Path to a file for writing the block profile.") 92 | run = flag.Int("run", 1, "The number of run. Has no impact on the benchmark, but gets included in the csv output line.") 93 | workload = flag.String("workload", "mutex", "The workload to simulate.") 94 | ) 95 | flag.Parse() 96 | 97 | if *blockprofilerate > 0 { 98 | runtime.SetBlockProfileRate(*blockprofilerate) 99 | } 100 | 101 | start := time.Now() 102 | switch *workload { 103 | case "mutex": 104 | if err := mutexWorkload(*goroutines, *ops, *depth); err != nil { 105 | return err 106 | } 107 | case "chan": 108 | if err := chanWorkload(*goroutines, *ops, *depth, *bufsize); err != nil { 109 | return err 110 | } 111 | default: 112 | return fmt.Errorf("unknown workload: %q", *workload) 113 | } 114 | duration := time.Since(start) 115 | 116 | if *blockprofilerate > 0 && *out != "" { 117 | f, err := os.Create(*out) 118 | if err != nil { 119 | return err 120 | } 121 | defer f.Close() 122 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil { 123 | return err 124 | } 125 | } 126 | 127 | cw := csv.NewWriter(os.Stdout) 128 | record, err := (&Record{ 129 | Blockprofilerate: *blockprofilerate, 130 | Bufsize: *bufsize, 131 | Depth: *depth, 132 | Duration: duration, 133 | Goroutines: *goroutines, 134 | Ops: *ops, 135 | Run: *run, 136 | Workload: *workload, 137 | }).MarshalRecord() 138 | if err != nil { 139 | return err 140 | } 141 | cw.Write(record) 142 | cw.Flush() 143 | return cw.Error() 144 | } 145 | 146 | func atStackDepth(depth int, fn func()) { 147 | pcs := make([]uintptr, depth*10) 148 | n := runtime.Callers(1, pcs) 149 | if n > depth { 150 | panic("depth exceeded") 151 | } else if n < depth { 152 | atStackDepth(depth, fn) 153 | return 154 | } 155 | 156 | fn() 157 | } 158 | 159 | func flagIntSlice(name string, value []int, usage string) *[]int { 160 | val := &intSlice{vals: value} 161 | flag.Var(val, name, usage) 162 | return &val.vals 163 | } 164 | 165 | type intSlice struct { 166 | vals []int 167 | } 168 | 169 | func (i *intSlice) Set(val string) error { 170 | var vals []int 171 | for _, val := range strings.Split(val, ",") { 172 | num, err := strconv.Atoi(val) 173 | if err != nil { 174 | return err 175 | } 176 | vals = append(vals, num) 177 | } 178 | i.vals = vals 179 | return nil 180 | } 181 | 182 | func (i *intSlice) String() string { 183 | return fmt.Sprintf("%v", i.vals) 184 | } 185 | 186 | func flagStringSlice(name string, value []string, usage string) *[]string { 187 | val := &strSlice{vals: value} 188 | flag.Var(val, name, usage) 189 | return &val.vals 190 | } 191 | 192 | type strSlice struct { 193 | vals []string 194 | } 195 | 196 | func (s *strSlice) Set(val string) error { 197 | s.vals = strings.Split(val, ",") 198 | return nil 199 | } 200 | 201 | func (s *strSlice) String() string { 202 | return fmt.Sprintf("%v", s.vals) 203 | } 204 | -------------------------------------------------------------------------------- /bench/run.sh: -------------------------------------------------------------------------------- 1 | os_arch() { 2 | echo "$(uname | tr '[:upper:]' '[:lower:]')_$(uname -m)" 3 | } 4 | 5 | #go run . \ 6 | #-workloads mutex,chan \ 7 | #-ops 100000 \ 8 | #-blockprofilerates 0,1,10,100,1000,10000,100000,1000000 \ 9 | #-runs 20 \ 10 | #-bufsizes 0 \ 11 | #-depths 16 \ 12 | #> "block_$(os_arch).csv" 13 | 14 | go run . \ 15 | -workloads chan \ 16 | -ops 100000 \ 17 | -blockprofilerates 0,1,10,100,1000,10000,100000,1000000 \ 18 | -bufsizes 128 \ 19 | -runs 20 \ 20 | -depths 16 \ 21 | > "block_bufchan_$(os_arch).csv" 22 | -------------------------------------------------------------------------------- /bench/workload_chan.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func chanWorkload(goroutines, ops, depth, bufsize int) error { 9 | if goroutines%2 != 0 { 10 | return fmt.Errorf("bad goroutines: %d: must be a multiple of 2", goroutines) 11 | } 12 | 13 | wg := &sync.WaitGroup{} 14 | for j := 0; j < goroutines/2; j++ { 15 | ch := make(chan struct{}, bufsize) 16 | wg.Add(1) 17 | go atStackDepth(depth, func() { 18 | defer wg.Done() 19 | for i := 0; i < ops; i++ { 20 | ch <- struct{}{} 21 | } 22 | }) 23 | wg.Add(1) 24 | go atStackDepth(depth, func() { 25 | defer wg.Done() 26 | for i := 0; i < ops; i++ { 27 | <-ch 28 | } 29 | }) 30 | } 31 | wg.Wait() 32 | return nil 33 | } 34 | -------------------------------------------------------------------------------- /bench/workload_mutex.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | ) 7 | 8 | func mutexWorkload(goroutines, ops, depth int) error { 9 | if goroutines%2 != 0 { 10 | return fmt.Errorf("bad goroutines: %d: must be a multiple of 2", goroutines) 11 | } 12 | 13 | wg := &sync.WaitGroup{} 14 | for j := 0; j < goroutines/2; j++ { 15 | m := &sync.Mutex{} 16 | wg.Add(1) 17 | go atStackDepth(depth, func() { 18 | defer wg.Done() 19 | for i := 0; i < ops; i++ { 20 | m.Lock() 21 | m.Unlock() 22 | } 23 | }) 24 | wg.Add(1) 25 | go atStackDepth(depth, func() { 26 | defer wg.Done() 27 | for i := 0; i < ops; i++ { 28 | m.Lock() 29 | m.Unlock() 30 | } 31 | }) 32 | } 33 | wg.Wait() 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /block-bias.md: -------------------------------------------------------------------------------- 1 | ⬅ [Block Profiling in Go](./block.md) 2 | 3 | ⚠️This document describes a sampling bias issue I discovered while researching the block profiler for Go. I have since [landed a fix for it](https://go-review.googlesource.com/c/go/+/299991) that should appear in Go 1.17. 4 | 5 | # Block Profiler Sampling Bias 6 | 7 | **tl;dr:** Setting your sampling `rate` too high will bias your results towards infrequent long events over frequent short events. 8 | 9 | As described in the [Usage](#usage) section, the block profiler will sample as follows: 10 | 11 | - Events with `duration >= rate` will be sampled 100% 12 | - Events with `duration < rate` have a `duration / rate` chance of getting sampled. 13 | 14 | The [implementation](https://github.com/golang/go/blob/go1.15.7/src/runtime/mprof.go#L408) for this looks like that: 15 | 16 | ```go 17 | func blocksampled(cycles int64) bool { 18 | rate := int64(atomic.Load64(&blockprofilerate)) 19 | if rate <= 0 || (rate > cycles && int64(fastrand())%rate > cycles) { 20 | return false 21 | } 22 | return true 23 | } 24 | ``` 25 | 26 | This means that if you set your profiling `rate` low enough, you'll get very accurate results. However, if your `rate` is higher than the `duration` of some of the events you are sampling, the sampling process will exhibit a bias favoring infrequent events of higher `duration` over frequent events with lower `duration` even so they may contribute to the same amount of overall block duration in your program. 27 | 28 | ## Simple Example 29 | 30 | Let's say your `blockprofilerate` is `100ns` and your application produces the following events: 31 | 32 | - `A`: `1` event with a duration of `100ns`. 33 | - `B`: `10` events with a duration of `10ns` each. 34 | 35 | Given this scenario, the `blockprofiler` is guaranteed to catch and accurately report event `A` as `100ns` in the profile. For event `B` the most likely outcome is that the profiler will capture only a single event (10% of 10 events) and report `B` as `10ns` in the profile. So you might find yourself in a situation where you think event `A` is causing 10x more blocking than event `B`, which is not true. 36 | 37 | ## Simulation & Proposal for Improvement 38 | 39 | For an even better intuition about this, consider the [simulated example](./sim/block_sampling.ipynb) below. Here we have a histogram of all durations collected from 3 types of blocking events. As you can see, they all have different mean durations (`1000ns`, `2000ns`, `3000ns`) and they are occurring at different frequencies, with `count(a) > count(b) > count(c)`. What's more difficult to see, is that the cumulative durations of these events are the same, i.e. `sum(a) = sum(b) = sum(c)`, but you can trust me on that : ). 40 | 41 | 42 | 43 | So given that your application might produce events like this, how will they show up in your block profile as you try out different `blockprofilerate` values? As you can see below, all is well and fine until a `blockprofilerate` of `1000ns`. Each event shows up with the same total duration in the profile (the red and green dots are hidden below the blue ones). However starting at `1000ns` you see that event `a` starts to fade from our profile and at `2000ns` you'd already think that events `b` and `c` are causing twice as much blocking time as event `a`. 44 | 45 | 46 | 47 | So what can we do? Do we always need to live in fear of bias when working with block profiles? No! If the [Overhead](#overhead) for your workload allows it, the simplest solution is to use a low enough `blockprofilerate` in order to capture most blocking events. 48 | 49 | But perhaps there is an even better way. I'm thinking we could correct for the current bias by keeping the same logic of sampling `duration / rate` fraction of events when `duration < rate`. However, when this happens we could simply multiply the sampled duration by `rate/duration` like this: 50 | 51 | ``` 52 | duration = duration * (rate/duration) 53 | # note: the expression above can be simplified to just `duration = rate` 54 | ``` 55 | 56 | Doing so could be done with a [trivial patch](https://github.com/felixge/go/compare/master...debias-blockprofile-rate) to the go runtime and the picture below shows the results from simulating it. 57 | 58 | 59 | 60 | ## Disclaimers 61 | 62 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ). 63 | 64 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! 65 | -------------------------------------------------------------------------------- /cpu.md: -------------------------------------------------------------------------------- 1 | 🚧 This note is still work in progress, please come back later! 🚧 2 | 3 | # CPU Profiling in Go 4 | 5 | ## How it works 6 | 7 | Go's CPU profiler works by sampling the profiled program at a configurable rate of 100 Hz. A sample consists mostly of the stack trace of the goroutine that was interrupted. Below is an example of a program that has a function `A()` that sometimes calls function `C()` and `B()` which doesn't call anything. 8 | 9 | ``` 10 | ------------------------------------> time 11 | | | | | | | | | | | | | 12 | A B A B A A A B A B A A 13 | C C C 14 | 15 | <--> 16 | 10ms 17 | ``` 18 | 19 | The resulting [pprof output](./pprof.md) doesn't contain the raw sample events. Instead it aggregates all samples and reports the **samples/count** and **cpu/nanoseconds** values. The latter is [derived](https://github.com/golang/go/blob/go1.15.6/src/runtime/pprof/proto.go#L354) by multiplying the sample count with the sample period and is therefore redundant. I suspect it's included because pprof doesnt support the count to time conversion, but if anybody has the full story please let me know! 20 | 21 | Back to the example above, the pprof output file would containing the following: 22 | 23 | | stack trace | samples/count | cpu/nanoseconds | 24 | | ----------- | ------------- | --------------- | 25 | | A | 5 | 50000000 | 26 | | A C | 3 | 30000000 | 27 | | B | 4 | 40000000 | 28 | 29 | Please note that the **stack trace** is a bit more complex in reality as it includes the exact program counter address at which your program was sampled as well as the line number in your code that it maps to. This information is usually omitted when visualizing the data as a [Flame Graph](http://www.brendangregg.com/flamegraphs.html), but can be very useful when drilling down into the details. Speaking of flame graphs, here is what the data above looks like when visualized: 30 | 31 | ![Flame Graph for data above](./flame-abc.png) 32 | 33 | ## Usage 34 | 35 | The various ways one can record CPU profiles in Go are listed below. 36 | 37 | 1. The `go test` command has a `-cpuprofile` flag described that can be used to profile the benchmarks in your test suite like shown below. You can also [add a flag like this](https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program) to you own programs which might makes sense for command line tools. 38 | 39 | ``` 40 | go test -cpuprofile benchmark.cpu.pb.gz -bench . 41 | go tool pprof -http=:6061 benchmark.cpu.pb.gz 42 | ``` 43 | 44 | 2. The [net/http/pprof](https://golang.org/pkg/net/http/pprof/) allows you to setup http endpoints that can start/stop the CPU profiler via http requests on-demand and return the resulting pprof data file. You can directly pass a URL to such an endpoint to the pprof tool. 45 | 46 | ``` 47 | go tool pprof -http=:6061 http://localhost:6060/debug/pprof/profile?seconds=30 48 | ``` 49 | 50 | 3. You can programmatically start/stop the profiler via the [runtime/pprof API](https://golang.org/pkg/runtime/pprof/#StartCPUProfile). 51 | 52 | ```go 53 | pprof.StartCPUProfile(outputFile) 54 | defer pprof.StopCPUProfile() 55 | // 56 | ``` 57 | 58 | 4. You can use a service such as [Datadog's Continious Profiler](https://www.datadoghq.com/product/code-profiling/) to automatically take and upload CPU profiles of your applications running in production. Disclaimer: I just started working for Datadog to take Go profiling to the next level. 59 | 60 | ## Use Cases 61 | 62 | - Break down CPU utilization by functions or even line numbers. 63 | - Break down CPU utilization by user-dimensions (endpoint, request_id, etc.) using [profiler labels](https://rakyll.org/profiler-labels/). 64 | 65 | ## Anti-Use Cases 66 | 67 | - Understanding [Off-CPU](http://www.brendangregg.com/offcpuanalysis.html) time your program spends while waiting on I/O, Locks, Timers, Scheduling, etc. 68 | - High-resolution sampling. The default rate is 100 Hz and you won't be able to go much higher for reasons outlined later on. 69 | 70 | ## Todo 71 | 72 | - Talk about multi threading. 73 | - setitimer 74 | - Profiler labels 75 | - bias 76 | - performance overhead (including negative!) 77 | - https://github.com/golang/go/issues/38325 78 | - accuracy 79 | - https://github.com/golang/go/issues/35057 80 | - Discuss [Proposal: hardware performance counters for CPU profiling.](https://go.googlesource.com/proposal/+/refs/changes/08/219508/2/design/36821-perf-counter-pprof.md) 81 | 82 | ## Disclaimers 83 | 84 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ). 85 | 86 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! 87 | -------------------------------------------------------------------------------- /datadog.md: -------------------------------------------------------------------------------- 1 | # Datadog's Go Profiler 2 | 3 | The profiler code is a sub-package in the [dd-trace-go](https://github.com/DataDog/dd-trace-go/tree/v1/profiler) repo. Basic integration is described in the [Getting Started](https://docs.datadoghq.com/tracing/profiler/getting_started/?tab=go) guide, and the [package docs](https://pkg.go.dev/gopkg.in/DataDog/dd-trace-go.v1/profiler#pkg-constants) explain additional API options. The minimum required Go version is 1.12. 4 | 5 | Users invoke the profiler by calling `profiler.Start()` with a few options, especially tags for identifying the source of the data. Every 60s the profiler then takes a 15s CPU profile as well as a heap profile and uploads it using the [payload format](#payload-format) described below. 6 | 7 | The data is sent to the local [datadog agent](https://docs.datadoghq.com/agent/) which forwards it to Datadog's backend. It's also possible to directly upload the profiles without an agent using an API key is given, but this method is deprecated and not supported. 8 | 9 | ## Operation Details 10 | 11 | The [`Start()`](https://pkg.go.dev/gopkg.in/DataDog/dd-trace-go.v1/profiler#Start) function invokes two background goroutines. The `collect()` routine captures profiling data as a `batch` struct and puts it into a Go channel for the `send()` routine to read. More details can be seen below. 12 | 13 | ![datadog go profiler](./datadog.png) 14 | 15 | A `batch` has a `start` and an `end` time as well as a slice of `profiles`. Each profile corresponds to one of the supported [profile type](https://pkg.go.dev/gopkg.in/DataDog/dd-trace-go.v1/profiler#ProfileType), e.g. CPU, Heap, etc.. 16 | 17 | ## Payload Format 18 | 19 | TODO: This is outdated, needs updating [based on this PR](https://github.com/DataDog/dd-trace-go/pull/781). 20 | 21 | The payload uses `multipart/form-data` encoding and includes the following form fields for every `batch` that is being uploaded. 22 | 23 | - `format`: Always `pprof` 24 | - `runtime`: Always `go` 25 | - `recording-start`: The batch start time formatted as `2006-01-02T15:04:05Z07:00` in UTC. 26 | - `recording-end`: The batch end time formatted as `2006-01-02T15:04:05Z07:00` in UTC. 27 | - `tags[]`: The profiler's `p.cfg.tags` + `service:p.cfg.service` + `env:p.cfg.env` + `host:bat.host` (if set) + `runtime:go` 28 | - `types[0..n]`: The comma separates types included in each profile, e.g. `alloc_objects,alloc_space,inuse_objects,inuse_space`. 29 | - `data[0..n]`: One file field for each profile. The filename is always `pprof-data`, and the pprof data is compressed (by Go). 30 | 31 | TODO: Link to a sample payload file. 32 | 33 | ## Disclaimers 34 | 35 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ). 36 | 37 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! -------------------------------------------------------------------------------- /datadog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/datadog.png -------------------------------------------------------------------------------- /delve/stackannotate.star: -------------------------------------------------------------------------------- 1 | word_size = 8 # 64bit only for now 2 | 3 | def get_reg(name): 4 | for req in registers().Regs: 5 | if req.Name == name: 6 | return int(req.Value, 16) 7 | 8 | def read_word(addr): 9 | v = 0 10 | m = examine_memory(addr, word_size) 11 | s = 1 12 | for b in m.Mem: 13 | v += b * s 14 | s = s * 256 15 | return v 16 | 17 | def hex(d, n = 0): 18 | if d == 0: 19 | return lpad("0", n, "0") 20 | 21 | lookup = {10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "e", 16: "f"} 22 | s = "" 23 | while d > 0: 24 | r = d % 16 25 | d = d // 16 26 | if r >= 10: 27 | r = lookup[r] 28 | else: 29 | r = str(r) 30 | s = r + s 31 | return lpad(s, n, "0") 32 | 33 | def lpad(s, n, c = " "): 34 | while len(s) < n: 35 | s = c + s 36 | return s 37 | 38 | def rpad(s, n, c = " "): 39 | while len(s) < n: 40 | s = s + c 41 | return s 42 | 43 | def ascii_table(rows, align_right = {}): 44 | widths = [] 45 | for row in rows: 46 | for i, col in enumerate(row): 47 | if len(widths) < i+1: 48 | widths.append(0) 49 | widths[i] = max(widths[i], len(col)) 50 | 51 | s = "" 52 | for row in rows: 53 | for i, col in enumerate(row): 54 | width = widths[i] 55 | if align_right.get(i, False): 56 | col = lpad(col, width) 57 | else: 58 | col = rpad(col, width) 59 | s += col + " " 60 | s += "\n" 61 | return s 62 | 63 | def getg(): 64 | # TODO(fg) there is probably a better way to implement this. 65 | g = raw_command("goroutine").State.SelectedGoroutine 66 | for gp in eval(None, "runtime.allgs").Variable.Value: 67 | if gp.goid == g.ID: 68 | return gp 69 | 70 | def stack(): 71 | g = getg() 72 | bp = get_reg("Rbp") 73 | ip = get_reg("Rip") 74 | sp = get_reg("Rsp") 75 | regs = {"sp": sp, "bp": bp, "ip": ip} 76 | 77 | addr_list = [] 78 | addr_dict = {} 79 | offset = 0 80 | while True: 81 | addr = g.stack.hi+offset-word_size 82 | addr_info = { 83 | "addr": addr, 84 | "val": read_word(addr), 85 | "offset": offset, 86 | "regs": [], 87 | "note": [], 88 | "func": None, 89 | "arg": None, 90 | "local": None, 91 | "fp": False, 92 | } 93 | for (name, val) in regs.items(): 94 | if addr == val: 95 | addr_info["regs"].append(name) 96 | 97 | addr_list.append(addr_info) 98 | addr_dict[addr] = addr_info 99 | offset -= word_size 100 | if addr <= sp: 101 | break 102 | 103 | for f in stacktrace(g.goid, 128, True).Locations: 104 | fp_addr = g.stack.hi+f.FramePointerOffset 105 | if fp_addr > 0 and len(addr_dict[fp_addr]["note"]) == 0: 106 | addr_dict[fp_addr]["note"].append("frame pointer for "+f.Function.Name_) 107 | pc_addr = fp_addr+word_size 108 | pc = read_word(pc_addr) 109 | ins = disassemble(None, pc, pc+1).Disassemble[0] 110 | addr_dict[pc_addr]["note"].append("return addr to "+ins.Loc.Function.Name_) 111 | 112 | for arg in f.Arguments: 113 | addr_dict[arg.Addr]["note"].append("arg "+arg.Name+" "+arg.Type) 114 | for local in f.Locals: 115 | addr = local.Addr // 8 * 8 116 | if addr_dict.get(addr): 117 | addr_dict[addr]["note"].append("var "+local.Name+" "+local.Type) 118 | 119 | return addr_list 120 | 121 | # stackannotate (alias sa) will print an annotated stack dump. 122 | def command_stackannotate(): 123 | rows = [["regs", "addr", "offset", "value", "explanation"]] 124 | for addr_info in stack(): 125 | regs = "" 126 | if len(addr_info["regs"]) > 0: 127 | regs = ",".join(addr_info["regs"])+" -->" 128 | 129 | note = "?" 130 | if len(addr_info["note"]) > 0: 131 | note = ", ".join(addr_info["note"]) 132 | 133 | rows.append([ 134 | regs, 135 | hex(addr_info["addr"]), 136 | lpad(str(addr_info["offset"]), 6), 137 | lpad(hex(addr_info["val"]), word_size*2+2), 138 | note, 139 | ]) 140 | print(ascii_table(rows)) 141 | 142 | def main(): 143 | dlv_command("config alias stackannotate sa") 144 | -------------------------------------------------------------------------------- /examples/block-bias/block.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block-bias/block.pb.gz -------------------------------------------------------------------------------- /examples/block-bias/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/block-bias 2 | 3 | go 1.15 4 | 5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 6 | -------------------------------------------------------------------------------- /examples/block-bias/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /examples/block-bias/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "runtime/pprof" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | func main() { 13 | if err := run(); err != nil { 14 | fmt.Fprintln(os.Stderr, err) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | var ( 20 | fastEventDuration = 1 * time.Millisecond 21 | slowEventDuration = 1 * fastEventDuration 22 | ) 23 | 24 | func run() error { 25 | runtime.SetBlockProfileRate(int(slowEventDuration.Nanoseconds())) 26 | 27 | var ( 28 | done = make(chan struct{}) 29 | wg = &sync.WaitGroup{} 30 | ) 31 | wg.Add(1) 32 | go func() { 33 | defer wg.Done() 34 | slowEvent(done) 35 | }() 36 | wg.Add(1) 37 | go func() { 38 | defer wg.Done() 39 | fastEvent(done) 40 | }() 41 | time.Sleep(1 * time.Second) 42 | close(done) 43 | wg.Wait() 44 | 45 | f, err := os.Create("block.pb.gz") 46 | if err != nil { 47 | return err 48 | } 49 | defer f.Close() 50 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil { 51 | return err 52 | } 53 | return nil 54 | } 55 | 56 | func slowEvent(done chan struct{}) error { 57 | return simulateBlockEvents(slowEventDuration, done) 58 | } 59 | 60 | func fastEvent(done chan struct{}) error { 61 | return simulateBlockEvents(fastEventDuration, done) 62 | } 63 | 64 | func simulateBlockEvents(duration time.Duration, done chan struct{}) error { 65 | ticker := time.NewTicker(duration) 66 | defer ticker.Stop() 67 | for { 68 | select { 69 | case <-ticker.C: 70 | // do nothing 71 | case <-done: 72 | return nil 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /examples/block-net/README.md: -------------------------------------------------------------------------------- 1 | # block-net 2 | 3 | This [program](./main.go) explores the [question](https://twitter.com/rogpeppe/status/1359202847708037124) whether network i/o (e.g. waiting on socket read/write operations) will show up in the [block profiler](../../block.md) or not. 4 | 5 | The program does the following: 6 | 7 | 1. Start a TCP server via `net.Listen()` and wait for a client using `l.Accept()`. 8 | 2. Sleep for 1s 9 | 3. Connect to the TCP server with a client using `net.Dial()`. 10 | 4. Let the server wait for a message from the client using `conn.Read()` 11 | 4. Sleep for 1s 12 | 5. Send a `"hello world"` message from the client to the server using `conn.Write()` 13 | 6. Server prints the message it received 14 | 7. Program exits 15 | 16 | Given this program and assuming the block profiler captures network i/o, we'd expect to capture two major blocking events: 17 | 18 | 1. `l.Accept()` waiting for 1s before a client connects. 19 | 2. `conn.Read()` waiting for 1s before receiving a message from the client 20 | 21 | However, as you can see below, the block profiler [captures](./block.pb.gz) only `~12ms` of activity related to channel operations involved in the listen/dial process. The accept/read activity however remains completely invisible. 22 | 23 | ![block-net](./block-net.png) 24 | 25 | This means that [block profiler](../../block.md) is generally not able to give a good idea about goroutines that are waiting on network i/o. 26 | 27 | -------------------------------------------------------------------------------- /examples/block-net/block-net.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block-net/block-net.png -------------------------------------------------------------------------------- /examples/block-net/block.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block-net/block.pb.gz -------------------------------------------------------------------------------- /examples/block-net/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/block-net 2 | 3 | go 1.15 4 | 5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 6 | -------------------------------------------------------------------------------- /examples/block-net/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /examples/block-net/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os" 7 | "runtime" 8 | "runtime/pprof" 9 | "time" 10 | 11 | "golang.org/x/sync/errgroup" 12 | ) 13 | 14 | func main() { 15 | if err := run(); err != nil { 16 | fmt.Fprintln(os.Stderr, err) 17 | os.Exit(1) 18 | } 19 | } 20 | 21 | func run() error { 22 | runtime.SetBlockProfileRate(1) 23 | 24 | listening := make(chan struct{}) 25 | g := &errgroup.Group{} 26 | g.Go(func() error { 27 | return server(listening) 28 | }) 29 | 30 | <-listening 31 | time.Sleep(time.Second) 32 | 33 | if err := client(); err != nil { 34 | return err 35 | } 36 | if err := g.Wait(); err != nil { 37 | return err 38 | } 39 | 40 | f, err := os.Create("block.pb.gz") 41 | if err != nil { 42 | return err 43 | } 44 | defer f.Close() 45 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil { 46 | return err 47 | } 48 | return nil 49 | } 50 | 51 | func server(listening chan struct{}) error { 52 | l, err := net.Listen("tcp", "localhost:9090") 53 | if err != nil { 54 | return err 55 | } 56 | 57 | listening <- struct{}{} 58 | 59 | conn, err := l.Accept() 60 | if err != nil { 61 | return err 62 | } 63 | fmt.Printf("[server] accepted: %v\n", conn) 64 | buf := make([]byte, 1024) 65 | n, err := conn.Read(buf) 66 | if err != nil { 67 | return err 68 | } 69 | buf = buf[:n] 70 | fmt.Printf("[server] read: %s\n", buf) 71 | return nil 72 | } 73 | 74 | func client() error { 75 | conn, err := net.Dial("tcp", "localhost:9090") 76 | if err != nil { 77 | return err 78 | } 79 | defer conn.Close() 80 | 81 | time.Sleep(time.Second) 82 | 83 | fmt.Printf("[client] connected: %v\n", conn) 84 | _, err = conn.Write([]byte("hello world")) 85 | return err 86 | } 87 | -------------------------------------------------------------------------------- /examples/block-sample/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/block-sample 2 | 3 | go 1.15 4 | 5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 6 | -------------------------------------------------------------------------------- /examples/block-sample/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /examples/block-sample/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "runtime" 8 | "runtime/pprof" 9 | "time" 10 | 11 | "golang.org/x/sync/errgroup" 12 | ) 13 | 14 | func main() { 15 | if err := run(); err != nil { 16 | fmt.Fprintln(os.Stderr, err) 17 | os.Exit(1) 18 | } 19 | } 20 | 21 | func run() error { 22 | labels := pprof.Labels("test_label", "test_value") 23 | ctx := pprof.WithLabels(context.Background(), labels) 24 | pprof.SetGoroutineLabels(ctx) 25 | 26 | runtime.SetBlockProfileRate(int((40 * time.Microsecond).Nanoseconds())) 27 | done := make(chan struct{}) 28 | g := errgroup.Group{} 29 | g.Go(func() error { 30 | return eventA(done) 31 | }) 32 | g.Go(func() error { 33 | return eventB(done) 34 | }) 35 | time.Sleep(time.Second) 36 | close(done) 37 | if err := g.Wait(); err != nil { 38 | return err 39 | } 40 | 41 | f, err := os.Create("block.pb.gz") 42 | if err != nil { 43 | return err 44 | } 45 | defer f.Close() 46 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil { 47 | return err 48 | } 49 | return nil 50 | } 51 | 52 | func eventA(done chan struct{}) error { 53 | return simulateBlockEvents(20*time.Microsecond, done) 54 | } 55 | 56 | func eventB(done chan struct{}) error { 57 | return simulateBlockEvents(40*time.Microsecond, done) 58 | } 59 | 60 | const tolerance = 1.1 61 | 62 | func simulateBlockEvents(meanDuration time.Duration, done chan struct{}) error { 63 | var ( 64 | prev time.Time 65 | sum time.Duration 66 | count int 67 | ticker = time.NewTicker(meanDuration) 68 | ) 69 | defer ticker.Stop() 70 | for { 71 | select { 72 | case <-ticker.C: 73 | now := time.Now() 74 | if !prev.IsZero() { 75 | sum += now.Sub(prev) 76 | count += 1 77 | if count > 1000 { 78 | actualMean := float64(sum) / float64(count) 79 | max := tolerance * float64(meanDuration) 80 | min := float64(meanDuration) / tolerance 81 | if actualMean <= min || actualMean >= max { 82 | return fmt.Errorf("low clock accuracy: got=%s want=%s", time.Duration(actualMean), meanDuration) 83 | } 84 | } 85 | } 86 | prev = now 87 | case <-done: 88 | return nil 89 | } 90 | } 91 | } 92 | 93 | /* 94 | 95 | Bias in current go version: 96 | 97 | $ go version 98 | go version go1.15.6 darwin/amd64 99 | $ go run . && go tool pprof -raw block.pb.gz 100 | PeriodType: contentions count 101 | Period: 1 102 | Time: 2021-02-05 15:27:01.371414 +0100 CET 103 | Samples: 104 | contentions/count delay/nanoseconds 105 | 23271 892063188: 1 2 3 4 106 | 22612 438270491: 1 2 5 4 107 | Locations 108 | 1: 0x10453af M=1 runtime.selectgo /usr/local/Cellar/go/1.15.6/libexec/src/runtime/select.go:511 s=0 109 | 2: 0x10cf56b M=1 main.simulateBlockEvents /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:66 s=0 110 | 3: 0x10cf8b2 M=1 main.eventB /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:52 s=0 111 | main.run.func2 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:28 s=0 112 | 4: 0x10cefd8 M=1 golang.org/x/sync/errgroup.(*Group).Go.func1 /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:57 s=0 113 | 5: 0x10cf852 M=1 main.eventA /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:48 s=0 114 | main.run.func1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:25 s=0 115 | Mappings 116 | 1: 0x0/0x0/0x0 [FN] 117 | 118 | After removing bias with this patch: 119 | https://github.com/felixge/go/commit/7c3f70c378d3f9a331a2079d17cd4cc420c70190 120 | 121 | $ sgo version 122 | go version devel +7c3f70c378 2021-02-05 15:19:48 +0100 darwin/amd64 123 | $ sgo run . && go tool pprof -raw block.pb.gz 124 | PeriodType: contentions count 125 | Period: 1 126 | Time: 2021-02-05 15:18:56.401244 +0100 CET 127 | Samples: 128 | contentions/count delay/nanoseconds 129 | 22833 931500628: 1 2 3 4 130 | 22453 902100036: 1 2 5 4 131 | 1 39999: 6 7 8 9 10 132 | Locations 133 | 1: 0x10471d9 M=1 runtime.selectgo /Users/felix.geisendoerfer/go/src/github.com/golang/go/src/runtime/select.go:492 s=0 134 | 2: 0x10d023e M=1 main.simulateBlockEvents /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:66 s=0 135 | 3: 0x10d0592 M=1 main.eventB /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:52 s=0 136 | main.run.func2 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:28 s=0 137 | 4: 0x10cfcd8 M=1 golang.org/x/sync/errgroup.(*Group).Go.func1 /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:57 s=0 138 | 5: 0x10d0532 M=1 main.eventA /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:48 s=0 139 | main.run.func1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:25 s=0 140 | 6: 0x107b024 M=1 sync.(*WaitGroup).Wait /Users/felix.geisendoerfer/go/src/github.com/golang/go/src/sync 141 | /waitgroup.go:130 s=0 142 | 7: 0x10cfb30 M=1 golang.org/x/sync/errgroup.(*Group).Wait /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:40 s=0 143 | 8: 0x10cff7a M=1 main.run /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:32 s=0 144 | 9: 0x10cfda5 M=1 main.main /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:14 s=0 145 | 10: 0x10371b5 M=1 runtime.main /Users/felix.geisendoerfer/go/src/github.com/golang/go/src/runtime/proc.go:225 s=0 146 | Mappings 147 | 1: 0x0/0x0/0x0 [FN] 148 | 149 | */ 150 | -------------------------------------------------------------------------------- /examples/block-vs-mutex/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "runtime/pprof" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | func main() { 13 | if err := run(); err != nil { 14 | fmt.Fprintln(os.Stderr, err) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | func run() error { 20 | runtime.SetBlockProfileRate(1) 21 | runtime.SetMutexProfileFraction(1) 22 | 23 | aquired := make(chan struct{}) 24 | var m sync.Mutex 25 | m.Lock() 26 | go func() { 27 | <-aquired 28 | m.Lock() 29 | aquired <- struct{}{} 30 | }() 31 | aquired <- struct{}{} 32 | time.Sleep(time.Nanosecond) 33 | m.Unlock() 34 | <-aquired 35 | 36 | if err := writeProfile("block"); err != nil { 37 | return err 38 | } else if err := writeProfile("mutex"); err != nil { 39 | return err 40 | } 41 | return nil 42 | } 43 | 44 | func writeProfile(name string) error { 45 | f, err := os.Create(name + ".pb.gz") 46 | if err != nil { 47 | return err 48 | } 49 | defer f.Close() 50 | 51 | if err := pprof.Lookup(name).WriteTo(f, 0); err != nil { 52 | return err 53 | } 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /examples/block/main: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block/main -------------------------------------------------------------------------------- /examples/block/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "runtime/pprof" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | var foo []string 12 | 13 | func main() { 14 | demonstrateSleep() 15 | 16 | f, err := os.Create("block.pb.gz") 17 | if err != nil { 18 | panic(err) 19 | } 20 | defer f.Close() 21 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil { 22 | panic(err) 23 | } 24 | } 25 | 26 | func demonstrateSleep() { 27 | runtime.SetBlockProfileRate(1) 28 | <-time.After(time.Millisecond) 29 | } 30 | 31 | func demonstrateSelect() { 32 | runtime.SetBlockProfileRate(1) 33 | 34 | ch1 := make(chan struct{}, 0) 35 | ch2 := make(chan struct{}, 1) 36 | ch3 := make(chan struct{}, 0) 37 | 38 | go func() { 39 | ch2 <- struct{}{} 40 | }() 41 | 42 | time.Sleep(20 * time.Millisecond) 43 | 44 | select { 45 | case <-ch1: 46 | case <-ch2: 47 | case <-ch3: 48 | } 49 | } 50 | 51 | func demonstrateSampling() { 52 | runtime.SetBlockProfileRate(int(40 * time.Microsecond.Nanoseconds())) 53 | for i := 0; i < 10000; i++ { 54 | blockMutex(10 * time.Microsecond) 55 | } 56 | } 57 | 58 | func blockMutex(d time.Duration) { 59 | m := &sync.Mutex{} 60 | m.Lock() 61 | go func() { 62 | spinSleep(d) 63 | m.Unlock() 64 | }() 65 | m.Lock() 66 | } 67 | 68 | // spinSleep is a more accurate version of time.Sleep() for short sleep 69 | // durations. Accuracy seems to be ~35ns. 70 | func spinSleep(d time.Duration) { 71 | start := time.Now() 72 | n := 0 73 | for time.Since(start) < d { 74 | n++ 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /examples/callers-func-length/README.md: -------------------------------------------------------------------------------- 1 | # callers-func-length 2 | 3 | A simple benchmark that shows that the costs of stack unwinding in Go increase for larger functions. This is due to the way `gopclntab` based unwinding is implemented. 4 | 5 | ``` 6 | go test -bench . 7 | goos: darwin 8 | goarch: amd64 9 | pkg: github.com/felixge/go-profiler-notes/examples/callers-func-length 10 | cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz 11 | BenchmarkCallers/short-12 2623002 454.1 ns/op 12 | BenchmarkCallers/loop-12 2590384 466.8 ns/op 13 | BenchmarkCallers/long-12 638096 1862 ns/op 14 | ``` 15 | -------------------------------------------------------------------------------- /examples/callers-func-length/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/callers-func-length 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /examples/callers-func-length/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | if err := run(); err != nil { 10 | fmt.Fprintln(os.Stderr, err) 11 | os.Exit(1) 12 | } 13 | } 14 | 15 | func run() error { 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /examples/callers-func-length/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "runtime" 5 | "testing" 6 | ) 7 | 8 | var callers = make([]uintptr, 32) 9 | 10 | func BenchmarkCallers(b *testing.B) { 11 | b.Run("short", func(b *testing.B) { 12 | short(func() { 13 | for i := 0; i < b.N; i++ { 14 | runtime.Callers(0, callers) 15 | } 16 | }) 17 | }) 18 | 19 | b.Run("loop", func(b *testing.B) { 20 | loop(func() { 21 | for i := 0; i < b.N; i++ { 22 | runtime.Callers(0, callers) 23 | } 24 | }) 25 | }) 26 | 27 | b.Run("long", func(b *testing.B) { 28 | long(func() { 29 | for i := 0; i < b.N; i++ { 30 | runtime.Callers(0, callers) 31 | } 32 | }) 33 | }) 34 | } 35 | 36 | func short(fn func()) { 37 | other() 38 | fn() 39 | } 40 | 41 | func loop(fn func()) { 42 | for i := 0; i < 1000; i++ { 43 | other() 44 | } 45 | fn() 46 | } 47 | 48 | func long(fn func()) { 49 | other() 50 | other() 51 | other() 52 | other() 53 | other() 54 | other() 55 | other() 56 | other() 57 | other() 58 | other() 59 | other() 60 | other() 61 | other() 62 | other() 63 | other() 64 | other() 65 | other() 66 | other() 67 | other() 68 | other() 69 | other() 70 | other() 71 | other() 72 | other() 73 | other() 74 | other() 75 | other() 76 | other() 77 | other() 78 | other() 79 | other() 80 | other() 81 | other() 82 | other() 83 | other() 84 | other() 85 | other() 86 | other() 87 | other() 88 | other() 89 | other() 90 | other() 91 | other() 92 | other() 93 | other() 94 | other() 95 | other() 96 | other() 97 | other() 98 | other() 99 | other() 100 | other() 101 | other() 102 | other() 103 | other() 104 | other() 105 | other() 106 | other() 107 | other() 108 | other() 109 | other() 110 | other() 111 | other() 112 | other() 113 | other() 114 | other() 115 | other() 116 | other() 117 | other() 118 | other() 119 | other() 120 | other() 121 | other() 122 | other() 123 | other() 124 | other() 125 | other() 126 | other() 127 | other() 128 | other() 129 | other() 130 | other() 131 | other() 132 | other() 133 | other() 134 | other() 135 | other() 136 | other() 137 | other() 138 | other() 139 | other() 140 | other() 141 | other() 142 | other() 143 | other() 144 | other() 145 | other() 146 | other() 147 | other() 148 | other() 149 | fn() 150 | } 151 | 152 | func other() int { 153 | m := map[int]string{} 154 | m[0] = "foo" 155 | m[100] = "bar" 156 | return len(m) 157 | } 158 | -------------------------------------------------------------------------------- /examples/cpu/benchmark.cpu.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/cpu/benchmark.cpu.pb.gz -------------------------------------------------------------------------------- /examples/cpu/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/cpu 2 | 3 | go 1.15 4 | 5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 6 | -------------------------------------------------------------------------------- /examples/cpu/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /examples/cpu/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | "time" 11 | 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | func main() { 16 | if err := run(); err != nil { 17 | fmt.Fprintln(os.Stderr, err) 18 | os.Exit(1) 19 | } 20 | } 21 | 22 | func run() error { 23 | g, _ := errgroup.WithContext(context.Background()) 24 | 25 | g.Go(func() error { 26 | addr := "localhost:6060" 27 | log.Printf("Listening on %s", addr) 28 | return http.ListenAndServe(addr, nil) 29 | }) 30 | 31 | g.Go(func() error { return computeSleepLoop(1000000, 10*time.Millisecond) }) 32 | 33 | return g.Wait() 34 | } 35 | 36 | func computeSleepLoop(n int, sleep time.Duration) error { 37 | for { 38 | compute(n) 39 | time.Sleep(sleep) 40 | } 41 | return nil 42 | } 43 | 44 | func compute(n int) int64 { 45 | var sum int64 46 | for i := 0; i < n; i++ { 47 | sum += int64(i) / 2 48 | sum += int64(i) / 3 49 | sum += int64(i) / 4 50 | sum += int64(i) / 5 51 | sum += int64(i) / 6 52 | sum += int64(i) / 7 53 | sum += int64(i) / 8 54 | } 55 | return sum 56 | } 57 | -------------------------------------------------------------------------------- /examples/cpu/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Benchmark_compute(b *testing.B) { 8 | compute(b.N) 9 | } 10 | -------------------------------------------------------------------------------- /examples/cpu/pprof.samples.cpu.001.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/cpu/pprof.samples.cpu.001.pb.gz -------------------------------------------------------------------------------- /examples/cpu/pprof.samples.cpu.001.pprof.txt: -------------------------------------------------------------------------------- 1 | PeriodType: cpu nanoseconds 2 | Period: 10000000 3 | Time: 2021-01-08 17:10:32.116825 +0100 CET 4 | Duration: 3.13 5 | Samples: 6 | samples/count cpu/nanoseconds 7 | 19 190000000: 1 2 3 8 | 5 50000000: 4 5 2 3 9 | 1 10000000: 6 7 8 9 10 11 12 13 14 10 | 1 10000000: 15 16 17 11 18 14 11 | 2 20000000: 6 7 8 9 10 11 18 14 12 | 7 70000000: 19 20 21 22 23 24 14 13 | 3 30000000: 25 26 27 28 14 | Locations 15 | 1: 0x1372f7f M=1 main.computeSum /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/cpu/main.go:39 s=0 16 | 2: 0x13730f2 M=1 main.run.func2 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/cpu/main.go:31 s=0 17 | 3: 0x1372cf8 M=1 golang.org/x/sync/errgroup.(*Group).Go.func1 /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:57 s=0 18 | 4: 0x10711c0 M=1 runtime.asyncPreempt /usr/local/Cellar/go/1.15.6/libexec/src/runtime/preempt_amd64.s:7 s=0 19 | 5: 0x1372f7e M=1 main.computeSum /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/cpu/main.go:37 s=0 20 | 6: 0x1058598 M=1 runtime.pthread_cond_wait /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:414 s=0 21 | 7: 0x10351ac M=1 runtime.semasleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/os_darwin.go:63 s=0 22 | 8: 0x100de06 M=1 runtime.notesleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/lock_sema.go:181 s=0 23 | 9: 0x103ee64 M=1 runtime.stopm /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1924 s=0 24 | 10: 0x104063e M=1 runtime.findrunnable /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2485 s=0 25 | 11: 0x10412b6 M=1 runtime.schedule /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2683 s=0 26 | 12: 0x1041a59 M=1 runtime.goschedImpl /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2866 s=0 27 | 13: 0x1041cd3 M=1 runtime.gopreempt_m /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2894 s=0 28 | 14: 0x106df3a M=1 runtime.mcall /usr/local/Cellar/go/1.15.6/libexec/src/runtime/asm_amd64.s:318 s=0 29 | 15: 0x1057ff7 M=1 runtime.nanotime1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:284 s=0 30 | 16: 0x1041704 M=1 runtime.nanotime /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time_nofake.go:19 s=0 31 | runtime.checkTimers /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2757 s=0 32 | 17: 0x103fd69 M=1 runtime.findrunnable /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2289 s=0 33 | 18: 0x104185c M=1 runtime.park_m /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2851 s=0 34 | 19: 0x1057f58 M=1 runtime.write1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:270 s=0 35 | 20: 0x1034be4 M=1 runtime.write /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time_nofake.go:30 s=0 36 | runtime.netpollBreak /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll_kqueue.go:89 s=0 37 | 21: 0x1040c37 M=1 runtime.wakeNetPoller /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2521 s=0 38 | 22: 0x10593e9 M=1 runtime.modtimer /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:480 s=0 39 | 23: 0x1058744 M=1 runtime.resettimer /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:529 s=0 40 | runtime.resetForSleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:197 s=0 41 | 24: 0x1041881 M=1 runtime.park_m /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2840 s=0 42 | 25: 0x1057f10 M=1 runtime.usleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:263 s=0 43 | 26: 0x1045cac M=1 runtime.sysmon /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:4660 s=0 44 | 27: 0x103d8a7 M=1 runtime.mstart1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1172 s=0 45 | 28: 0x103d7c5 M=1 runtime.mstart /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1137 s=0 46 | Mappings 47 | 1: 0x0/0x0/0x0 [FN] 48 | -------------------------------------------------------------------------------- /examples/datadog/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/datadog 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/DataDog/datadog-go v4.2.0+incompatible // indirect 7 | github.com/google/uuid v1.1.4 // indirect 8 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 9 | ) 10 | -------------------------------------------------------------------------------- /examples/datadog/go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/datadog-go v4.2.0+incompatible h1:Q73jzyKHwyA04Gf4SSukRF+KR4wJEimU6tAuU0B8Y4Y= 2 | github.com/DataDog/datadog-go v4.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 3 | github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0= 4 | github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 5 | github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= 6 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 h1:EmglUJuykRsTwsQDcKaAo3CmOunWU6Dqk7U2lo7Pjss= 7 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0/go.mod h1:Sp1lku8WJMvNV0kjDI4Ni/T7J/U3BO5ct5kEaoVU8+I= 8 | -------------------------------------------------------------------------------- /examples/datadog/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | _ "net/http/pprof" 6 | "os" 7 | "time" 8 | 9 | "gopkg.in/DataDog/dd-trace-go.v1/profiler" 10 | ) 11 | 12 | func main() { 13 | if err := run(); err != nil { 14 | fmt.Fprintln(os.Stderr, err) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | func run() error { 20 | if err := startProfiler(); err != nil { 21 | return err 22 | } 23 | defer profiler.Stop() 24 | 25 | for { 26 | time.Sleep(10 * time.Millisecond) 27 | } 28 | } 29 | 30 | func startProfiler() error { 31 | return profiler.Start( 32 | profiler.WithService("datadog-example"), 33 | profiler.WithEnv("localhost"), 34 | profiler.WithVersion("1.0"), 35 | profiler.WithProfileTypes( 36 | profiler.CPUProfile, 37 | profiler.HeapProfile, 38 | profiler.MutexProfile, 39 | profiler.GoroutineProfile, 40 | profiler.BlockProfile, 41 | ), 42 | ) 43 | } 44 | -------------------------------------------------------------------------------- /examples/delve-trace/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/delve-trace 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /examples/delve-trace/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | if err := run(); err != nil { 10 | fmt.Fprintln(os.Stderr, err) 11 | os.Exit(1) 12 | } 13 | } 14 | 15 | func run() error { 16 | for i := 0; i < 10; i++ { 17 | foo(i) 18 | } 19 | return nil 20 | } 21 | 22 | func foo(i int) int { 23 | return i * 2 24 | } 25 | -------------------------------------------------------------------------------- /examples/frame-order/frames.txt: -------------------------------------------------------------------------------- 1 | goroutine 1 [running]: 2 | runtime/pprof.writeGoroutineStacks(0x111e660, 0xc0000a0018, 0x30, 0x12b3aa8) 3 | /usr/local/Cellar/go/1.15.7_1/libexec/src/runtime/pprof/pprof.go:693 +0x9f 4 | runtime/pprof.writeGoroutine(0x111e660, 0xc0000a0018, 0x2, 0x0, 0x602e6537) 5 | /usr/local/Cellar/go/1.15.7_1/libexec/src/runtime/pprof/pprof.go:682 +0x45 6 | runtime/pprof.(*Profile).WriteTo(0x11ae4e0, 0x111e660, 0xc0000a0018, 0x2, 0xc0000a0020, 0x0) 7 | /usr/local/Cellar/go/1.15.7_1/libexec/src/runtime/pprof/pprof.go:331 +0x3f2 8 | main.bar() 9 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/frame-order/main.go:19 +0xd5 10 | main.foo(...) 11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/frame-order/main.go:13 12 | main.main() 13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/frame-order/main.go:9 +0x25 14 | -------------------------------------------------------------------------------- /examples/frame-order/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/frame-order 2 | 3 | go 1.15 4 | -------------------------------------------------------------------------------- /examples/frame-order/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "runtime/pprof" 6 | ) 7 | 8 | func main() { 9 | foo() 10 | } 11 | 12 | func foo() { 13 | bar() 14 | } 15 | func bar() { 16 | debug2, _ := os.Create("frames.txt") 17 | debug0, _ := os.Create("frames.pb.gz") 18 | 19 | pprof.Lookup("goroutine").WriteTo(debug2, 2) 20 | pprof.Lookup("goroutine").WriteTo(debug0, 0) 21 | } 22 | -------------------------------------------------------------------------------- /examples/goroutine/1.net.http.pprof.goroutine.debug0.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/1.net.http.pprof.goroutine.debug0.pb.gz -------------------------------------------------------------------------------- /examples/goroutine/1.net.http.pprof.goroutine.debug1.txt: -------------------------------------------------------------------------------- 1 | goroutine profile: total 10 2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81 3 | # labels: {"test_label":"test_value"} 4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 6 | 7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81 8 | # labels: {"test_label":"test_value"} 9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 10 | 11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12ff937 0x11707c5 0x117092f 0x13005e8 0x106fd81 12 | # labels: {"test_label":"test_value"} 13 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 14 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 15 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 16 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 17 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 18 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 19 | # 0x12ff936 net/http.(*persistConn).Read+0x76 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 20 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 21 | # 0x117092e bufio.(*Reader).Peek+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 22 | # 0x13005e7 net/http.(*persistConn).readLoop+0x1a7 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 23 | 24 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81 25 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 26 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 27 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 28 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 29 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 30 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 31 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 32 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 33 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 34 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 35 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 36 | 37 | 1 @ 0x103b125 0x104afcf 0x130223c 0x106fd81 38 | # labels: {"test_label":"test_value"} 39 | # 0x130223b net/http.(*persistConn).writeLoop+0x11b /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 40 | 41 | 1 @ 0x103b125 0x104afcf 0x1302f59 0x12f74a5 0x12dd435 0x129e853 0x129e23f 0x12a025f 0x129fbbe 0x129fbce 0x13abba5 0x13abb8a 0x13acafe 0x13abe67 0x13ac247 0x103ad29 0x106fd81 42 | # labels: {"test_label":"test_value"} 43 | # 0x1302f58 net/http.(*persistConn).roundTrip+0x778 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2565 44 | # 0x12f74a4 net/http.(*Transport).roundTrip+0xa64 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:582 45 | # 0x12dd434 net/http.(*Transport).RoundTrip+0x34 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/roundtrip.go:17 46 | # 0x129e852 net/http.send+0x452 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:252 47 | # 0x129e23e net/http.(*Client).send+0xfe /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:176 48 | # 0x12a025e net/http.(*Client).do+0x45e /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:718 49 | # 0x129fbbd net/http.(*Client).Do+0xbd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:586 50 | # 0x129fbcd net/http.(*Client).Get+0xcd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:475 51 | # 0x13abba4 net/http.Get+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:447 52 | # 0x13abb89 main.writeHttpProfile+0xe9 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:92 53 | # 0x13acafd main.glob..func7+0x3d /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:79 54 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 55 | # 0x13ac246 main.main+0x2c6 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 56 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204 57 | 58 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81 59 | # labels: {"test_label":"test_value"} 60 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 61 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 62 | 63 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13a9865 0x13ab145 0x12e6424 0x12e834d 0x12e9a23 0x12e522d 0x106fd81 64 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716 65 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724 66 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684 67 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 68 | # 0x13a9864 net/http/pprof.handler.ServeHTTP+0x384 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:256 69 | # 0x13ab144 net/http/pprof.Index+0x944 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:367 70 | # 0x12e6423 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2042 71 | # 0x12e834c net/http.(*ServeMux).ServeHTTP+0x1ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2417 72 | # 0x12e9a22 net/http.serverHandler.ServeHTTP+0xa2 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2843 73 | # 0x12e522c net/http.(*conn).serve+0x8ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1925 74 | 75 | 1 @ 0x12def61 0x106fd81 76 | # 0x12def60 net/http.(*connReader).backgroundRead+0x0 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:689 77 | 78 | -------------------------------------------------------------------------------- /examples/goroutine/1.net.http.pprof.goroutine.debug2.txt: -------------------------------------------------------------------------------- 1 | goroutine 41 [running]: 2 | runtime/pprof.writeGoroutineStacks(0x14e5f60, 0xc0001d6000, 0xc0001b6270, 0x0) 3 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:693 +0x9f 4 | runtime/pprof.writeGoroutine(0x14e5f60, 0xc0001d6000, 0x2, 0x1714f40, 0xc0001ba420) 5 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:682 +0x45 6 | runtime/pprof.(*Profile).WriteTo(0x17179e0, 0x14e5f60, 0xc0001d6000, 0x2, 0xc0001d6000, 0xc0003379d8) 7 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 +0x3f2 8 | net/http/pprof.handler.ServeHTTP(0xc0001ca071, 0x9, 0x14ec9a0, 0xc0001d6000, 0xc000110300) 9 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:256 +0x385 10 | net/http/pprof.Index(0x14ec9a0, 0xc0001d6000, 0xc000110300) 11 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:367 +0x945 12 | net/http.HandlerFunc.ServeHTTP(0x1486d90, 0x14ec9a0, 0xc0001d6000, 0xc000110300) 13 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2042 +0x44 14 | net/http.(*ServeMux).ServeHTTP(0x1725ba0, 0x14ec9a0, 0xc0001d6000, 0xc000110300) 15 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2417 +0x1ad 16 | net/http.serverHandler.ServeHTTP(0xc00019c000, 0x14ec9a0, 0xc0001d6000, 0xc000110300) 17 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2843 +0xa3 18 | net/http.(*conn).serve(0xc0000c6320, 0x14ed4a0, 0xc000322000) 19 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1925 +0x8ad 20 | created by net/http.(*Server).Serve 21 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2969 +0x36c 22 | 23 | goroutine 1 [select]: 24 | net/http.(*persistConn).roundTrip(0xc0000cea20, 0xc0001b40c0, 0x0, 0x0, 0x0) 25 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2565 +0x779 26 | net/http.(*Transport).roundTrip(0x171d020, 0xc0001ce000, 0x30, 0x30, 0x180de98) 27 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:582 +0xa65 28 | net/http.(*Transport).RoundTrip(0x171d020, 0xc0001ce000, 0x171d020, 0x0, 0x0) 29 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/roundtrip.go:17 +0x35 30 | net/http.send(0xc0001ce000, 0x14e5d80, 0x171d020, 0x0, 0x0, 0x0, 0xc0001c8018, 0x203000, 0x1, 0x0) 31 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:252 +0x453 32 | net/http.(*Client).send(0x17259e0, 0xc0001ce000, 0x0, 0x0, 0x0, 0xc0001c8018, 0x0, 0x1, 0xf8) 33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:176 +0xff 34 | net/http.(*Client).do(0x17259e0, 0xc0001ce000, 0x0, 0x0, 0x0) 35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:718 +0x45f 36 | net/http.(*Client).Do(...) 37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:586 38 | net/http.(*Client).Get(0x17259e0, 0xc0001cc000, 0x33, 0x2, 0x2, 0xc0001cc000) 39 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:475 +0xbe 40 | net/http.Get(...) 41 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:447 42 | main.writeHttpProfile(0x14e5940, 0xc0001b6090, 0x2, 0x0, 0x0) 43 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:92 +0x105 44 | main.glob..func8(0x14e5940, 0xc0001b6090, 0xc000213eb0, 0x2) 45 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:85 +0x3e 46 | main.writeProfiles(0x1, 0xc0000c4008, 0x146641d) 47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187 48 | main.main() 49 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 +0x2c7 50 | 51 | goroutine 22 [sleep]: 52 | time.Sleep(0x3b9aca00) 53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 54 | main.shortSleepLoop() 55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 56 | created by main.indirectShortSleepLoop2 57 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35 58 | 59 | goroutine 3 [IO wait]: 60 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0) 61 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 62 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786) 63 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 64 | internal/poll.(*pollDesc).waitRead(...) 65 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 66 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) 67 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc 68 | net.(*netFD).accept(0xc00019e000, 0x7d667d63cbbded3e, 0x1789ccbbded3e, 0x100000001) 69 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45 70 | net.(*TCPListener).accept(0xc000188060, 0x60006709, 0xc000196da8, 0x109abe6) 71 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32 72 | net.(*TCPListener).Accept(0xc000188060, 0xc000196df8, 0x18, 0xc000001200, 0x12e9eec) 73 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65 74 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0) 75 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266 76 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536) 77 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7 78 | net/http.ListenAndServe(...) 79 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 80 | main.main.func1(0xc000032120) 81 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126 82 | created by main.main 83 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5 84 | 85 | goroutine 4 [sleep]: 86 | time.Sleep(0x3b9aca00) 87 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 88 | main.shortSleepLoop() 89 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 90 | created by main.main 91 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195 92 | 93 | goroutine 5 [sleep]: 94 | time.Sleep(0x34630b8a000) 95 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 96 | main.sleepLoop(0x34630b8a000) 97 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b 98 | created by main.main 99 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc 100 | 101 | goroutine 6 [chan receive]: 102 | main.chanReceiveForever() 103 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d 104 | created by main.main 105 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4 106 | 107 | goroutine 24 [select]: 108 | net/http.(*persistConn).writeLoop(0xc0000cea20) 109 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 +0x11c 110 | created by net/http.(*Transport).dialConn 111 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1709 +0xcdc 112 | 113 | goroutine 23 [IO wait]: 114 | internal/poll.runtime_pollWait(0x1e91da0, 0x72, 0x14e6ca0) 115 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 116 | internal/poll.(*pollDesc).wait(0xc00010e198, 0x72, 0x14e6c00, 0x16db878, 0x0) 117 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 118 | internal/poll.(*pollDesc).waitRead(...) 119 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 120 | internal/poll.(*FD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 121 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5 122 | net.(*netFD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x103b1dc, 0xc000199b58, 0x10680e0) 123 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f 124 | net.(*conn).Read(0xc000010008, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 125 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e 126 | net/http.(*persistConn).Read(0xc0000cea20, 0xc000256000, 0x1000, 0x1000, 0xc00009e300, 0xc000199c58, 0x10074b5) 127 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 +0x77 128 | bufio.(*Reader).fill(0xc0001801e0) 129 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105 130 | bufio.(*Reader).Peek(0xc0001801e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc000030180) 131 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 +0x4f 132 | net/http.(*persistConn).readLoop(0xc0000cea20) 133 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 +0x1a8 134 | created by net/http.(*Transport).dialConn 135 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1708 +0xcb7 136 | 137 | goroutine 67 [IO wait]: 138 | internal/poll.runtime_pollWait(0x1e91cb8, 0x72, 0x14e6ca0) 139 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 140 | internal/poll.(*pollDesc).wait(0xc00019e098, 0x72, 0x14e6c00, 0x16db878, 0x0) 141 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 142 | internal/poll.(*pollDesc).waitRead(...) 143 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 144 | internal/poll.(*FD).Read(0xc00019e080, 0xc00007c311, 0x1, 0x1, 0x0, 0x0, 0x0) 145 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5 146 | net.(*netFD).Read(0xc00019e080, 0xc00007c311, 0x1, 0x1, 0x0, 0x0, 0x0) 147 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f 148 | net.(*conn).Read(0xc000186028, 0xc00007c311, 0x1, 0x1, 0x0, 0x0, 0x0) 149 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e 150 | net/http.(*connReader).backgroundRead(0xc00007c300) 151 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:690 +0x58 152 | created by net/http.(*connReader).startBackgroundRead 153 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:686 +0xd5 154 | -------------------------------------------------------------------------------- /examples/goroutine/1.pprof.lookup.goroutine.debug0.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/1.pprof.lookup.goroutine.debug0.pb.gz -------------------------------------------------------------------------------- /examples/goroutine/1.pprof.lookup.goroutine.debug1.txt: -------------------------------------------------------------------------------- 1 | goroutine profile: total 6 2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81 3 | # labels: {"test_label":"test_value"} 4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 6 | 7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81 8 | # labels: {"test_label":"test_value"} 9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 10 | 11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81 12 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 13 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 14 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 15 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 16 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 17 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 18 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 19 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 20 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 21 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 22 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 23 | 24 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81 25 | # labels: {"test_label":"test_value"} 26 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 27 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 28 | 29 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13ac965 0x13abe67 0x13ac247 0x103ad29 0x106fd81 30 | # labels: {"test_label":"test_value"} 31 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716 32 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724 33 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684 34 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 35 | # 0x13ac964 main.glob..func4+0x64 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:60 36 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 37 | # 0x13ac246 main.main+0x2c6 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 38 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204 39 | 40 | -------------------------------------------------------------------------------- /examples/goroutine/1.pprof.lookup.goroutine.debug2.txt: -------------------------------------------------------------------------------- 1 | goroutine 1 [running]: 2 | runtime/pprof.writeGoroutineStacks(0x14e5940, 0xc0000abb00, 0x101b8a5, 0xc000213d20) 3 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:693 +0x9f 4 | runtime/pprof.writeGoroutine(0x14e5940, 0xc0000abb00, 0x2, 0xc000213dc0, 0x10ee5c8) 5 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:682 +0x45 6 | runtime/pprof.(*Profile).WriteTo(0x17179e0, 0x14e5940, 0xc0000abb00, 0x2, 0xc00002c210, 0xc0002041a0) 7 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 +0x3f2 8 | main.glob..func5(0x14e5940, 0xc0000abb00, 0xc000213eb0, 0x2) 9 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:67 +0x65 10 | main.writeProfiles(0x1, 0xc0000c4008, 0x146641d) 11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187 12 | main.main() 13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 +0x2c7 14 | 15 | goroutine 22 [sleep]: 16 | time.Sleep(0x3b9aca00) 17 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 18 | main.shortSleepLoop() 19 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 20 | created by main.indirectShortSleepLoop2 21 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35 22 | 23 | goroutine 3 [IO wait]: 24 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0) 25 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 26 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786) 27 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 28 | internal/poll.(*pollDesc).waitRead(...) 29 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 30 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) 31 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc 32 | net.(*netFD).accept(0xc00019e000, 0xc0001822d0, 0x1010038, 0xc000088000) 33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45 34 | net.(*TCPListener).accept(0xc000188060, 0xc000196da8, 0x1010038, 0x30) 35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32 36 | net.(*TCPListener).Accept(0xc000188060, 0x1436c40, 0xc0001822d0, 0x13f4620, 0x1714f50) 37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65 38 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0) 39 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266 40 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536) 41 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7 42 | net/http.ListenAndServe(...) 43 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 44 | main.main.func1(0xc000032120) 45 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126 46 | created by main.main 47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5 48 | 49 | goroutine 4 [sleep]: 50 | time.Sleep(0x3b9aca00) 51 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 52 | main.shortSleepLoop() 53 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 54 | created by main.main 55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195 56 | 57 | goroutine 5 [sleep]: 58 | time.Sleep(0x34630b8a000) 59 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 60 | main.sleepLoop(0x34630b8a000) 61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b 62 | created by main.main 63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc 64 | 65 | goroutine 6 [chan receive]: 66 | main.chanReceiveForever() 67 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d 68 | created by main.main 69 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4 70 | -------------------------------------------------------------------------------- /examples/goroutine/1.runtime.goroutineprofile.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Stack0": [ 4 | 20629256, 5 | 20629212, 6 | 20627047, 7 | 20628039, 8 | 17018153, 9 | 17235329, 10 | 0, 11 | 0, 12 | 0, 13 | 0, 14 | 0, 15 | 0, 16 | 0, 17 | 0, 18 | 0, 19 | 0, 20 | 0, 21 | 0, 22 | 0, 23 | 0, 24 | 0, 25 | 0, 26 | 0, 27 | 0, 28 | 0, 29 | 0, 30 | 0, 31 | 0, 32 | 0, 33 | 0, 34 | 0, 35 | 0 36 | ] 37 | }, 38 | { 39 | "Stack0": [ 40 | 17019173, 41 | 17222943, 42 | 20628554, 43 | 17235329, 44 | 0, 45 | 0, 46 | 0, 47 | 0, 48 | 0, 49 | 0, 50 | 0, 51 | 0, 52 | 0, 53 | 0, 54 | 0, 55 | 0, 56 | 0, 57 | 0, 58 | 0, 59 | 0, 60 | 0, 61 | 0, 62 | 0, 63 | 0, 64 | 0, 65 | 0, 66 | 0, 67 | 0, 68 | 0, 69 | 0, 70 | 0, 71 | 0 72 | ] 73 | }, 74 | { 75 | "Stack0": [ 76 | 17019173, 77 | 16990811, 78 | 17211861, 79 | 17662341, 80 | 17673596, 81 | 17673566, 82 | 18587653, 83 | 18694226, 84 | 18689765, 85 | 19832294, 86 | 19831575, 87 | 20630758, 88 | 20630672, 89 | 17235329, 90 | 0, 91 | 0, 92 | 0, 93 | 0, 94 | 0, 95 | 0, 96 | 0, 97 | 0, 98 | 0, 99 | 0, 100 | 0, 101 | 0, 102 | 0, 103 | 0, 104 | 0, 105 | 0, 106 | 0, 107 | 0 108 | ] 109 | }, 110 | { 111 | "Stack0": [ 112 | 17019173, 113 | 17222943, 114 | 20628554, 115 | 17235329, 116 | 0, 117 | 0, 118 | 0, 119 | 0, 120 | 0, 121 | 0, 122 | 0, 123 | 0, 124 | 0, 125 | 0, 126 | 0, 127 | 0, 128 | 0, 129 | 0, 130 | 0, 131 | 0, 132 | 0, 133 | 0, 134 | 0, 135 | 0, 136 | 0, 137 | 0, 138 | 0, 139 | 0, 140 | 0, 141 | 0, 142 | 0, 143 | 0 144 | ] 145 | }, 146 | { 147 | "Stack0": [ 148 | 17019173, 149 | 17222943, 150 | 20628619, 151 | 17235329, 152 | 0, 153 | 0, 154 | 0, 155 | 0, 156 | 0, 157 | 0, 158 | 0, 159 | 0, 160 | 0, 161 | 0, 162 | 0, 163 | 0, 164 | 0, 165 | 0, 166 | 0, 167 | 0, 168 | 0, 169 | 0, 170 | 0, 171 | 0, 172 | 0, 173 | 0, 174 | 0, 175 | 0, 176 | 0, 177 | 0, 178 | 0, 179 | 0 180 | ] 181 | }, 182 | { 183 | "Stack0": [ 184 | 17019173, 185 | 16810991, 186 | 16810027, 187 | 20628717, 188 | 17235329, 189 | 0, 190 | 0, 191 | 0, 192 | 0, 193 | 0, 194 | 0, 195 | 0, 196 | 0, 197 | 0, 198 | 0, 199 | 0, 200 | 0, 201 | 0, 202 | 0, 203 | 0, 204 | 0, 205 | 0, 206 | 0, 207 | 0, 208 | 0, 209 | 0, 210 | 0, 211 | 0, 212 | 0, 213 | 0, 214 | 0, 215 | 0 216 | ] 217 | } 218 | ] 219 | -------------------------------------------------------------------------------- /examples/goroutine/1.runtime.stack.txt: -------------------------------------------------------------------------------- 1 | goroutine 1 [running]: 2 | main.glob..func1(0x14e5940, 0xc0000aa7b0, 0xc000064eb0, 0x2) 3 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:29 +0x6f 4 | main.writeProfiles(0x1, 0xc0000c4008, 0x146641d) 5 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187 6 | main.main() 7 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 +0x2c7 8 | 9 | goroutine 22 [sleep]: 10 | time.Sleep(0x3b9aca00) 11 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 12 | main.shortSleepLoop() 13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 14 | created by main.indirectShortSleepLoop2 15 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35 16 | 17 | goroutine 3 [IO wait]: 18 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0) 19 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 20 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786) 21 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 22 | internal/poll.(*pollDesc).waitRead(...) 23 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 24 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) 25 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc 26 | net.(*netFD).accept(0xc00019e000, 0xc0001822d0, 0x1010038, 0xc000088000) 27 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45 28 | net.(*TCPListener).accept(0xc000188060, 0xc000196da8, 0x1010038, 0x30) 29 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32 30 | net.(*TCPListener).Accept(0xc000188060, 0x1436c40, 0xc0001822d0, 0x13f4620, 0x1714f50) 31 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65 32 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0) 33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266 34 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536) 35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7 36 | net/http.ListenAndServe(...) 37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 38 | main.main.func1(0xc000032120) 39 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126 40 | created by main.main 41 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5 42 | 43 | goroutine 4 [sleep]: 44 | time.Sleep(0x3b9aca00) 45 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 46 | main.shortSleepLoop() 47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 48 | created by main.main 49 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195 50 | 51 | goroutine 5 [sleep]: 52 | time.Sleep(0x34630b8a000) 53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 54 | main.sleepLoop(0x34630b8a000) 55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b 56 | created by main.main 57 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc 58 | 59 | goroutine 6 [chan receive]: 60 | main.chanReceiveForever() 61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d 62 | created by main.main 63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4 64 | -------------------------------------------------------------------------------- /examples/goroutine/2.net.http.pprof.goroutine.debug0.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/2.net.http.pprof.goroutine.debug0.pb.gz -------------------------------------------------------------------------------- /examples/goroutine/2.net.http.pprof.goroutine.debug1.txt: -------------------------------------------------------------------------------- 1 | goroutine profile: total 10 2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81 3 | # labels: {"test_label":"test_value"} 4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 6 | 7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81 8 | # labels: {"test_label":"test_value"} 9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 10 | 11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12ff937 0x11707c5 0x117092f 0x13005e8 0x106fd81 12 | # labels: {"test_label":"test_value"} 13 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 14 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 15 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 16 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 17 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 18 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 19 | # 0x12ff936 net/http.(*persistConn).Read+0x76 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 20 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 21 | # 0x117092e bufio.(*Reader).Peek+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 22 | # 0x13005e7 net/http.(*persistConn).readLoop+0x1a7 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 23 | 24 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81 25 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 26 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 27 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 28 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 29 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 30 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 31 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 32 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 33 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 34 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 35 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 36 | 37 | 1 @ 0x103b125 0x104afcf 0x130223c 0x106fd81 38 | # labels: {"test_label":"test_value"} 39 | # 0x130223b net/http.(*persistConn).writeLoop+0x11b /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 40 | 41 | 1 @ 0x103b125 0x104afcf 0x1302f59 0x12f74a5 0x12dd435 0x129e853 0x129e23f 0x12a025f 0x129fbbe 0x129fbce 0x13abba5 0x13abb8a 0x13acafe 0x13abe67 0x13ac352 0x103ad29 0x106fd81 42 | # labels: {"test_label":"test_value"} 43 | # 0x1302f58 net/http.(*persistConn).roundTrip+0x778 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2565 44 | # 0x12f74a4 net/http.(*Transport).roundTrip+0xa64 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:582 45 | # 0x12dd434 net/http.(*Transport).RoundTrip+0x34 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/roundtrip.go:17 46 | # 0x129e852 net/http.send+0x452 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:252 47 | # 0x129e23e net/http.(*Client).send+0xfe /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:176 48 | # 0x12a025e net/http.(*Client).do+0x45e /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:718 49 | # 0x129fbbd net/http.(*Client).Do+0xbd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:586 50 | # 0x129fbcd net/http.(*Client).Get+0xcd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:475 51 | # 0x13abba4 net/http.Get+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:447 52 | # 0x13abb89 main.writeHttpProfile+0xe9 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:92 53 | # 0x13acafd main.glob..func7+0x3d /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:79 54 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 55 | # 0x13ac351 main.main+0x3d1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152 56 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204 57 | 58 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81 59 | # labels: {"test_label":"test_value"} 60 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 61 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 62 | 63 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13a9865 0x13ab145 0x12e6424 0x12e834d 0x12e9a23 0x12e522d 0x106fd81 64 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716 65 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724 66 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684 67 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 68 | # 0x13a9864 net/http/pprof.handler.ServeHTTP+0x384 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:256 69 | # 0x13ab144 net/http/pprof.Index+0x944 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:367 70 | # 0x12e6423 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2042 71 | # 0x12e834c net/http.(*ServeMux).ServeHTTP+0x1ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2417 72 | # 0x12e9a22 net/http.serverHandler.ServeHTTP+0xa2 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2843 73 | # 0x12e522c net/http.(*conn).serve+0x8ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1925 74 | 75 | 1 @ 0x12def61 0x106fd81 76 | # 0x12def60 net/http.(*connReader).backgroundRead+0x0 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:689 77 | 78 | -------------------------------------------------------------------------------- /examples/goroutine/2.pprof.lookup.goroutine.debug0.pb.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/2.pprof.lookup.goroutine.debug0.pb.gz -------------------------------------------------------------------------------- /examples/goroutine/2.pprof.lookup.goroutine.debug1.txt: -------------------------------------------------------------------------------- 1 | goroutine profile: total 9 2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81 3 | # labels: {"test_label":"test_value"} 4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 6 | 7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81 8 | # labels: {"test_label":"test_value"} 9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 10 | 11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12df52d 0x11707c5 0x117151d 0x1171754 0x1263c2c 0x12d96ca 0x12d96f9 0x12e09ba 0x12e5085 0x106fd81 12 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 13 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 14 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 15 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 16 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 17 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 18 | # 0x12df52c net/http.(*connReader).Read+0x1ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:798 19 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 20 | # 0x117151c bufio.(*Reader).ReadSlice+0x3c /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:360 21 | # 0x1171753 bufio.(*Reader).ReadLine+0x33 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:389 22 | # 0x1263c2b net/textproto.(*Reader).readLineSlice+0x6b /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:58 23 | # 0x12d96c9 net/textproto.(*Reader).ReadLine+0xa9 /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:39 24 | # 0x12d96f8 net/http.readRequest+0xd8 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/request.go:1012 25 | # 0x12e09b9 net/http.(*conn).readRequest+0x199 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:984 26 | # 0x12e5084 net/http.(*conn).serve+0x704 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1851 27 | 28 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12ff937 0x11707c5 0x117092f 0x13005e8 0x106fd81 29 | # labels: {"test_label":"test_value"} 30 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 31 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 32 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 33 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 34 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 35 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 36 | # 0x12ff936 net/http.(*persistConn).Read+0x76 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 37 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 38 | # 0x117092e bufio.(*Reader).Peek+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 39 | # 0x13005e7 net/http.(*persistConn).readLoop+0x1a7 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 40 | 41 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81 42 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 43 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 44 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 45 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 46 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 47 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 48 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 49 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 50 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 51 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 52 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 53 | 54 | 1 @ 0x103b125 0x104afcf 0x130223c 0x106fd81 55 | # labels: {"test_label":"test_value"} 56 | # 0x130223b net/http.(*persistConn).writeLoop+0x11b /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 57 | 58 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81 59 | # labels: {"test_label":"test_value"} 60 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 61 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 62 | 63 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13ac965 0x13abe67 0x13ac352 0x103ad29 0x106fd81 64 | # labels: {"test_label":"test_value"} 65 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716 66 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724 67 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684 68 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 69 | # 0x13ac964 main.glob..func4+0x64 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:60 70 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 71 | # 0x13ac351 main.main+0x3d1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152 72 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204 73 | 74 | -------------------------------------------------------------------------------- /examples/goroutine/2.pprof.lookup.goroutine.debug2.txt: -------------------------------------------------------------------------------- 1 | goroutine 1 [running]: 2 | runtime/pprof.writeGoroutineStacks(0x14e5940, 0xc0000abc50, 0x101b8a5, 0xc000333d20) 3 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:693 +0x9f 4 | runtime/pprof.writeGoroutine(0x14e5940, 0xc0000abc50, 0x2, 0xc000333dc0, 0x10ee5c8) 5 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:682 +0x45 6 | runtime/pprof.(*Profile).WriteTo(0x17179e0, 0x14e5940, 0xc0000abc50, 0x2, 0xc00002c210, 0xc0000ac4e0) 7 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 +0x3f2 8 | main.glob..func5(0x14e5940, 0xc0000abc50, 0xc000333eb0, 0x2) 9 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:67 +0x65 10 | main.writeProfiles(0x2, 0xc0000c4008, 0x1466424) 11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187 12 | main.main() 13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152 +0x3d2 14 | 15 | goroutine 22 [sleep, 1 minutes]: 16 | time.Sleep(0x3b9aca00) 17 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 18 | main.shortSleepLoop() 19 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 20 | created by main.indirectShortSleepLoop2 21 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35 22 | 23 | goroutine 3 [IO wait, 1 minutes]: 24 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0) 25 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 26 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786) 27 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 28 | internal/poll.(*pollDesc).waitRead(...) 29 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 30 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) 31 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc 32 | net.(*netFD).accept(0xc00019e000, 0x7d667d63cbbded3e, 0x1789ccbbded3e, 0x100000001) 33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45 34 | net.(*TCPListener).accept(0xc000188060, 0x60006709, 0xc000196da8, 0x109abe6) 35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32 36 | net.(*TCPListener).Accept(0xc000188060, 0xc000196df8, 0x18, 0xc000001200, 0x12e9eec) 37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65 38 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0) 39 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266 40 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536) 41 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7 42 | net/http.ListenAndServe(...) 43 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 44 | main.main.func1(0xc000032120) 45 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126 46 | created by main.main 47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5 48 | 49 | goroutine 4 [sleep, 1 minutes]: 50 | time.Sleep(0x3b9aca00) 51 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 52 | main.shortSleepLoop() 53 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 54 | created by main.main 55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195 56 | 57 | goroutine 5 [sleep, 1 minutes]: 58 | time.Sleep(0x34630b8a000) 59 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 60 | main.sleepLoop(0x34630b8a000) 61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b 62 | created by main.main 63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc 64 | 65 | goroutine 6 [chan receive, 1 minutes]: 66 | main.chanReceiveForever() 67 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d 68 | created by main.main 69 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4 70 | 71 | goroutine 24 [select, 1 minutes]: 72 | net/http.(*persistConn).writeLoop(0xc0000cea20) 73 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 +0x11c 74 | created by net/http.(*Transport).dialConn 75 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1709 +0xcdc 76 | 77 | goroutine 23 [IO wait, 1 minutes]: 78 | internal/poll.runtime_pollWait(0x1e91da0, 0x72, 0x14e6ca0) 79 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 80 | internal/poll.(*pollDesc).wait(0xc00010e198, 0x72, 0x14e6c00, 0x16db878, 0x0) 81 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 82 | internal/poll.(*pollDesc).waitRead(...) 83 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 84 | internal/poll.(*FD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 85 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5 86 | net.(*netFD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x103b1dc, 0xc000199b58, 0x10680e0) 87 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f 88 | net.(*conn).Read(0xc000010008, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 89 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e 90 | net/http.(*persistConn).Read(0xc0000cea20, 0xc000256000, 0x1000, 0x1000, 0xc00009e300, 0xc000199c58, 0x10074b5) 91 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 +0x77 92 | bufio.(*Reader).fill(0xc0001801e0) 93 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105 94 | bufio.(*Reader).Peek(0xc0001801e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0001d0060) 95 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 +0x4f 96 | net/http.(*persistConn).readLoop(0xc0000cea20) 97 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 +0x1a8 98 | created by net/http.(*Transport).dialConn 99 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1708 +0xcb7 100 | 101 | goroutine 41 [IO wait, 1 minutes]: 102 | internal/poll.runtime_pollWait(0x1e91cb8, 0x72, 0x14e6ca0) 103 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 104 | internal/poll.(*pollDesc).wait(0xc00019e098, 0x72, 0x14e6c00, 0x16db878, 0x0) 105 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 106 | internal/poll.(*pollDesc).waitRead(...) 107 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 108 | internal/poll.(*FD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 109 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5 110 | net.(*netFD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x203000, 0x203000, 0x203000) 111 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f 112 | net.(*conn).Read(0xc000186028, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 113 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e 114 | net/http.(*connReader).Read(0xc00007c300, 0xc000326000, 0x1000, 0x1000, 0x100000006, 0x10, 0x1819408) 115 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:798 +0x1ad 116 | bufio.(*Reader).fill(0xc000290060) 117 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105 118 | bufio.(*Reader).ReadSlice(0xc000290060, 0xa, 0x1819408, 0xc000337988, 0x100f6d0, 0xc000110000, 0x100) 119 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:360 +0x3d 120 | bufio.(*Reader).ReadLine(0xc000290060, 0xc000110000, 0x1079694, 0xc0001a4000, 0x0, 0x1010038, 0x30) 121 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:389 +0x34 122 | net/textproto.(*Reader).readLineSlice(0xc000182300, 0xc000110000, 0x10d7c4d, 0xc00019e080, 0x1068000, 0xc000282900) 123 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:58 +0x6c 124 | net/textproto.(*Reader).ReadLine(...) 125 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:39 126 | net/http.readRequest(0xc000290060, 0x0, 0xc000110000, 0x0, 0x0) 127 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/request.go:1012 +0xaa 128 | net/http.(*conn).readRequest(0xc0000c6320, 0x14ed4a0, 0xc000322000, 0x0, 0x0, 0x0) 129 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:984 +0x19a 130 | net/http.(*conn).serve(0xc0000c6320, 0x14ed4a0, 0xc000322000) 131 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1851 +0x705 132 | created by net/http.(*Server).Serve 133 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2969 +0x36c 134 | -------------------------------------------------------------------------------- /examples/goroutine/2.runtime.goroutineprofile.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Stack0": [ 4 | 20629256, 5 | 20629212, 6 | 20627047, 7 | 20628306, 8 | 17018153, 9 | 17235329, 10 | 0, 11 | 0, 12 | 0, 13 | 0, 14 | 0, 15 | 0, 16 | 0, 17 | 0, 18 | 0, 19 | 0, 20 | 0, 21 | 0, 22 | 0, 23 | 0, 24 | 0, 25 | 0, 26 | 0, 27 | 0, 28 | 0, 29 | 0, 30 | 0, 31 | 0, 32 | 0, 33 | 0, 34 | 0, 35 | 0 36 | ] 37 | }, 38 | { 39 | "Stack0": [ 40 | 17019173, 41 | 17222943, 42 | 20628554, 43 | 17235329, 44 | 0, 45 | 0, 46 | 0, 47 | 0, 48 | 0, 49 | 0, 50 | 0, 51 | 0, 52 | 0, 53 | 0, 54 | 0, 55 | 0, 56 | 0, 57 | 0, 58 | 0, 59 | 0, 60 | 0, 61 | 0, 62 | 0, 63 | 0, 64 | 0, 65 | 0, 66 | 0, 67 | 0, 68 | 0, 69 | 0, 70 | 0, 71 | 0 72 | ] 73 | }, 74 | { 75 | "Stack0": [ 76 | 17019173, 77 | 16990811, 78 | 17211861, 79 | 17662341, 80 | 17673596, 81 | 17673566, 82 | 18587653, 83 | 18694226, 84 | 18689765, 85 | 19832294, 86 | 19831575, 87 | 20630758, 88 | 20630672, 89 | 17235329, 90 | 0, 91 | 0, 92 | 0, 93 | 0, 94 | 0, 95 | 0, 96 | 0, 97 | 0, 98 | 0, 99 | 0, 100 | 0, 101 | 0, 102 | 0, 103 | 0, 104 | 0, 105 | 0, 106 | 0, 107 | 0 108 | ] 109 | }, 110 | { 111 | "Stack0": [ 112 | 17019173, 113 | 17222943, 114 | 20628554, 115 | 17235329, 116 | 0, 117 | 0, 118 | 0, 119 | 0, 120 | 0, 121 | 0, 122 | 0, 123 | 0, 124 | 0, 125 | 0, 126 | 0, 127 | 0, 128 | 0, 129 | 0, 130 | 0, 131 | 0, 132 | 0, 133 | 0, 134 | 0, 135 | 0, 136 | 0, 137 | 0, 138 | 0, 139 | 0, 140 | 0, 141 | 0, 142 | 0, 143 | 0 144 | ] 145 | }, 146 | { 147 | "Stack0": [ 148 | 17019173, 149 | 17222943, 150 | 20628619, 151 | 17235329, 152 | 0, 153 | 0, 154 | 0, 155 | 0, 156 | 0, 157 | 0, 158 | 0, 159 | 0, 160 | 0, 161 | 0, 162 | 0, 163 | 0, 164 | 0, 165 | 0, 166 | 0, 167 | 0, 168 | 0, 169 | 0, 170 | 0, 171 | 0, 172 | 0, 173 | 0, 174 | 0, 175 | 0, 176 | 0, 177 | 0, 178 | 0, 179 | 0 180 | ] 181 | }, 182 | { 183 | "Stack0": [ 184 | 17019173, 185 | 16810991, 186 | 16810027, 187 | 20628717, 188 | 17235329, 189 | 0, 190 | 0, 191 | 0, 192 | 0, 193 | 0, 194 | 0, 195 | 0, 196 | 0, 197 | 0, 198 | 0, 199 | 0, 200 | 0, 201 | 0, 202 | 0, 203 | 0, 204 | 0, 205 | 0, 206 | 0, 207 | 0, 208 | 0, 209 | 0, 210 | 0, 211 | 0, 212 | 0, 213 | 0, 214 | 0, 215 | 0 216 | ] 217 | }, 218 | { 219 | "Stack0": [ 220 | 17019173, 221 | 17084367, 222 | 19931708, 223 | 17235329, 224 | 0, 225 | 0, 226 | 0, 227 | 0, 228 | 0, 229 | 0, 230 | 0, 231 | 0, 232 | 0, 233 | 0, 234 | 0, 235 | 0, 236 | 0, 237 | 0, 238 | 0, 239 | 0, 240 | 0, 241 | 0, 242 | 0, 243 | 0, 244 | 0, 245 | 0, 246 | 0, 247 | 0, 248 | 0, 249 | 0, 250 | 0, 251 | 0 252 | ] 253 | }, 254 | { 255 | "Stack0": [ 256 | 17019173, 257 | 16990811, 258 | 17211861, 259 | 17662341, 260 | 17666501, 261 | 17666467, 262 | 18582159, 263 | 18659118, 264 | 19921207, 265 | 18286533, 266 | 18286895, 267 | 19924456, 268 | 17235329, 269 | 0, 270 | 0, 271 | 0, 272 | 0, 273 | 0, 274 | 0, 275 | 0, 276 | 0, 277 | 0, 278 | 0, 279 | 0, 280 | 0, 281 | 0, 282 | 0, 283 | 0, 284 | 0, 285 | 0, 286 | 0, 287 | 0 288 | ] 289 | }, 290 | { 291 | "Stack0": [ 292 | 17019173, 293 | 16990811, 294 | 17211861, 295 | 17662341, 296 | 17666501, 297 | 17666467, 298 | 18582159, 299 | 18659118, 300 | 19789101, 301 | 18286533, 302 | 18289949, 303 | 18290516, 304 | 19282988, 305 | 19764938, 306 | 19764985, 307 | 19794362, 308 | 19812485, 309 | 17235329, 310 | 0, 311 | 0, 312 | 0, 313 | 0, 314 | 0, 315 | 0, 316 | 0, 317 | 0, 318 | 0, 319 | 0, 320 | 0, 321 | 0, 322 | 0, 323 | 0 324 | ] 325 | } 326 | ] 327 | -------------------------------------------------------------------------------- /examples/goroutine/2.runtime.stack.txt: -------------------------------------------------------------------------------- 1 | goroutine 1 [running]: 2 | main.glob..func1(0x14e5940, 0xc0000aa7b0, 0xc000064eb0, 0x2) 3 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:29 +0x6f 4 | main.writeProfiles(0x2, 0xc0000c4008, 0x1466424) 5 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187 6 | main.main() 7 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152 +0x3d2 8 | 9 | goroutine 22 [sleep, 1 minutes]: 10 | time.Sleep(0x3b9aca00) 11 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 12 | main.shortSleepLoop() 13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 14 | created by main.indirectShortSleepLoop2 15 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35 16 | 17 | goroutine 3 [IO wait, 1 minutes]: 18 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0) 19 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 20 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786) 21 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 22 | internal/poll.(*pollDesc).waitRead(...) 23 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 24 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) 25 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc 26 | net.(*netFD).accept(0xc00019e000, 0x7d667d63cbbded3e, 0x1789ccbbded3e, 0x100000001) 27 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45 28 | net.(*TCPListener).accept(0xc000188060, 0x60006709, 0xc000196da8, 0x109abe6) 29 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32 30 | net.(*TCPListener).Accept(0xc000188060, 0xc000196df8, 0x18, 0xc000001200, 0x12e9eec) 31 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65 32 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0) 33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266 34 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536) 35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7 36 | net/http.ListenAndServe(...) 37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120 38 | main.main.func1(0xc000032120) 39 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126 40 | created by main.main 41 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5 42 | 43 | goroutine 4 [sleep, 1 minutes]: 44 | time.Sleep(0x3b9aca00) 45 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 46 | main.shortSleepLoop() 47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a 48 | created by main.main 49 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195 50 | 51 | goroutine 5 [sleep, 1 minutes]: 52 | time.Sleep(0x34630b8a000) 53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 54 | main.sleepLoop(0x34630b8a000) 55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b 56 | created by main.main 57 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc 58 | 59 | goroutine 6 [chan receive, 1 minutes]: 60 | main.chanReceiveForever() 61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d 62 | created by main.main 63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4 64 | 65 | goroutine 24 [select, 1 minutes]: 66 | net/http.(*persistConn).writeLoop(0xc0000cea20) 67 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 +0x11c 68 | created by net/http.(*Transport).dialConn 69 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1709 +0xcdc 70 | 71 | goroutine 23 [IO wait, 1 minutes]: 72 | internal/poll.runtime_pollWait(0x1e91da0, 0x72, 0x14e6ca0) 73 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 74 | internal/poll.(*pollDesc).wait(0xc00010e198, 0x72, 0x14e6c00, 0x16db878, 0x0) 75 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 76 | internal/poll.(*pollDesc).waitRead(...) 77 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 78 | internal/poll.(*FD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 79 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5 80 | net.(*netFD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x103b1dc, 0xc000199b58, 0x10680e0) 81 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f 82 | net.(*conn).Read(0xc000010008, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 83 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e 84 | net/http.(*persistConn).Read(0xc0000cea20, 0xc000256000, 0x1000, 0x1000, 0xc00009e300, 0xc000199c58, 0x10074b5) 85 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 +0x77 86 | bufio.(*Reader).fill(0xc0001801e0) 87 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105 88 | bufio.(*Reader).Peek(0xc0001801e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0001d0060) 89 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 +0x4f 90 | net/http.(*persistConn).readLoop(0xc0000cea20) 91 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 +0x1a8 92 | created by net/http.(*Transport).dialConn 93 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1708 +0xcb7 94 | 95 | goroutine 41 [IO wait, 1 minutes]: 96 | internal/poll.runtime_pollWait(0x1e91cb8, 0x72, 0x14e6ca0) 97 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55 98 | internal/poll.(*pollDesc).wait(0xc00019e098, 0x72, 0x14e6c00, 0x16db878, 0x0) 99 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45 100 | internal/poll.(*pollDesc).waitRead(...) 101 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92 102 | internal/poll.(*FD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 103 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5 104 | net.(*netFD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x203000, 0x203000, 0x203000) 105 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f 106 | net.(*conn).Read(0xc000186028, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0) 107 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e 108 | net/http.(*connReader).Read(0xc00007c300, 0xc000326000, 0x1000, 0x1000, 0x100000006, 0x10, 0x1819408) 109 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:798 +0x1ad 110 | bufio.(*Reader).fill(0xc000290060) 111 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105 112 | bufio.(*Reader).ReadSlice(0xc000290060, 0xa, 0x1819408, 0xc000337988, 0x100f6d0, 0xc000110000, 0x100) 113 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:360 +0x3d 114 | bufio.(*Reader).ReadLine(0xc000290060, 0xc000110000, 0x1079694, 0xc0001a4000, 0x0, 0x1010038, 0x30) 115 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:389 +0x34 116 | net/textproto.(*Reader).readLineSlice(0xc000182300, 0xc000110000, 0x10d7c4d, 0xc00019e080, 0x1068000, 0xc000282900) 117 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:58 +0x6c 118 | net/textproto.(*Reader).ReadLine(...) 119 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:39 120 | net/http.readRequest(0xc000290060, 0x0, 0xc000110000, 0x0, 0x0) 121 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/request.go:1012 +0xaa 122 | net/http.(*conn).readRequest(0xc0000c6320, 0x14ed4a0, 0xc000322000, 0x0, 0x0, 0x0) 123 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:984 +0x19a 124 | net/http.(*conn).serve(0xc0000c6320, 0x14ed4a0, 0xc000322000) 125 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1851 +0x705 126 | created by net/http.(*Server).Serve 127 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2969 +0x36c 128 | -------------------------------------------------------------------------------- /examples/goroutine/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/goroutine 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /examples/goroutine/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "flag" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "net/http" 13 | _ "net/http/pprof" 14 | "runtime" 15 | "runtime/pprof" 16 | "time" 17 | ) 18 | 19 | type Profile struct { 20 | Name string 21 | WriteTo func(w io.Writer) error 22 | } 23 | 24 | var profiles = []Profile{ 25 | { 26 | Name: "runtime.stack.txt", 27 | WriteTo: func(w io.Writer) error { 28 | buf := make([]byte, 1024*1024) 29 | n := runtime.Stack(buf, true) 30 | buf = buf[:n] 31 | _, err := w.Write(buf) 32 | return err 33 | }, 34 | }, 35 | { 36 | Name: "runtime.goroutineprofile.json", 37 | WriteTo: func(w io.Writer) error { 38 | p := make([]runtime.StackRecord, 1000) 39 | n, ok := runtime.GoroutineProfile(p) 40 | if !ok { 41 | return errors.New("runtime.GoroutineProfile: not ok") 42 | } 43 | p = p[0:n] 44 | e := json.NewEncoder(w) 45 | e.SetIndent("", " ") 46 | return e.Encode(p) 47 | }, 48 | }, 49 | { 50 | Name: "pprof.lookup.goroutine.debug0.pb.gz", 51 | WriteTo: func(w io.Writer) error { 52 | profile := pprof.Lookup("goroutine") 53 | return profile.WriteTo(w, 0) 54 | }, 55 | }, 56 | { 57 | Name: "pprof.lookup.goroutine.debug1.txt", 58 | WriteTo: func(w io.Writer) error { 59 | profile := pprof.Lookup("goroutine") 60 | return profile.WriteTo(w, 1) 61 | }, 62 | }, 63 | { 64 | Name: "pprof.lookup.goroutine.debug2.txt", 65 | WriteTo: func(w io.Writer) error { 66 | profile := pprof.Lookup("goroutine") 67 | return profile.WriteTo(w, 2) 68 | }, 69 | }, 70 | { 71 | Name: "net.http.pprof.goroutine.debug0.pb.gz", 72 | WriteTo: func(w io.Writer) error { 73 | return writeHttpProfile(w, 0) 74 | }, 75 | }, 76 | { 77 | Name: "net.http.pprof.goroutine.debug1.txt", 78 | WriteTo: func(w io.Writer) error { 79 | return writeHttpProfile(w, 1) 80 | }, 81 | }, 82 | { 83 | Name: "net.http.pprof.goroutine.debug2.txt", 84 | WriteTo: func(w io.Writer) error { 85 | return writeHttpProfile(w, 2) 86 | }, 87 | }, 88 | } 89 | 90 | func writeHttpProfile(w io.Writer, debug int) error { 91 | url := fmt.Sprintf("http://%s/debug/pprof/goroutine?debug=%d", listenAddr, debug) 92 | res, err := http.Get(url) 93 | if err != nil { 94 | return err 95 | } 96 | defer res.Body.Close() 97 | 98 | _, err = io.Copy(w, res.Body) 99 | return err 100 | } 101 | 102 | func writeProfiles(n int) error { 103 | for _, profile := range profiles { 104 | buf := &bytes.Buffer{} 105 | filename := fmt.Sprintf("%d.%s", n, profile.Name) 106 | if err := profile.WriteTo(buf); err != nil { 107 | return err 108 | } else if err := ioutil.WriteFile(filename, buf.Bytes(), 0666); err != nil { 109 | return err 110 | } 111 | } 112 | return nil 113 | } 114 | 115 | var listenAddr = "127.0.0.1:8080" 116 | 117 | func main() { 118 | flag.Parse() 119 | 120 | errCh := make(chan error, 1) 121 | go func() { 122 | fmt.Printf("Listening for pprof requests on %s\n", listenAddr) 123 | errCh <- http.ListenAndServe(listenAddr, nil) 124 | }() 125 | 126 | labels := pprof.Labels("test_label", "test_value") 127 | ctx := pprof.WithLabels(context.Background(), labels) 128 | pprof.SetGoroutineLabels(ctx) 129 | 130 | go shortSleepLoop() 131 | go sleepLoop(time.Hour) 132 | go chanReceiveForever() 133 | go indirectShortSleepLoop() 134 | 135 | sleep := time.Second 136 | fmt.Printf("Sleeping for %s followed by gc\n", sleep) 137 | 138 | time.Sleep(time.Second) 139 | runtime.GC() 140 | 141 | fmt.Printf("Dump 1\n") 142 | if err := writeProfiles(1); err != nil { 143 | panic(err) 144 | } 145 | 146 | sleep = 1*time.Minute + 10*time.Second 147 | fmt.Printf("Sleeping for %s followed by gc\n", sleep) 148 | time.Sleep(sleep) 149 | runtime.GC() 150 | 151 | fmt.Printf("Dump 2\n") 152 | if err := writeProfiles(2); err != nil { 153 | panic(err) 154 | } 155 | 156 | fmt.Printf("Waiting forever\n") 157 | 158 | //fmt.Printf("waiting indefinitely so you can press ctrl+\\ to compare the output\n") 159 | //runtime.GC() 160 | <-errCh 161 | } 162 | 163 | func shortSleepLoop() { 164 | for { 165 | time.Sleep(time.Second) 166 | } 167 | } 168 | 169 | func sleepLoop(d time.Duration) { 170 | for { 171 | time.Sleep(d) 172 | } 173 | } 174 | 175 | func chanReceiveForever() { 176 | forever := make(chan struct{}) 177 | <-forever 178 | } 179 | 180 | func indirectShortSleepLoop() { 181 | indirectShortSleepLoop2() 182 | } 183 | 184 | func indirectShortSleepLoop2() { 185 | go shortSleepLoop() 186 | } 187 | -------------------------------------------------------------------------------- /examples/goroutine/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func BenchmarkProfilerGoroutines(b *testing.B) { 11 | max := 1024 * 1024 12 | buf := make([]runtime.StackRecord, max*2) 13 | for g := 1; g <= max; g = g * 2 { 14 | g := g 15 | name := fmt.Sprintf("%d goroutines", g) 16 | 17 | b.Run(name, func(b *testing.B) { 18 | initalRoutines := runtime.NumGoroutine() 19 | 20 | readyCh := make(chan struct{}) 21 | stopCh := make(chan struct{}) 22 | for i := 0; i < g; i++ { 23 | go atStackDepth(16, func() { 24 | defer func() { stopCh <- struct{}{} }() 25 | readyCh <- struct{}{} 26 | }) 27 | <-readyCh 28 | } 29 | 30 | b.ResetTimer() 31 | for i := 0; i < b.N; i++ { 32 | n, ok := runtime.GoroutineProfile(buf) 33 | if !ok { 34 | b.Logf("GoroutineProfile not ok") 35 | } else if gotRoutines := n - initalRoutines; gotRoutines != g { 36 | b.Logf("want %d goroutines, but got %d on iteration %d", g, gotRoutines, i) 37 | } 38 | } 39 | b.StopTimer() 40 | for i := 0; i < g; i++ { 41 | <-stopCh 42 | } 43 | start := time.Now() 44 | for i := 0; ; i++ { 45 | if runtime.NumGoroutine() == initalRoutines { 46 | break 47 | } 48 | time.Sleep(20 * time.Millisecond) 49 | if time.Since(start) > 10*time.Second { 50 | b.Fatalf("%d goroutines still running, want %d", runtime.NumGoroutine(), initalRoutines) 51 | } 52 | } 53 | }) 54 | } 55 | } 56 | 57 | func atStackDepth(depth int, fn func()) { 58 | pcs := make([]uintptr, depth+10) 59 | n := runtime.Callers(1, pcs) 60 | if n > depth { 61 | panic("depth exceeded") 62 | } else if n < depth { 63 | atStackDepth(depth, fn) 64 | return 65 | } 66 | 67 | fn() 68 | } 69 | -------------------------------------------------------------------------------- /examples/guide/main: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/guide/main -------------------------------------------------------------------------------- /examples/guide/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | func main() { 8 | start := time.Now() 9 | for time.Since(start) < time.Second { 10 | } 11 | //time.Sleep(time.Second) 12 | } 13 | -------------------------------------------------------------------------------- /examples/guide/main2.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "runtime/trace" 6 | "time" 7 | ) 8 | 9 | func main() { 10 | trace.Start(os.Stdout) 11 | defer trace.Stop() 12 | 13 | go b() 14 | a() 15 | } 16 | 17 | func a() { 18 | start := time.Now() 19 | for time.Since(start) < time.Second { 20 | } 21 | } 22 | 23 | func b() { 24 | start := time.Now() 25 | for time.Since(start) < time.Second { 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /examples/guide/main3.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "runtime/trace" 8 | ) 9 | 10 | func main() { 11 | trace.Start(os.Stderr) 12 | defer trace.Stop() 13 | 14 | res, err := http.Get("https://example.org/") 15 | if err != nil { 16 | panic(err) 17 | } 18 | fmt.Printf("%d\n", res.StatusCode) 19 | } 20 | -------------------------------------------------------------------------------- /examples/guide/trace.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/guide/trace.bin -------------------------------------------------------------------------------- /examples/memory/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/memory 2 | 3 | go 1.15 4 | 5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a 6 | -------------------------------------------------------------------------------- /examples/memory/go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= 2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 3 | -------------------------------------------------------------------------------- /examples/memory/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | "runtime" 11 | "time" 12 | 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | func main() { 17 | if err := run(); err != nil { 18 | fmt.Fprintln(os.Stderr, err) 19 | os.Exit(1) 20 | } 21 | } 22 | 23 | func run() error { 24 | runtime.MemProfileRate = 1 25 | 26 | g, _ := errgroup.WithContext(context.Background()) 27 | 28 | g.Go(func() error { 29 | addr := "localhost:6060" 30 | log.Printf("Listening on %s", addr) 31 | return http.ListenAndServe(addr, nil) 32 | }) 33 | 34 | //g.Go(allocStuff) 35 | 36 | // Memory leak that leaks 1 kB / sec at a rate of 10 Hz. 37 | g.Go(func() error { return leakStuff(1024, 100*time.Millisecond) }) 38 | g.Go(func() error { return allocStuff(1024, 100*time.Millisecond) }) 39 | g.Go(func() error { return forceGc(time.Second) }) 40 | 41 | return g.Wait() 42 | } 43 | 44 | var leak []*Data 45 | 46 | // leakStuff leaks ~bytes every interval. 47 | func leakStuff(bytes int, interval time.Duration) error { 48 | for { 49 | leak = append(leak, newData(bytes)) 50 | time.Sleep(interval) 51 | } 52 | } 53 | 54 | // allocStuff is allocating things but not leaking them. 55 | func allocStuff(bytes int, interval time.Duration) error { 56 | for { 57 | newData(bytes) 58 | time.Sleep(interval) 59 | } 60 | } 61 | 62 | // forceGc forces a GC to occur every interval. 63 | func forceGc(interval time.Duration) error { 64 | for { 65 | time.Sleep(interval) 66 | runtime.GC() 67 | } 68 | } 69 | 70 | func newData(size int) *Data { 71 | return &Data{data: make([]byte, size)} 72 | } 73 | 74 | type Data struct { 75 | data []byte 76 | } 77 | -------------------------------------------------------------------------------- /examples/pclnttab/darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | package main 5 | 6 | import ( 7 | "debug/gosym" 8 | "debug/macho" 9 | "os" 10 | ) 11 | 12 | // from https://github.com/lizrice/debugger-from-scratch/blob/master/symbols.go 13 | func goSymTable() (*gosym.Table, error) { 14 | exe, err := macho.Open(os.Args[0]) 15 | if err != nil { 16 | return nil, nil 17 | } 18 | defer exe.Close() 19 | 20 | addr := exe.Section("__text").Addr 21 | 22 | lineTableData, err := exe.Section("__gopclntab").Data() 23 | if err != nil { 24 | return nil, nil 25 | } 26 | lineTable := gosym.NewLineTable(lineTableData, addr) 27 | if err != nil { 28 | return nil, nil 29 | } 30 | 31 | symTableData, err := exe.Section("__gosymtab").Data() 32 | if err != nil { 33 | return nil, nil 34 | } 35 | 36 | return gosym.NewTable(symTableData, lineTable) 37 | } 38 | -------------------------------------------------------------------------------- /examples/pclnttab/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/pclnttab 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /examples/pclnttab/linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package main 5 | 6 | import ( 7 | "debug/elf" 8 | "errors" 9 | "fmt" 10 | "os" 11 | ) 12 | 13 | // from https://github.com/lizrice/debugger-from-scratch/blob/master/symbols.go 14 | func goSymTable() (*gosym.Table, error) { 15 | exe, err := elf.Open(os.Args[0]) 16 | if err != nil { 17 | return nil, nil 18 | } 19 | defer exe.Close() 20 | 21 | addr := exe.Section(".text").Addr 22 | 23 | lineTableData, err := exe.Section(".gopclntab").Data() 24 | if err != nil { 25 | return nil, nil 26 | } 27 | lineTable := gosym.NewLineTable(lineTableData, addr) 28 | if err != nil { 29 | return nil, nil 30 | } 31 | 32 | symTableData, err := exe.Section(".gosymtab").Data() 33 | if err != nil { 34 | return nil, nil 35 | } 36 | 37 | return gosym.NewTable(symTableData, lineTable) 38 | } 39 | -------------------------------------------------------------------------------- /examples/pclnttab/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | ) 8 | 9 | func main() { 10 | if err := run(); err != nil { 11 | fmt.Fprintln(os.Stderr, err) 12 | os.Exit(1) 13 | } 14 | } 15 | 16 | func run() error { 17 | symTable, err := goSymTable() 18 | if err != nil { 19 | return err 20 | } 21 | for _, pc := range callers() { 22 | file, line, fn := symTable.PCToLine(uint64(pc)) 23 | fmt.Printf("%x: %s() %s:%d\n", pc, fn.Name, file, line) 24 | } 25 | return nil 26 | } 27 | 28 | func callers() []uintptr { 29 | pcs := make([]uintptr, 32) 30 | n := runtime.Callers(2, pcs) 31 | pcs = pcs[0:n] 32 | return pcs 33 | } 34 | -------------------------------------------------------------------------------- /examples/presentation/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func main() { 8 | foo() 9 | foobar() 10 | } 11 | 12 | func foo() { 13 | bar() 14 | } 15 | 16 | func bar() { 17 | fmt.Printf("Hello from bar\n") 18 | panic("oh no") 19 | } 20 | 21 | func foobar() { 22 | fmt.Printf("Hello from foobar\n") 23 | } 24 | -------------------------------------------------------------------------------- /examples/presentation/main.simple.txt: -------------------------------------------------------------------------------- 1 | 0x10a46a0 mov rcx, qword ptr gs:[0x30] 2 | 0x10a46a9 cmp rsp, qword ptr [rcx+0x10] 3 | 0x10a46ad jbe 0x10a46d3 4 | 0x10a46af sub rsp, 0x8 5 | 0x10a46b3 mov qword ptr [rsp], rbp 6 | 0x10a46b7 lea rbp, ptr [rsp] 7 | 0x10a46bb nop dword ptr [rax+rax*1], eax 8 | 0x10a46c0 call $main.foo 9 | 0x10a46c5 call $main.foobar 10 | 0x10a46ca mov rbp, qword ptr [rsp] 11 | 0x10a46ce add rsp, 0x8 12 | 0x10a46d2 ret 13 | 0x10a46d3 call $runtime.morestack_noctxt 14 | 0x10a46d8 jmp $main.main 15 | 0x10a46e0 mov rcx, qword ptr gs:[0x30] 16 | 0x10a46e9 cmp rsp, qword ptr [rcx+0x10] 17 | 0x10a46ed jbe 0x10a470e 18 | 0x10a46ef sub rsp, 0x8 19 | 0x10a46f3 mov qword ptr [rsp], rbp 20 | 0x10a46f7 lea rbp, ptr [rsp] 21 | 0x10a46fb nop dword ptr [rax+rax*1], eax 22 | 0x10a4700 call $main.bar 23 | 0x10a4705 mov rbp, qword ptr [rsp] 24 | 0x10a4709 add rsp, 0x8 25 | 0x10a470d ret 26 | 0x10a470e call $runtime.morestack_noctxt 27 | 0x10a4713 jmp $main.foo 28 | 0x10a4720 mov rcx, qword ptr gs:[0x30] 29 | 0x10a4729 cmp rsp, qword ptr [rcx+0x10] 30 | 0x10a472d jbe 0x10a4771 31 | 0x10a472f sub rsp, 0x48 32 | 0x10a4733 mov qword ptr [rsp+0x40], rbp 33 | 0x10a4738 lea rbp, ptr [rsp+0x40] 34 | 0x10a473d lea rax, ptr [rip+0x25c23] 35 | 0x10a4744 mov qword ptr [rsp], rax 36 | 0x10a4748 mov qword ptr [rsp+0x8], 0xf 37 | 0x10a4751 xorps xmm0, xmm0 38 | 0x10a4754 movups xmmword ptr [rsp+0x10], xmm0 39 | 0x10a4759 mov qword ptr [rsp+0x20], 0x0 40 | 0x10a4762 call $fmt.Printf 41 | 0x10a4767 mov rbp, qword ptr [rsp+0x40] 42 | 0x10a476c add rsp, 0x48 43 | 0x10a4770 ret 44 | 0x10a4771 call $runtime.morestack_noctxt 45 | 0x10a4776 jmp $main.bar 46 | 0x10a4780 mov rcx, qword ptr gs:[0x30] 47 | 0x10a4789 cmp rsp, qword ptr [rcx+0x10] 48 | 0x10a478d jbe 0x10a47d1 49 | 0x10a478f sub rsp, 0x48 50 | 0x10a4793 mov qword ptr [rsp+0x40], rbp 51 | 0x10a4798 lea rbp, ptr [rsp+0x40] 52 | 0x10a479d lea rax, ptr [rip+0x263ed] 53 | 0x10a47a4 mov qword ptr [rsp], rax 54 | 0x10a47a8 mov qword ptr [rsp+0x8], 0x12 55 | 0x10a47b1 xorps xmm0, xmm0 56 | 0x10a47b4 movups xmmword ptr [rsp+0x10], xmm0 57 | 0x10a47b9 mov qword ptr [rsp+0x20], 0x0 58 | 0x10a47c2 call $fmt.Printf 59 | 0x10a47c7 mov rbp, qword ptr [rsp+0x40] 60 | 0x10a47cc add rsp, 0x48 61 | 0x10a47d0 ret 62 | 0x10a47d1 call $runtime.morestack_noctxt 63 | 0x10a47d6 jmp $main.foobar 64 | -------------------------------------------------------------------------------- /examples/presentation/main.txt: -------------------------------------------------------------------------------- 1 | (dlv) disassemble -l main.main 2 | TEXT main.main(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go 3 | main.go:7 0x10a46a0 65488b0c2530000000 mov rcx, qword ptr gs:[0x30] 4 | main.go:7 0x10a46a9 483b6110 cmp rsp, qword ptr [rcx+0x10] 5 | main.go:7 0x10a46ad 7624 jbe 0x10a46d3 6 | main.go:7 0x10a46af 4883ec08 sub rsp, 0x8 7 | main.go:7 0x10a46b3 48892c24 mov qword ptr [rsp], rbp 8 | main.go:7 0x10a46b7 488d2c24 lea rbp, ptr [rsp] 9 | main.go:8 0x10a46bb 0f1f440000 nop dword ptr [rax+rax*1], eax 10 | main.go:8 0x10a46c0 e81b000000 call $main.foo 11 | main.go:9 0x10a46c5 e8b6000000 call $main.foobar 12 | main.go:10 0x10a46ca 488b2c24 mov rbp, qword ptr [rsp] 13 | main.go:10 0x10a46ce 4883c408 add rsp, 0x8 14 | main.go:10 0x10a46d2 c3 ret 15 | main.go:7 0x10a46d3 e8a8ddfbff call $runtime.morestack_noctxt 16 | .:0 0x10a46d8 ebc6 jmp $main.main 17 | (dlv) disassemble -l main.foo 18 | TEXT main.foo(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go 19 | main.go:12 0x10a46e0 65488b0c2530000000 mov rcx, qword ptr gs:[0x30] 20 | main.go:12 0x10a46e9 483b6110 cmp rsp, qword ptr [rcx+0x10] 21 | main.go:12 0x10a46ed 761f jbe 0x10a470e 22 | main.go:12 0x10a46ef 4883ec08 sub rsp, 0x8 23 | main.go:12 0x10a46f3 48892c24 mov qword ptr [rsp], rbp 24 | main.go:12 0x10a46f7 488d2c24 lea rbp, ptr [rsp] 25 | main.go:13 0x10a46fb 0f1f440000 nop dword ptr [rax+rax*1], eax 26 | main.go:13 0x10a4700 e81b000000 call $main.bar 27 | main.go:14 0x10a4705 488b2c24 mov rbp, qword ptr [rsp] 28 | main.go:14 0x10a4709 4883c408 add rsp, 0x8 29 | main.go:14 0x10a470d c3 ret 30 | main.go:12 0x10a470e e86dddfbff call $runtime.morestack_noctxt 31 | .:0 0x10a4713 ebcb jmp $main.foo 32 | (dlv) disassemble -l main.bar 33 | TEXT main.bar(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go 34 | main.go:16 0x10a4720 65488b0c2530000000 mov rcx, qword ptr gs:[0x30] 35 | main.go:16 0x10a4729 483b6110 cmp rsp, qword ptr [rcx+0x10] 36 | main.go:16 0x10a472d 7642 jbe 0x10a4771 37 | main.go:16 0x10a472f 4883ec48 sub rsp, 0x48 38 | main.go:16 0x10a4733 48896c2440 mov qword ptr [rsp+0x40], rbp 39 | main.go:16 0x10a4738 488d6c2440 lea rbp, ptr [rsp+0x40] 40 | main.go:17 0x10a473d 488d05235c0200 lea rax, ptr [rip+0x25c23] 41 | main.go:17 0x10a4744 48890424 mov qword ptr [rsp], rax 42 | main.go:17 0x10a4748 48c74424080f000000 mov qword ptr [rsp+0x8], 0xf 43 | main.go:17 0x10a4751 0f57c0 xorps xmm0, xmm0 44 | main.go:17 0x10a4754 0f11442410 movups xmmword ptr [rsp+0x10], xmm0 45 | main.go:17 0x10a4759 48c744242000000000 mov qword ptr [rsp+0x20], 0x0 46 | main.go:17 0x10a4762 e8b982ffff call $fmt.Printf 47 | main.go:18 0x10a4767 488b6c2440 mov rbp, qword ptr [rsp+0x40] 48 | main.go:18 0x10a476c 4883c448 add rsp, 0x48 49 | main.go:18 0x10a4770 c3 ret 50 | main.go:16 0x10a4771 e80addfbff call $runtime.morestack_noctxt 51 | .:0 0x10a4776 eba8 jmp $main.bar 52 | (dlv) disassemble -l main.foobar 53 | TEXT main.foobar(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go 54 | main.go:20 0x10a4780 65488b0c2530000000 mov rcx, qword ptr gs:[0x30] 55 | main.go:20 0x10a4789 483b6110 cmp rsp, qword ptr [rcx+0x10] 56 | main.go:20 0x10a478d 7642 jbe 0x10a47d1 57 | main.go:20 0x10a478f 4883ec48 sub rsp, 0x48 58 | main.go:20 0x10a4793 48896c2440 mov qword ptr [rsp+0x40], rbp 59 | main.go:20 0x10a4798 488d6c2440 lea rbp, ptr [rsp+0x40] 60 | main.go:21 0x10a479d 488d05ed630200 lea rax, ptr [rip+0x263ed] 61 | main.go:21 0x10a47a4 48890424 mov qword ptr [rsp], rax 62 | main.go:21 0x10a47a8 48c744240812000000 mov qword ptr [rsp+0x8], 0x12 63 | main.go:21 0x10a47b1 0f57c0 xorps xmm0, xmm0 64 | main.go:21 0x10a47b4 0f11442410 movups xmmword ptr [rsp+0x10], xmm0 65 | main.go:21 0x10a47b9 48c744242000000000 mov qword ptr [rsp+0x20], 0x0 66 | main.go:21 0x10a47c2 e85982ffff call $fmt.Printf 67 | main.go:22 0x10a47c7 488b6c2440 mov rbp, qword ptr [rsp+0x40] 68 | main.go:22 0x10a47cc 4883c448 add rsp, 0x48 69 | main.go:22 0x10a47d0 c3 ret 70 | main.go:20 0x10a47d1 e8aadcfbff call $runtime.morestack_noctxt 71 | .:0 0x10a47d6 eba8 jmp $main.foobar 72 | -------------------------------------------------------------------------------- /examples/presentation/stack.txt: -------------------------------------------------------------------------------- 1 | panic: oh no 2 | 3 | goroutine 1 [running]: 4 | main.bar(...) 5 | /go/src/presentation/main.go:18 6 | main.foo() 7 | /go/src/presentation/main.go:13 +0x85 8 | main.main() 9 | /go/src/presentation/main.go:8 +0x25 10 | -------------------------------------------------------------------------------- /examples/pseudo-code/pseudo-block.txt: -------------------------------------------------------------------------------- 1 | func chansend(channel, msg): 2 | // non-blocking send 3 | if ready(channel): 4 | send(channel, msg) 5 | return 6 | 7 | t0 = now() 8 | // let scheduler run another goroutine 9 | // until the channel is ready 10 | wait_ready(channel) 11 | duration = now() - t0 12 | send(channel, msg) 13 | // sample a fraction of blocking events 14 | if random_sample(duration): 15 | s = stacktrace() 16 | profile[s].count++ 17 | profile[s].duration += duration 18 | -------------------------------------------------------------------------------- /examples/pseudo-code/pseudo-heap.txt: -------------------------------------------------------------------------------- 1 | func malloc(size): 2 | object = ... // alloc magic 3 | 4 | if random_sample(): 5 | s = stacktrace() 6 | profile[s].allocs++ 7 | profile[s].alloc_bytes += sizeof(object) 8 | track_profiled(object, s) 9 | 10 | return object 11 | 12 | func sweep(object): 13 | // do gc stuff to free object 14 | 15 | if is_profiled(object) 16 | s = alloc_stacktrace(object) 17 | profile[s].frees++ 18 | profile[s].free_bytes += sizeof(object) 19 | 20 | return object 21 | -------------------------------------------------------------------------------- /examples/runtime-stack/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/runtime-stack 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /examples/runtime-stack/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "time" 7 | ) 8 | 9 | func main() { 10 | sleep := 1*time.Minute + 10*time.Second 11 | 12 | go sleepLoop(sleep) 13 | go sleepLoop(time.Second) 14 | go chanReceiveForever() 15 | go indirectSleepLoop(time.Second) 16 | 17 | time.Sleep(time.Second) 18 | // g.wait 19 | runtime.GC() 20 | fmt.Printf("sleeping for %s before showing stack traces\n", sleep) 21 | time.Sleep(sleep) 22 | runtime.GC() 23 | 24 | fmt.Printf("%s has passed, dumping all stacks", sleep) 25 | buf := make([]byte, 1024*1024) 26 | n := runtime.Stack(buf, true) 27 | buf = buf[:n] 28 | fmt.Printf("%s\n", buf) 29 | 30 | fmt.Printf("waiting indefinitely so you can press ctrl+\\ to compare the output\n") 31 | runtime.GC() 32 | chanReceiveForever() 33 | } 34 | 35 | func sleepLoop(d time.Duration) { 36 | for { 37 | time.Sleep(d) 38 | } 39 | } 40 | 41 | func chanReceiveForever() { 42 | forever := make(chan struct{}) 43 | <-forever 44 | } 45 | 46 | func indirectSleepLoop(d time.Duration) { 47 | indirectSleepLoop2(d) 48 | } 49 | 50 | func indirectSleepLoop2(d time.Duration) { 51 | go sleepLoop(d) 52 | } 53 | -------------------------------------------------------------------------------- /examples/runtime-stack/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func BenchmarkRuntimeStack(b *testing.B) { 11 | buf := make([]byte, 1024*1024*64) 12 | for g := 1; g <= 1024*1024; g = g * 2 { 13 | g := g 14 | name := fmt.Sprintf("%d goroutines", g) 15 | 16 | b.Run(name, func(b *testing.B) { 17 | initalRoutines := runtime.NumGoroutine() 18 | 19 | readyCh := make(chan struct{}) 20 | stopCh := make(chan struct{}) 21 | for i := 0; i < g; i++ { 22 | go atStackDepth(16, func() { 23 | defer func() { stopCh <- struct{}{} }() 24 | readyCh <- struct{}{} 25 | }) 26 | <-readyCh 27 | } 28 | 29 | gotRoutines := runtime.NumGoroutine() - initalRoutines 30 | if gotRoutines != g { 31 | b.Logf("want %d goroutines, but got %d", g, gotRoutines) 32 | } 33 | 34 | b.ResetTimer() 35 | for i := 0; i < b.N; i++ { 36 | runtime.Stack(buf, true) 37 | } 38 | b.StopTimer() 39 | for i := 0; i < g; i++ { 40 | <-stopCh 41 | } 42 | start := time.Now() 43 | for i := 0; ; i++ { 44 | if runtime.NumGoroutine() == initalRoutines { 45 | break 46 | } 47 | time.Sleep(20 * time.Millisecond) 48 | if time.Since(start) > 10*time.Second { 49 | b.Fatalf("%d goroutines still running, want %d", runtime.NumGoroutine(), initalRoutines) 50 | } 51 | } 52 | }) 53 | } 54 | } 55 | 56 | func atStackDepth(depth int, fn func()) { 57 | pcs := make([]uintptr, depth*10) 58 | n := runtime.Callers(1, pcs) 59 | if n > depth { 60 | panic("depth exceeded") 61 | } else if n < depth { 62 | atStackDepth(depth, fn) 63 | return 64 | } 65 | 66 | fn() 67 | } 68 | -------------------------------------------------------------------------------- /examples/runtime-stack/runtime-stack.txt: -------------------------------------------------------------------------------- 1 | $ go run examples/runtime-stack/main.go 2 | sleeping for 1m10s before showing stack traces 3 | 1m10s has passed, dumping all stacksgoroutine 1 [running]: 4 | main.main() 5 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:26 +0x228 6 | 7 | goroutine 18 [sleep, 1 minutes]: 8 | time.Sleep(0x104c533c00) 9 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 10 | main.sleepLoop(0x104c533c00) 11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b 12 | created by main.main 13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:12 +0x56 14 | 15 | goroutine 19 [sleep, 1 minutes]: 16 | time.Sleep(0x3b9aca00) 17 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 18 | main.sleepLoop(0x3b9aca00) 19 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b 20 | created by main.main 21 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:13 +0x77 22 | 23 | goroutine 20 [chan receive, 1 minutes]: 24 | main.chanReceiveForever() 25 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:43 +0x4d 26 | created by main.main 27 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:14 +0x8f 28 | 29 | goroutine 22 [sleep, 1 minutes]: 30 | time.Sleep(0x3b9aca00) 31 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 32 | main.sleepLoop(0x3b9aca00) 33 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b 34 | created by main.indirectSleepLoop2 35 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:51 +0x3f 36 | 37 | waiting indefinitely so you can press ctrl+\ to compare the output 38 | ^\SIGQUIT: quit 39 | PC=0x7fff739f9882 m=0 sigcode=0 40 | 41 | goroutine 0 [idle]: 42 | runtime.pthread_cond_wait(0x1172620, 0x11725e0, 0x7ffe00000000) 43 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:414 +0x39 44 | runtime.semasleep(0xffffffffffffffff, 0x103af45) 45 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/os_darwin.go:63 +0x8d 46 | runtime.notesleep(0x11723e8) 47 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/lock_sema.go:181 +0xe7 48 | runtime.stopm() 49 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1924 +0xc5 50 | runtime.findrunnable(0xc000029800, 0x0) 51 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2485 +0xa7f 52 | runtime.schedule() 53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2683 +0x2d7 54 | runtime.park_m(0xc000082600) 55 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2851 +0x9d 56 | runtime.mcall(0x10613d6) 57 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/asm_amd64.s:318 +0x5b 58 | 59 | goroutine 1 [chan receive, 3 minutes]: 60 | main.chanReceiveForever(...) 61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:43 62 | main.main() 63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:32 +0x33d 64 | 65 | goroutine 18 [sleep]: 66 | time.Sleep(0x104c533c00) 67 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 68 | main.sleepLoop(0x104c533c00) 69 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b 70 | created by main.main 71 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:12 +0x56 72 | 73 | goroutine 19 [sleep]: 74 | time.Sleep(0x3b9aca00) 75 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 76 | main.sleepLoop(0x3b9aca00) 77 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b 78 | created by main.main 79 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:13 +0x77 80 | 81 | goroutine 20 [chan receive, 4 minutes]: 82 | main.chanReceiveForever() 83 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:43 +0x4d 84 | created by main.main 85 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:14 +0x8f 86 | 87 | goroutine 22 [sleep]: 88 | time.Sleep(0x3b9aca00) 89 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf 90 | main.sleepLoop(0x3b9aca00) 91 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b 92 | created by main.indirectSleepLoop2 93 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:51 +0x3f 94 | 95 | rax 0x104 96 | rbx 0x2 97 | rcx 0x7ffeefbff248 98 | rdx 0xec00 99 | rdi 0x1172620 100 | rsi 0xec010000ed00 101 | rbp 0x7ffeefbff2e0 102 | rsp 0x7ffeefbff248 103 | r8 0x0 104 | r9 0xa0 105 | r10 0x0 106 | r11 0x202 107 | r12 0x1172620 108 | r13 0x16 109 | r14 0xec010000ed00 110 | r15 0x167fdc0 111 | rip 0x7fff739f9882 112 | rflags 0x203 113 | cs 0x7 114 | fs 0x0 115 | gs 0x0 116 | exit status 2 117 | -------------------------------------------------------------------------------- /examples/stack-unwind-overhead/README.md: -------------------------------------------------------------------------------- 1 | # stack-unwind-overhead 2 | 3 | This directory contains benchmarks to explore which factors impact stack unwinding in Go. It's informed by an analysis of the `gopclntab` unwinding implementation. 4 | 5 | ## Result 1: O(N) Stack Depth 6 | 7 | The benchmark below shows that stack unwinding has O(N) complexity with regard to the number of call frames on the stack: 8 | 9 | ``` 10 | BenchmarkStackDepth/8-12 1968214 612.2 ns/op 11 | BenchmarkStackDepth/16-12 975457 1184 ns/op 12 | BenchmarkStackDepth/32-12 572706 2101 ns/op 13 | BenchmarkStackDepth/64-12 333598 3596 ns/op 14 | BenchmarkStackDepth/128-12 182450 6561 ns/op 15 | BenchmarkStackDepth/256-12 94783 12548 ns/op 16 | BenchmarkStackDepth/512-12 48439 24471 ns/op 17 | BenchmarkStackDepth/1024-12 24884 48310 ns/op 18 | ``` 19 | 20 | Tests were done on my local machine: 21 | 22 | ``` 23 | go test -bench . 24 | goos: darwin 25 | goarch: amd64 26 | pkg: github.com/felixge/go-profiler-notes/examples/stack-unwind-overhead 27 | cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz 28 | ``` 29 | -------------------------------------------------------------------------------- /examples/stack-unwind-overhead/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/stack-unwind-overhead 2 | 3 | go 1.16 4 | -------------------------------------------------------------------------------- /examples/stack-unwind-overhead/main.go: -------------------------------------------------------------------------------- 1 | // Package main is a code generator that creates functions of different size. 2 | package main 3 | 4 | import ( 5 | "fmt" 6 | "os" 7 | ) 8 | 9 | //go:generate go run . 10 | 11 | func main() { 12 | if err := run(); err != nil { 13 | fmt.Fprintln(os.Stderr, err) 14 | os.Exit(1) 15 | } 16 | } 17 | 18 | func run() error { 19 | file, err := os.Create("generated.go") 20 | if err != nil { 21 | return err 22 | } 23 | defer file.Close() 24 | 25 | fmt.Fprint(file, `// Code generated by go generate; DO NOT EDIT. 26 | 27 | package main 28 | 29 | //go:noinline 30 | func stackdepth1(fn func()) { fn() } 31 | `) 32 | 33 | max := 1024 34 | for i := 2; i <= max; i++ { 35 | fmt.Fprintf( 36 | file, 37 | "//go:noinline\nfunc stackdepth%d(fn func()) { stackdepth%d(fn) }\n", 38 | i, 39 | i-1, 40 | ) 41 | } 42 | 43 | fmt.Fprint(file, "\nvar stackdepth = map[int]func(func()) {\n") 44 | for i := 1; i <= max; i++ { 45 | fmt.Fprintf(file, "\t%d: stackdepth%d,\n", i, i) 46 | } 47 | fmt.Fprint(file, "}\n") 48 | 49 | // Disabled: Turned out to be a deadend 50 | //for i := 1; i <= max; i *= 2 { 51 | //fmt.Fprintf(file, "//go:noinline\n") 52 | //fmt.Fprintf(file, "func funcsize%d(v0, n int, fn func()) int {\n", i) 53 | //for n := 1; n <= i; n++ { 54 | //fmt.Fprintf(file, "\tv%d := v%d\n", n, n-1) 55 | //fmt.Fprintf(file, "\tv%d = (^v%d + %d)\n", n, n, n) 56 | //fmt.Fprintf(file, "\tif v%d == 23 {\n", n) 57 | //sum := []string{} 58 | //for j := 0; j <= n; j++ { 59 | //sum = append(sum, fmt.Sprintf("v%d", j)) 60 | //} 61 | //fmt.Fprintf(file, "\t\treturn %s\n", strings.Join(sum, " + ")) 62 | //fmt.Fprintf(file, "\t}\n") 63 | //} 64 | //fmt.Fprintf(file, "\tfn()\n") 65 | //fmt.Fprintf(file, "\treturn -1\n") 66 | //fmt.Fprintf(file, "}\n\n") 67 | //} 68 | 69 | //fmt.Fprint(file, "\nvar funcsize = map[int]func(int, int, func()) int {\n") 70 | //for i := 1; i <= max; i *= 2 { 71 | //fmt.Fprintf(file, "\t%d: funcsize%d,\n", i, i) 72 | //} 73 | //fmt.Fprint(file, "}\n") 74 | 75 | return file.Close() 76 | } 77 | -------------------------------------------------------------------------------- /examples/stack-unwind-overhead/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "testing" 7 | ) 8 | 9 | // BenchmarkStackDepth measures the impact of stack depth on the time it takes 10 | // to create a stack trace. 11 | func BenchmarkStackDepth(b *testing.B) { 12 | for d := 8; d <= 1024; d = d * 2 { 13 | b.Run(fmt.Sprintf("%d", d), func(b *testing.B) { 14 | callers := make([]uintptr, d*2) 15 | atStackDepth(d, func() { 16 | for i := 0; i < b.N; i++ { 17 | if n := runtime.Callers(1, callers); n != d { 18 | b.Fatalf("got=%d want=%d", n, d) 19 | } 20 | } 21 | }) 22 | }) 23 | } 24 | } 25 | 26 | // atStackDepth calls functions that call each other until there are depth-1 27 | // functions on the stack and then calls fn. 28 | //go:noinline 29 | func atStackDepth(depth int, fn func()) { 30 | pcs := make([]uintptr, depth+10) 31 | remaining := depth - runtime.Callers(1, pcs) - 1 32 | if remaining < 1 { 33 | panic("can't simulate desired stack depth: too low") 34 | } else if remaining == 1 { 35 | fn() 36 | } else if f, ok := stackdepth[remaining]; !ok { 37 | panic("can't simulate desired stack depth: no map entry") 38 | } else { 39 | f(fn) 40 | } 41 | } 42 | 43 | // disabled: turned out to be a deadend 44 | //func BenchmarkFunctionSize(b *testing.B) { 45 | //var callers = make([]uintptr, 32) 46 | //for s := 1; s <= 1024; s = s * 2 { 47 | //b.Run(fmt.Sprintf("%d", s), func(b *testing.B) { 48 | //called := false 49 | //funcsize[s](0, 0, func() { 50 | //for i := 0; i < b.N; i++ { 51 | //runtime.Callers(0, callers) 52 | //} 53 | //called = true 54 | //}) 55 | //if !called { 56 | //b.Fatal("not called") 57 | //} 58 | //}) 59 | //} 60 | //} 61 | -------------------------------------------------------------------------------- /examples/stackannotate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func main() { 4 | var n int 5 | for i := 0; i < 10; i++ { 6 | n += foo(1) 7 | } 8 | println(n) 9 | <-chan struct{}(nil) 10 | } 11 | 12 | func foo(a int) int { 13 | return bar(a, 2) 14 | } 15 | 16 | func bar(a int, b int) int { 17 | s := 3 18 | for i := 0; i < 100; i++ { 19 | s += a * b 20 | } 21 | return s 22 | } 23 | -------------------------------------------------------------------------------- /examples/stuck-deadlock/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/stuck-program 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/DataDog/datadog-go v4.4.0+incompatible // indirect 7 | github.com/Microsoft/go-winio v0.4.16 // indirect 8 | github.com/google/uuid v1.2.0 // indirect 9 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 10 | ) 11 | 12 | replace gopkg.in/DataDog/dd-trace-go.v1 => /Users/felix.geisendoerfer/go/src/github.com/DataDog/dd-trace-go 13 | -------------------------------------------------------------------------------- /examples/stuck-deadlock/go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/datadog-go v4.4.0+incompatible h1:R7WqXWP4fIOAqWJtUKmSfuc7eDsBT58k9AY5WSHVosk= 2 | github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 3 | github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= 4 | github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= 5 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 6 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 7 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998 h1:ruQkWz0PK91vTVrWtzAgv3VqTMgIN1FAIvwWr5MY+GQ= 11 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 12 | github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= 13 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 14 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 15 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 16 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 18 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 20 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 21 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 22 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= 24 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 25 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= 26 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 27 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 28 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 h1:EmglUJuykRsTwsQDcKaAo3CmOunWU6Dqk7U2lo7Pjss= 29 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0/go.mod h1:Sp1lku8WJMvNV0kjDI4Ni/T7J/U3BO5ct5kEaoVU8+I= 30 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 31 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 32 | -------------------------------------------------------------------------------- /examples/stuck-deadlock/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "runtime" 8 | "sync" 9 | "time" 10 | 11 | "gopkg.in/DataDog/dd-trace-go.v1/profiler" 12 | ) 13 | 14 | func main() { 15 | if err := run(); err != nil { 16 | fmt.Fprintln(os.Stderr, err) 17 | os.Exit(1) 18 | } 19 | } 20 | func run() error { 21 | if err := os.Setenv("DD_PROFILING_WAIT_PROFILE", "yes"); err != nil { 22 | return err 23 | } else if err := profiler.Start( 24 | profiler.WithService("stuck-deadlock"), 25 | profiler.WithVersion(time.Now().String()), 26 | profiler.WithPeriod(60*time.Second), 27 | profiler.WithProfileTypes( 28 | profiler.GoroutineProfile, 29 | ), 30 | ); err != nil { 31 | return err 32 | } 33 | 34 | var ( 35 | a = &sync.Mutex{} 36 | b = &sync.Mutex{} 37 | ) 38 | go bob(a, b) 39 | go alice(a, b) 40 | 41 | c := make(chan os.Signal, 1) 42 | signal.Notify(c, os.Interrupt) 43 | ticker := time.NewTicker(10 * time.Second) 44 | defer ticker.Stop() 45 | for { 46 | select { 47 | case <-ticker.C: 48 | // force gc on a regular basis to make sure g.waitsince gets populated. 49 | fmt.Printf("gc\n") 50 | runtime.GC() 51 | case sig := <-c: 52 | fmt.Printf("sig: %s\n", sig) 53 | return nil 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | func bob(a, b *sync.Mutex) { 60 | for { 61 | fmt.Println("bob is okay") 62 | a.Lock() 63 | b.Lock() 64 | // do stuff 65 | a.Unlock() 66 | b.Unlock() 67 | } 68 | } 69 | 70 | func alice(a, b *sync.Mutex) { 71 | for { 72 | fmt.Println("alice is okay") 73 | b.Lock() 74 | a.Lock() 75 | // do stuff 76 | b.Unlock() 77 | a.Unlock() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /examples/stuck-producer-consumer/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/felixge/go-profiler-notes/examples/stuck-producer-consumer 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/DataDog/datadog-go v4.4.0+incompatible // indirect 7 | github.com/Microsoft/go-winio v0.4.16 // indirect 8 | github.com/google/uuid v1.2.0 // indirect 9 | gopkg.in/DataDog/dd-trace-go.v1 v1.29.0-alpha.1.0.20210216140755-2d091eca40bb 10 | ) 11 | 12 | replace gopkg.in/DataDog/dd-trace-go.v1 => /Users/felix.geisendoerfer/go/src/github.com/DataDog/dd-trace-go 13 | -------------------------------------------------------------------------------- /examples/stuck-producer-consumer/go.sum: -------------------------------------------------------------------------------- 1 | github.com/DataDog/datadog-go v4.4.0+incompatible h1:R7WqXWP4fIOAqWJtUKmSfuc7eDsBT58k9AY5WSHVosk= 2 | github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 3 | github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= 4 | github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= 5 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 6 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 7 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998 h1:ruQkWz0PK91vTVrWtzAgv3VqTMgIN1FAIvwWr5MY+GQ= 11 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 12 | github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= 13 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 14 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 15 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 16 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 18 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 20 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 21 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 22 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= 24 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 25 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= 26 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 27 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e h1:9vRrk9YW2BTzLP0VCB9ZDjU4cPqkg+IDWL7XgxA1yxQ= 28 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 29 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 h1:EmglUJuykRsTwsQDcKaAo3CmOunWU6Dqk7U2lo7Pjss= 30 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0/go.mod h1:Sp1lku8WJMvNV0kjDI4Ni/T7J/U3BO5ct5kEaoVU8+I= 31 | gopkg.in/DataDog/dd-trace-go.v1 v1.29.0-alpha.1.0.20210216140755-2d091eca40bb h1:BQMy4tHVzHLWsGeQMmYrgWbHkVPp0ibHM/IDREdchyk= 32 | gopkg.in/DataDog/dd-trace-go.v1 v1.29.0-alpha.1.0.20210216140755-2d091eca40bb/go.mod h1:FLwUDeuH0z5hkvgvd04/M3MHQN4AF5pQDnedeWRWvok= 33 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 34 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 35 | -------------------------------------------------------------------------------- /examples/stuck-producer-consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "os/signal" 8 | "runtime" 9 | "time" 10 | 11 | "gopkg.in/DataDog/dd-trace-go.v1/profiler" 12 | ) 13 | 14 | func main() { 15 | if err := run(); err != nil { 16 | fmt.Fprintln(os.Stderr, err) 17 | os.Exit(1) 18 | } 19 | } 20 | 21 | func run() error { 22 | if err := os.Setenv("DD_PROFILING_WAIT_PROFILE", "yes"); err != nil { 23 | return err 24 | } else if err := profiler.Start( 25 | profiler.WithService("stuck-producer-consumer"), 26 | profiler.WithVersion(time.Now().String()), 27 | profiler.WithPeriod(60*time.Second), 28 | profiler.WithProfileTypes( 29 | profiler.GoroutineProfile, 30 | ), 31 | ); err != nil { 32 | return err 33 | } 34 | 35 | rand.Seed(time.Now().UnixNano()) 36 | 37 | workCh := make(chan struct{}) 38 | go consumer(workCh) 39 | go producer(workCh) 40 | 41 | c := make(chan os.Signal, 1) 42 | signal.Notify(c, os.Interrupt) 43 | ticker := time.NewTicker(10 * time.Second) 44 | defer ticker.Stop() 45 | for { 46 | select { 47 | case <-ticker.C: 48 | // force gc on a regular basis to make sure g.waitsince gets populated. 49 | fmt.Printf("gc\n") 50 | runtime.GC() 51 | case sig := <-c: 52 | fmt.Printf("sig: %s\n", sig) 53 | return nil 54 | } 55 | } 56 | } 57 | 58 | func producer(workCh chan<- struct{}) { 59 | for { 60 | select { 61 | case workCh <- struct{}{}: 62 | if rand.Int63n(10) == 0 { 63 | takeNap() 64 | } 65 | default: 66 | } 67 | } 68 | } 69 | 70 | func consumer(workCh <-chan struct{}) { 71 | for { 72 | <-workCh 73 | if rand.Int63n(10) == 0 { 74 | takeNap() 75 | } 76 | } 77 | } 78 | 79 | func takeNap() { 80 | fmt.Printf("taking a nap\n") 81 | var forever chan struct{} 82 | <-forever 83 | } 84 | -------------------------------------------------------------------------------- /examples/threadcreate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "runtime" 7 | "runtime/pprof" 8 | "syscall" 9 | "time" 10 | ) 11 | 12 | func main() { 13 | if err := run(); err != nil { 14 | fmt.Fprintln(os.Stderr, err) 15 | os.Exit(1) 16 | } 17 | } 18 | 19 | func run() error { 20 | runtime.GOMAXPROCS(1) 21 | n := 16 22 | ch := make(chan int) 23 | fmt.Printf("start work\n") 24 | for i := 0; i < n; i++ { 25 | go func() { 26 | buf := make([]byte, 1024) 27 | fmt.Printf("syscall\n") 28 | n, err := syscall.Read(1, buf) 29 | if err != nil { 30 | panic(err) 31 | } 32 | fmt.Printf("%d\n", n) 33 | }() 34 | go dowork(ch) 35 | } 36 | runtime.GOMAXPROCS(8) 37 | time.Sleep(50 * time.Millisecond) 38 | runtime.GC() 39 | if err := writeProfile("threadcreate"); err != nil { 40 | return err 41 | } 42 | fmt.Printf("profile\n") 43 | for i := 0; i < n; i++ { 44 | fmt.Printf("%d\n", <-ch) 45 | } 46 | fmt.Printf("done\n") 47 | 48 | return nil 49 | } 50 | 51 | func dowork(done chan int) { 52 | s := 0 53 | for i := 0; i < 1000000000; i++ { 54 | if i%2 == 0 { 55 | s++ 56 | } 57 | } 58 | done <- s 59 | } 60 | 61 | func writeProfile(name string) error { 62 | f, err := os.Create(name + ".pb.gz") 63 | if err != nil { 64 | return err 65 | } 66 | defer f.Close() 67 | 68 | if err := pprof.Lookup(name).WriteTo(f, 0); err != nil { 69 | return err 70 | } 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /flame-abc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/flame-abc.png -------------------------------------------------------------------------------- /flame-abc.txt: -------------------------------------------------------------------------------- 1 | A 5 2 | A;C 3 3 | B 4 4 | -------------------------------------------------------------------------------- /go-binary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/go-binary.png -------------------------------------------------------------------------------- /goroutine-matrix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/goroutine-matrix.png -------------------------------------------------------------------------------- /goroutine-stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/goroutine-stack.png -------------------------------------------------------------------------------- /guide/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: scheduler.gif 2 | scheduler.gif: 3 | convert -delay 100 scheduler.*.png scheduler.gif 4 | 5 | .PHONY: stack.gif 6 | stack.gif: 7 | convert -delay 200 stack*.png stack.gif 8 | 9 | .PHONY: heap-gc.gif 10 | heap-gc.gif: 11 | convert -delay 200 heap-gc.*.png heap-gc.gif 12 | -------------------------------------------------------------------------------- /guide/cpu-max-stack-depth.go: -------------------------------------------------------------------------------- 1 | //go:build ignore 2 | // +build ignore 3 | 4 | package main 5 | 6 | import ( 7 | "os" 8 | "runtime/pprof" 9 | ) 10 | 11 | func main() { 12 | pprof.StartCPUProfile(os.Stdout) 13 | defer pprof.StopCPUProfile() 14 | 15 | belowLimit() 16 | aboveLimit() 17 | } 18 | 19 | func belowLimit() { 20 | atDepth(32, cpuHog) 21 | } 22 | 23 | func aboveLimit() { 24 | atDepth(64, cpuHog) 25 | } 26 | 27 | func cpuHog() { 28 | for i := 0; i < 5000*1000*1000; i++ { 29 | } 30 | } 31 | 32 | func atDepth(n int, fn func()) { 33 | if n > 0 { 34 | atDepth(n-1, fn) 35 | } else { 36 | fn() 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /guide/cpu-max-stack-depth.pprof: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-max-stack-depth.pprof -------------------------------------------------------------------------------- /guide/cpu-profiler-labels.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "context" 7 | "os" 8 | "runtime/pprof" 9 | "time" 10 | ) 11 | 12 | func main() { 13 | pprof.StartCPUProfile(os.Stdout) 14 | defer pprof.StopCPUProfile() 15 | 16 | go work(context.Background(), "alice") 17 | go work(context.Background(), "bob") 18 | 19 | time.Sleep(50 * time.Millisecond) 20 | } 21 | 22 | func work(ctx context.Context, user string) { 23 | labels := pprof.Labels("user", user) 24 | pprof.Do(ctx, labels, func(_ context.Context) { 25 | go backgroundWork() 26 | directWork() 27 | }) 28 | } 29 | 30 | func directWork() { 31 | for { 32 | } 33 | } 34 | 35 | func backgroundWork() { 36 | for { 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /guide/cpu-profiler-labels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-profiler-labels.png -------------------------------------------------------------------------------- /guide/cpu-profiler-labels.pprof: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-profiler-labels.pprof -------------------------------------------------------------------------------- /guide/cpu-rate.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "os" 7 | "runtime" 8 | "runtime/pprof" 9 | "time" 10 | ) 11 | 12 | func main() { 13 | runtime.SetCPUProfileRate(800) 14 | 15 | pprof.StartCPUProfile(os.Stdout) 16 | defer pprof.StopCPUProfile() 17 | 18 | go cpuHog() 19 | time.Sleep(time.Second) 20 | } 21 | 22 | func cpuHog() { 23 | for { 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /guide/cpu-rate.pprof: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-rate.pprof -------------------------------------------------------------------------------- /guide/cpu-utilization.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "os" 7 | "runtime/pprof" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | file, _ := os.Create("./cpu-utilization.pprof") 13 | pprof.StartCPUProfile(file) 14 | defer pprof.StopCPUProfile() 15 | 16 | go cpuHog() 17 | go cpuHog() 18 | 19 | time.Sleep(time.Second) 20 | } 21 | 22 | func cpuHog() { 23 | for { 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /guide/cpu-utilization.pprof: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-utilization.pprof -------------------------------------------------------------------------------- /guide/heap-gc.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-gc.1.png -------------------------------------------------------------------------------- /guide/heap-gc.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-gc.2.png -------------------------------------------------------------------------------- /guide/heap-gc.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-gc.gif -------------------------------------------------------------------------------- /guide/heap-simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-simple.png -------------------------------------------------------------------------------- /guide/heap.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | func main() { 10 | fmt.Println(*add(23, 42)) 11 | } 12 | 13 | func add(a, b int) *int { 14 | sum := a + b 15 | return &sum 16 | } 17 | -------------------------------------------------------------------------------- /guide/heap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap.png -------------------------------------------------------------------------------- /guide/http-get.go: -------------------------------------------------------------------------------- 1 | //go:build ignore 2 | 3 | package main 4 | 5 | import ( 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "os" 10 | "runtime/trace" 11 | "time" 12 | ) 13 | 14 | func main() { 15 | traceF, _ := os.Create("trace.out") 16 | trace.Start(traceF) 17 | defer trace.Stop() 18 | 19 | start := time.Now() 20 | res, err := http.Get("https://example.org/") 21 | if err != nil { 22 | panic(err) 23 | } 24 | fmt.Printf("%d\n", res.StatusCode) 25 | log.Printf("main() took: %s\n", time.Since(start)) 26 | } 27 | -------------------------------------------------------------------------------- /guide/memory-profiler.go: -------------------------------------------------------------------------------- 1 | //go:build ignore 2 | // +build ignore 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "os" 9 | "runtime" 10 | "runtime/pprof" 11 | "time" 12 | ) 13 | 14 | func main() { 15 | file, _ := os.Create("./memory-profiler.pprof") 16 | defer pprof.Lookup("allocs").WriteTo(file, 0) 17 | defer runtime.GC() 18 | 19 | go allocSmall() 20 | go allocBig() 21 | 22 | time.Sleep(1 * time.Second) 23 | 24 | var m runtime.MemStats 25 | runtime.ReadMemStats(&m) 26 | fmt.Printf("%#v\n", m) 27 | } 28 | 29 | //go:noinline 30 | func allocSmall() { 31 | for i := 0; ; i++ { 32 | _ = alloc(32) 33 | } 34 | } 35 | 36 | //go:noinline 37 | func allocBig() { 38 | for i := 0; ; i++ { 39 | _ = alloc(256) 40 | } 41 | } 42 | 43 | //go:noinline 44 | func alloc(size int) []byte { 45 | return make([]byte, size) 46 | } 47 | -------------------------------------------------------------------------------- /guide/memory-profiler.pprof: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/memory-profiler.pprof -------------------------------------------------------------------------------- /guide/profiler-venn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/profiler-venn.png -------------------------------------------------------------------------------- /guide/scheduler-complete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler-complete.png -------------------------------------------------------------------------------- /guide/scheduler.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.1.png -------------------------------------------------------------------------------- /guide/scheduler.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.2.png -------------------------------------------------------------------------------- /guide/scheduler.3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.3.png -------------------------------------------------------------------------------- /guide/scheduler.4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.4.png -------------------------------------------------------------------------------- /guide/scheduler.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.gif -------------------------------------------------------------------------------- /guide/stack.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack.gif -------------------------------------------------------------------------------- /guide/stack.go: -------------------------------------------------------------------------------- 1 | // +build ignore 2 | 3 | package main 4 | 5 | import "fmt" 6 | 7 | func main() { 8 | sum := 0 9 | sum = add(23, 42) 10 | fmt.Println(sum) 11 | } 12 | 13 | func add(a, b int) int { 14 | return a + b 15 | } 16 | -------------------------------------------------------------------------------- /guide/stack1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack1.png -------------------------------------------------------------------------------- /guide/stack2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack2.png -------------------------------------------------------------------------------- /guide/stack3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack3.png -------------------------------------------------------------------------------- /guide/time.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/time.1.png -------------------------------------------------------------------------------- /guide/time.2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/time.2.png -------------------------------------------------------------------------------- /guide/time.3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/time.3.png -------------------------------------------------------------------------------- /guide/timeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/timeline.png -------------------------------------------------------------------------------- /guide/trace.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/trace.out -------------------------------------------------------------------------------- /heap.md: -------------------------------------------------------------------------------- 1 | # Go's Heap Profiler 2 | 3 | Totally a work in progress ... !!! Please come back later : ). 4 | 5 | https://twitter.com/felixge/status/1355846360562589696 6 | 7 | 8 | 9 | ## How does it work? 10 | 11 | Look at the go source code to understand how the data is captured, what role runtime.MemProfileRate plays, etc. 12 | 13 | ## How does it fail? 14 | 15 | Think through sampling rate issues, alloc size classes, etc. 16 | 17 | ## Performance Overhead 18 | 19 | Talk about performance overhead ... 20 | 21 | ## Data Format 22 | 23 | Figure out how the data ends up in the pprof file. 24 | 25 | ## GC Control 26 | 27 | ``` 28 | # turn of gc 29 | GOGC=off go run 30 | # print gc events to stdout 31 | GODEBUG=gctrace=1 go run 32 | ``` 33 | 34 | ## Questions 35 | 36 | - What are the [docs](https://golang.org/pkg/runtime/pprof/#Profile) talking about here? How do I actually use this? 37 | 38 | > Pprof's -inuse_space, -inuse_objects, -alloc_space, and -alloc_objects flags select which to display, defaulting to -inuse_space (live objects, scaled by size). 39 | 40 | A: Those flags are deprecated. Easiest way to select this stuff is via the pprof web ui's sample drop down. 41 | 42 | - The [docs](https://golang.org/pkg/runtime/pprof/#Profile) say I should get some kind of data, even if there is no GC. I can reproduce that, but the data seems to not change? 43 | 44 | > If there has been no garbage collection at all, the heap profile reports all known allocations. This exception helps mainly in programs running without garbage collection enabled, usually for debugging purposes. 45 | 46 | ## Disclaimers 47 | 48 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ). 49 | 50 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! -------------------------------------------------------------------------------- /hello.trace: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/hello.trace -------------------------------------------------------------------------------- /mutex.md: -------------------------------------------------------------------------------- 1 | 🚧 This note is still work in progress, please come back later! 🚧 2 | 3 | # Mutex Profiling in Go 4 | 5 | ## History 6 | 7 | Mutex profiling was [implemented](https://go-review.googlesource.com/c/go/+/29650/) by [Peter J. Weinberger](https://en.wikipedia.org/wiki/Peter_J._Weinberger) and first appeared in the [go1.8](https://golang.org/doc/go1.8#mutex_prof) release (2017-02-16). 8 | 9 | ## Links 10 | 11 | - https://rakyll.org/mutexprofile/ 12 | - https://talks.golang.org/2017/state-of-go.slide#23 13 | 14 | ## Disclaimers 15 | 16 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ). 17 | 18 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome! -------------------------------------------------------------------------------- /profile.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/profile.png -------------------------------------------------------------------------------- /profile.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Profile is a common stacktrace profile format. 16 | // 17 | // Measurements represented with this format should follow the 18 | // following conventions: 19 | // 20 | // - Consumers should treat unset optional fields as if they had been 21 | // set with their default value. 22 | // 23 | // - When possible, measurements should be stored in "unsampled" form 24 | // that is most useful to humans. There should be enough 25 | // information present to determine the original sampled values. 26 | // 27 | // - On-disk, the serialized proto must be gzip-compressed. 28 | // 29 | // - The profile is represented as a set of samples, where each sample 30 | // references a sequence of locations, and where each location belongs 31 | // to a mapping. 32 | // - There is a N->1 relationship from sample.location_id entries to 33 | // locations. For every sample.location_id entry there must be a 34 | // unique Location with that id. 35 | // - There is an optional N->1 relationship from locations to 36 | // mappings. For every nonzero Location.mapping_id there must be a 37 | // unique Mapping with that id. 38 | 39 | syntax = "proto3"; 40 | 41 | package perftools.profiles; 42 | 43 | option java_package = "com.google.perftools.profiles"; 44 | option java_outer_classname = "ProfileProto"; 45 | 46 | message Profile { 47 | // A description of the samples associated with each Sample.value. 48 | // For a cpu profile this might be: 49 | // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]] 50 | // For a heap profile, this might be: 51 | // [["allocations","count"], ["space","bytes"]], 52 | // If one of the values represents the number of events represented 53 | // by the sample, by convention it should be at index 0 and use 54 | // sample_type.unit == "count". 55 | repeated ValueType sample_type = 1; 56 | // The set of samples recorded in this profile. 57 | repeated Sample sample = 2; 58 | // Mapping from address ranges to the image/binary/library mapped 59 | // into that address range. mapping[0] will be the main binary. 60 | repeated Mapping mapping = 3; 61 | // Useful program location 62 | repeated Location location = 4; 63 | // Functions referenced by locations 64 | repeated Function function = 5; 65 | // A common table for strings referenced by various messages. 66 | // string_table[0] must always be "". 67 | repeated string string_table = 6; 68 | // frames with Function.function_name fully matching the following 69 | // regexp will be dropped from the samples, along with their successors. 70 | int64 drop_frames = 7; // Index into string table. 71 | // frames with Function.function_name fully matching the following 72 | // regexp will be kept, even if it matches drop_functions. 73 | int64 keep_frames = 8; // Index into string table. 74 | 75 | // The following fields are informational, do not affect 76 | // interpretation of results. 77 | 78 | // Time of collection (UTC) represented as nanoseconds past the epoch. 79 | int64 time_nanos = 9; 80 | // Duration of the profile, if a duration makes sense. 81 | int64 duration_nanos = 10; 82 | // The kind of events between sampled ocurrences. 83 | // e.g [ "cpu","cycles" ] or [ "heap","bytes" ] 84 | ValueType period_type = 11; 85 | // The number of events between sampled occurrences. 86 | int64 period = 12; 87 | // Freeform text associated to the profile. 88 | repeated int64 comment = 13; // Indices into string table. 89 | // Index into the string table of the type of the preferred sample 90 | // value. If unset, clients should default to the last sample value. 91 | int64 default_sample_type = 14; 92 | } 93 | 94 | // ValueType describes the semantics and measurement units of a value. 95 | message ValueType { 96 | int64 type = 1; // Index into string table. 97 | int64 unit = 2; // Index into string table. 98 | } 99 | 100 | // Each Sample records values encountered in some program 101 | // context. The program context is typically a stack trace, perhaps 102 | // augmented with auxiliary information like the thread-id, some 103 | // indicator of a higher level request being handled etc. 104 | message Sample { 105 | // The ids recorded here correspond to a Profile.location.id. 106 | // The leaf is at location_id[0]. 107 | repeated uint64 location_id = 1; 108 | // The type and unit of each value is defined by the corresponding 109 | // entry in Profile.sample_type. All samples must have the same 110 | // number of values, the same as the length of Profile.sample_type. 111 | // When aggregating multiple samples into a single sample, the 112 | // result has a list of values that is the elemntwise sum of the 113 | // lists of the originals. 114 | repeated int64 value = 2; 115 | // label includes additional context for this sample. It can include 116 | // things like a thread id, allocation size, etc 117 | repeated Label label = 3; 118 | } 119 | 120 | message Label { 121 | int64 key = 1; // Index into string table 122 | 123 | // At most one of the following must be present 124 | int64 str = 2; // Index into string table 125 | int64 num = 3; 126 | 127 | // Should only be present when num is present. 128 | // Specifies the units of num. 129 | // Use arbitrary string (for example, "requests") as a custom count unit. 130 | // If no unit is specified, consumer may apply heuristic to deduce the unit. 131 | // Consumers may also interpret units like "bytes" and "kilobytes" as memory 132 | // units and units like "seconds" and "nanoseconds" as time units, 133 | // and apply appropriate unit conversions to these. 134 | int64 num_unit = 4; // Index into string table 135 | } 136 | 137 | message Mapping { 138 | // Unique nonzero id for the mapping. 139 | uint64 id = 1; 140 | // Address at which the binary (or DLL) is loaded into memory. 141 | uint64 memory_start = 2; 142 | // The limit of the address range occupied by this mapping. 143 | uint64 memory_limit = 3; 144 | // Offset in the binary that corresponds to the first mapped address. 145 | uint64 file_offset = 4; 146 | // The object this entry is loaded from. This can be a filename on 147 | // disk for the main binary and shared libraries, or virtual 148 | // abstractions like "[vdso]". 149 | int64 filename = 5; // Index into string table 150 | // A string that uniquely identifies a particular program version 151 | // with high probability. E.g., for binaries generated by GNU tools, 152 | // it could be the contents of the .note.gnu.build-id field. 153 | int64 build_id = 6; // Index into string table 154 | 155 | // The following fields indicate the resolution of symbolic info. 156 | bool has_functions = 7; 157 | bool has_filenames = 8; 158 | bool has_line_numbers = 9; 159 | bool has_inline_frames = 10; 160 | } 161 | 162 | // Describes function and line table debug information. 163 | message Location { 164 | // Unique nonzero id for the location. A profile could use 165 | // instruction addresses or any integer sequence as ids. 166 | uint64 id = 1; 167 | // The id of the corresponding profile.Mapping for this location. 168 | // It can be unset if the mapping is unknown or not applicable for 169 | // this profile type. 170 | uint64 mapping_id = 2; 171 | // The instruction address for this location, if available. It 172 | // should be within [Mapping.memory_start...Mapping.memory_limit] 173 | // for the corresponding mapping. A non-leaf address may be in the 174 | // middle of a call instruction. It is up to display tools to find 175 | // the beginning of the instruction if necessary. 176 | uint64 address = 3; 177 | // Multiple line indicates this location has inlined functions, 178 | // where the last entry represents the caller into which the 179 | // preceding entries were inlined. 180 | // 181 | // E.g., if memcpy() is inlined into printf: 182 | // line[0].function_name == "memcpy" 183 | // line[1].function_name == "printf" 184 | repeated Line line = 4; 185 | // Provides an indication that multiple symbols map to this location's 186 | // address, for example due to identical code folding by the linker. In that 187 | // case the line information above represents one of the multiple 188 | // symbols. This field must be recomputed when the symbolization state of the 189 | // profile changes. 190 | bool is_folded = 5; 191 | } 192 | 193 | message Line { 194 | // The id of the corresponding profile.Function for this line. 195 | uint64 function_id = 1; 196 | // Line number in source code. 197 | int64 line = 2; 198 | } 199 | 200 | message Function { 201 | // Unique nonzero id for the function. 202 | uint64 id = 1; 203 | // Name of the function, in human-readable form if available. 204 | int64 name = 2; // Index into string table 205 | // Name of the function, as identified by the system. 206 | // For instance, it can be a C++ mangled name. 207 | int64 system_name = 3; // Index into string table 208 | // Source file containing the function. 209 | int64 filename = 4; // Index into string table 210 | // Line number in source file. 211 | int64 start_line = 5; 212 | } -------------------------------------------------------------------------------- /scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/scheduler.png -------------------------------------------------------------------------------- /sim/block_sampling_biased.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/sim/block_sampling_biased.png -------------------------------------------------------------------------------- /sim/block_sampling_debiased.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/sim/block_sampling_debiased.png -------------------------------------------------------------------------------- /sim/block_sampling_hist.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/sim/block_sampling_hist.png --------------------------------------------------------------------------------