├── .github
└── workflows
│ └── ci.yml
├── .gitignore
├── LICENSE.txt
├── README.md
├── bench
├── README.md
├── analysis.ipynb
├── block_bufchan_linux_x86_64.csv
├── block_darwin_x86_64.csv
├── block_linux_x86_64.csv
├── block_linux_x86_64.png
├── csv.go
├── main.go
├── run.sh
├── workload_chan.go
└── workload_mutex.go
├── block-bias.md
├── block.md
├── cpu.md
├── datadog.excalidraw
├── datadog.md
├── datadog.png
├── delve
└── stackannotate.star
├── examples
├── block-bias
│ ├── block.pb.gz
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── block-net
│ ├── README.md
│ ├── block-net.png
│ ├── block.pb.gz
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── block-sample
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── block-vs-mutex
│ └── main.go
├── block
│ ├── main
│ └── main.go
├── callers-func-length
│ ├── README.md
│ ├── go.mod
│ ├── main.go
│ └── main_test.go
├── cpu
│ ├── benchmark.cpu.pb.gz
│ ├── go.mod
│ ├── go.sum
│ ├── main.go
│ ├── main_test.go
│ ├── pprof.samples.cpu.001.pb.gz
│ ├── pprof.samples.cpu.001.pprof.txt
│ └── pprof.samples.cpu.001.protoc.txt
├── datadog
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── delve-trace
│ ├── go.mod
│ └── main.go
├── frame-order
│ ├── frames.txt
│ ├── go.mod
│ └── main.go
├── goroutine
│ ├── 1.net.http.pprof.goroutine.debug0.pb.gz
│ ├── 1.net.http.pprof.goroutine.debug1.txt
│ ├── 1.net.http.pprof.goroutine.debug2.txt
│ ├── 1.pprof.lookup.goroutine.debug0.pb.gz
│ ├── 1.pprof.lookup.goroutine.debug1.txt
│ ├── 1.pprof.lookup.goroutine.debug2.txt
│ ├── 1.runtime.goroutineprofile.json
│ ├── 1.runtime.stack.txt
│ ├── 2.net.http.pprof.goroutine.debug0.pb.gz
│ ├── 2.net.http.pprof.goroutine.debug1.txt
│ ├── 2.net.http.pprof.goroutine.debug2.txt
│ ├── 2.pprof.lookup.goroutine.debug0.pb.gz
│ ├── 2.pprof.lookup.goroutine.debug1.txt
│ ├── 2.pprof.lookup.goroutine.debug2.txt
│ ├── 2.runtime.goroutineprofile.json
│ ├── 2.runtime.stack.txt
│ ├── go.mod
│ ├── main.go
│ └── main_test.go
├── guide
│ ├── main
│ ├── main.go
│ ├── main2.go
│ ├── main3.go
│ └── trace.bin
├── memory
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── pclnttab
│ ├── darwin.go
│ ├── go.mod
│ ├── linux.go
│ └── main.go
├── presentation
│ ├── main.go
│ ├── main.simple.txt
│ ├── main.txt
│ └── stack.txt
├── pseudo-code
│ ├── pseudo-block.txt
│ └── pseudo-heap.txt
├── runtime-stack
│ ├── go.mod
│ ├── main.go
│ ├── main_test.go
│ └── runtime-stack.txt
├── stack-unwind-overhead
│ ├── README.md
│ ├── generated.go
│ ├── go.mod
│ ├── main.go
│ └── main_test.go
├── stackannotate
│ └── main.go
├── stuck-deadlock
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── stuck-producer-consumer
│ ├── go.mod
│ ├── go.sum
│ └── main.go
└── threadcreate
│ └── main.go
├── flame-abc.png
├── flame-abc.txt
├── go-binary.png
├── goroutine-matrix.png
├── goroutine-stack.png
├── goroutine.md
├── guide.excalidraw
├── guide
├── Makefile
├── README.md
├── cpu-max-stack-depth.go
├── cpu-max-stack-depth.pprof
├── cpu-profiler-labels.go
├── cpu-profiler-labels.png
├── cpu-profiler-labels.pprof
├── cpu-rate.go
├── cpu-rate.pprof
├── cpu-utilization.go
├── cpu-utilization.pprof
├── guide.excalidraw
├── heap-gc.1.png
├── heap-gc.2.png
├── heap-gc.gif
├── heap-simple.png
├── heap.go
├── heap.png
├── http-get.go
├── memory-profiler.go
├── memory-profiler.pprof
├── profiler-venn.png
├── scheduler-complete.png
├── scheduler.1.png
├── scheduler.2.png
├── scheduler.3.png
├── scheduler.4.png
├── scheduler.gif
├── stack.gif
├── stack.go
├── stack1.png
├── stack2.png
├── stack3.png
├── time.1.png
├── time.2.png
├── time.3.png
├── timeline.png
└── trace.out
├── heap.md
├── hello.trace
├── mutex.md
├── pprof.md
├── profile.png
├── profile.proto
├── profilers.excalidraw
├── scheduler.png
├── sim
├── block_sampling.ipynb
├── block_sampling_biased.png
├── block_sampling_debiased.png
└── block_sampling_hist.png
├── stack-traces.excalidraw
└── stack-traces.md
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: CI
2 |
3 | on:
4 | push:
5 | branches: [ "main" ]
6 | pull_request:
7 | branches: [ "main" ]
8 |
9 | jobs:
10 | check-links:
11 | runs-on: ubuntu-latest
12 | steps:
13 | - name: Checkout
14 | uses: actions/checkout@main
15 | - name: Markup Link Checker (mlc)
16 | uses: becheran/mlc@v0.15.4
17 | with:
18 | args: --no-web-links
19 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | venv
3 | *.pb.gz
4 | .ipynb_checkpoints
5 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # go-profiler-notes
2 |
3 | Hey there 👋🏻, I'm [felixge](https://github.com/felixge) and I've just started a new job at [Datadog](https://www.datadoghq.com/) to work on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go.
4 |
5 | I found that Go has a lot of profilers and there are many tools for looking at the data, but that there is very little information on what any of it means. So in order to make sure that I know what I'm talking about, I've started to research the existing profilers and how they work. This repository is my attempt to summarize my findings in the hope that it might be useful to others.
6 |
7 | - [The Busy Developers's Guide to Go Profiling, Tracing and Observability](./guide/README.md)
8 | - [pprof tool & format](./pprof.md): Describes the pprof tool and the binary data format for storing profiles.
9 | - [Stack Traces](./stack-traces.md): An in-depth look at stack traces in Go which are a load bearing subsystem for profiling.
10 | - [Goroutine Profiling](./goroutine.md): Allows you to get a list of all active goroutines and what they're currently doing.
11 | - [Block Profiling](./block.md): Understand how much time your code spends waiting on channels and locks.
12 | - CPU Profiling (🚧 coming soon!)
13 | - Heap Profiling (🚧 coming soon!)
14 | - Mutex Profiling (🚧 coming soon!)
15 | - Wallclock Profiling (🚧 coming soon!)
16 |
17 | ## External Links
18 |
19 | - Go Docs
20 | - [Diagnostics](https://golang.org/doc/diagnostics.html): Has a very good overview over the available profiling and tracing facilities but doesn't go into a lot of depth.
21 | - [runtime/pprof](https://golang.org/pkg/runtime/pprof/#Profile): Lists the available profiles and has a little more explanation about what kind of data they produce.
22 | - [runtime](https://golang.org/pkg/runtime/): Has documentation on the various control knobs and pprof facilities, e.g. `MemProfileRate`.
23 | - [net/http/pprof](https://golang.org/src/net/http/pprof/pprof.go): Not a lot of docs, but diving into the code from there shows how the various profilers can be started/stopped on demand.
24 | - JBD
25 | - [Profiler labels in Go](https://rakyll.org/profiler-labels/): An introduction to using pprof labels and how they allow you to add additional context to your profiles.
26 | - [Custom pprof profiles](https://rakyll.org/custom-profiles/): Example for using custom profiles, shows tracking open/close events of a blob store and how to figure out how many blobs are open at a given time.
27 | - [Mutex profile](https://rakyll.org/mutexprofile/): Brief intro to the mutex profile.
28 | - [Using Instruments to profile Go programs](https://rakyll.org/instruments/): How to use the macOS Instruments app (I think it's built on dtrace) to profile Go programs. Not clear what the benfits are, if any.
29 | - [Profiling Go programs with pprof](https://jvns.ca/blog/2017/09/24/profiling-go-with-pprof/) by Julia Evans: A nice tour with a focus on heap profiling and the pprof output format.
30 |
31 | Got great links to recommend? Open an issue or PR, I'd happy to add your suggestions : ).
32 |
33 | ## Stargazers over time
34 |
35 | [](https://starchart.cc/DataDog/go-profiler-notes)
36 |
37 | ## License
38 |
39 | The markdown files in this repository are licensed under the [CC BY-SA 4.0 license](https://creativecommons.org/licenses/by-sa/4.0/).
40 |
41 | ## Disclaimers
42 |
43 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ).
44 |
45 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
46 |
--------------------------------------------------------------------------------
/bench/README.md:
--------------------------------------------------------------------------------
1 | # Go Profiler Overheads
2 |
3 | This page is documenting the benchmark methodology used to analyze the performance overhead of the various go profilers. The results are discussed in the documents for each individual profiler.
4 |
5 | Benchmarking is done by invoking the Go program included in this directory. You can look at [run.sh](./run.sh) to see the current arguments that are being used, but here is an example for block profiling with two workloads and various profiling rates:
6 |
7 | ```
8 | go run . \
9 | -workloads mutex,chan \
10 | -ops 100000 \
11 | -blockprofilerates 0,1,10,100,1000,10000,100000,1000000 \
12 | -runs 20 \
13 | -depths 16 \
14 | > "result.csv"
15 | ```
16 |
17 | The benchmark works by spawning a new child process for the given number of `-runs` and every unique combination of parameters. The child reports the results to the parent process which then combines all the results in a CSV file. The hope is that using a new child process for every config/run eliminates scheduler, GC and other runtime state building up as a source of errors.
18 |
19 | Workloads are defined in the [workloads_chan.go](./workload_chan.go) and [workloads_mutex.go](./workload_mutex.go) files. For now the workloads are designed to be **pathological**, i.e. they try to show the worst performance impact the profiler might have on applications that are not doing anything useful other than stressing the profiler. The numbers are not intended to scare you away from profiling in production, but to guide you towards universally **safe profiling rates** as a starting point.
20 |
21 | The CSV files are visualized using the [analysis.ipynb](./analysis.ipynb) notebook that's included in this directory.
22 |
23 | For now the data is only collected from my local MacBook Pro machine (using docker for mac), but more realistic environments will be included in the future. But it's probably a good setup for finding pathological scenarios : ).
24 |
25 | ## Disclaimers
26 |
27 | I work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go (you should check it out) and they generously allowed me to do all this research and publish it.
28 |
29 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
--------------------------------------------------------------------------------
/bench/block_bufchan_linux_x86_64.csv:
--------------------------------------------------------------------------------
1 | workload,ops,goroutines,depth,bufsize,blockprofilerate,run,ms
2 | chan,100000,12,16,128,0,1,26.111602
3 | chan,100000,12,16,128,0,2,24.330099
4 | chan,100000,12,16,128,0,3,25.564551
5 | chan,100000,12,16,128,0,4,21.197282
6 | chan,100000,12,16,128,0,5,25.208467
7 | chan,100000,12,16,128,0,6,24.942660
8 | chan,100000,12,16,128,0,7,25.745805
9 | chan,100000,12,16,128,0,8,25.952320
10 | chan,100000,12,16,128,0,9,25.502639
11 | chan,100000,12,16,128,0,10,25.505424
12 | chan,100000,12,16,128,0,11,19.021869
13 | chan,100000,12,16,128,0,12,24.093446
14 | chan,100000,12,16,128,0,13,25.470600
15 | chan,100000,12,16,128,0,14,24.750474
16 | chan,100000,12,16,128,0,15,25.318373
17 | chan,100000,12,16,128,0,16,24.801701
18 | chan,100000,12,16,128,0,17,25.410303
19 | chan,100000,12,16,128,0,18,23.951828
20 | chan,100000,12,16,128,0,19,25.501909
21 | chan,100000,12,16,128,0,20,25.479501
22 | chan,100000,12,16,128,1,1,28.818918
23 | chan,100000,12,16,128,1,2,27.955490
24 | chan,100000,12,16,128,1,3,27.657836
25 | chan,100000,12,16,128,1,4,28.679312
26 | chan,100000,12,16,128,1,5,28.240668
27 | chan,100000,12,16,128,1,6,27.620967
28 | chan,100000,12,16,128,1,7,25.435637
29 | chan,100000,12,16,128,1,8,28.245240
30 | chan,100000,12,16,128,1,9,24.528563
31 | chan,100000,12,16,128,1,10,27.792067
32 | chan,100000,12,16,128,1,11,28.284682
33 | chan,100000,12,16,128,1,12,28.398926
34 | chan,100000,12,16,128,1,13,28.064085
35 | chan,100000,12,16,128,1,14,27.722551
36 | chan,100000,12,16,128,1,15,27.945585
37 | chan,100000,12,16,128,1,16,27.936740
38 | chan,100000,12,16,128,1,17,27.591487
39 | chan,100000,12,16,128,1,18,28.128410
40 | chan,100000,12,16,128,1,19,27.597430
41 | chan,100000,12,16,128,1,20,27.567145
42 | chan,100000,12,16,128,10,1,27.021942
43 | chan,100000,12,16,128,10,2,27.381591
44 | chan,100000,12,16,128,10,3,27.646931
45 | chan,100000,12,16,128,10,4,26.951227
46 | chan,100000,12,16,128,10,5,28.278806
47 | chan,100000,12,16,128,10,6,27.964904
48 | chan,100000,12,16,128,10,7,28.244240
49 | chan,100000,12,16,128,10,8,27.615860
50 | chan,100000,12,16,128,10,9,28.215444
51 | chan,100000,12,16,128,10,10,27.167347
52 | chan,100000,12,16,128,10,11,27.595361
53 | chan,100000,12,16,128,10,12,27.618797
54 | chan,100000,12,16,128,10,13,27.547148
55 | chan,100000,12,16,128,10,14,27.300065
56 | chan,100000,12,16,128,10,15,27.808355
57 | chan,100000,12,16,128,10,16,27.218818
58 | chan,100000,12,16,128,10,17,27.017832
59 | chan,100000,12,16,128,10,18,27.885908
60 | chan,100000,12,16,128,10,19,26.738916
61 | chan,100000,12,16,128,10,20,27.247155
62 | chan,100000,12,16,128,100,1,27.034945
63 | chan,100000,12,16,128,100,2,28.234691
64 | chan,100000,12,16,128,100,3,27.598459
65 | chan,100000,12,16,128,100,4,27.291299
66 | chan,100000,12,16,128,100,5,27.678947
67 | chan,100000,12,16,128,100,6,27.056659
68 | chan,100000,12,16,128,100,7,28.539225
69 | chan,100000,12,16,128,100,8,27.786019
70 | chan,100000,12,16,128,100,9,27.309631
71 | chan,100000,12,16,128,100,10,27.551574
72 | chan,100000,12,16,128,100,11,27.652476
73 | chan,100000,12,16,128,100,12,27.467752
74 | chan,100000,12,16,128,100,13,24.288803
75 | chan,100000,12,16,128,100,14,27.395383
76 | chan,100000,12,16,128,100,15,24.060403
77 | chan,100000,12,16,128,100,16,28.417372
78 | chan,100000,12,16,128,100,17,27.703919
79 | chan,100000,12,16,128,100,18,27.478132
80 | chan,100000,12,16,128,100,19,23.975185
81 | chan,100000,12,16,128,100,20,27.867348
82 | chan,100000,12,16,128,1000,1,27.918440
83 | chan,100000,12,16,128,1000,2,27.861850
84 | chan,100000,12,16,128,1000,3,26.594190
85 | chan,100000,12,16,128,1000,4,27.599727
86 | chan,100000,12,16,128,1000,5,27.658375
87 | chan,100000,12,16,128,1000,6,27.025214
88 | chan,100000,12,16,128,1000,7,27.803807
89 | chan,100000,12,16,128,1000,8,27.876512
90 | chan,100000,12,16,128,1000,9,27.634843
91 | chan,100000,12,16,128,1000,10,27.179596
92 | chan,100000,12,16,128,1000,11,27.866468
93 | chan,100000,12,16,128,1000,12,27.517947
94 | chan,100000,12,16,128,1000,13,27.595231
95 | chan,100000,12,16,128,1000,14,28.281941
96 | chan,100000,12,16,128,1000,15,27.365247
97 | chan,100000,12,16,128,1000,16,27.375731
98 | chan,100000,12,16,128,1000,17,27.552872
99 | chan,100000,12,16,128,1000,18,27.886964
100 | chan,100000,12,16,128,1000,19,27.468668
101 | chan,100000,12,16,128,1000,20,27.491871
102 | chan,100000,12,16,128,10000,1,27.505759
103 | chan,100000,12,16,128,10000,2,28.449661
104 | chan,100000,12,16,128,10000,3,27.502533
105 | chan,100000,12,16,128,10000,4,28.213946
106 | chan,100000,12,16,128,10000,5,28.175616
107 | chan,100000,12,16,128,10000,6,28.522483
108 | chan,100000,12,16,128,10000,7,23.045822
109 | chan,100000,12,16,128,10000,8,27.745446
110 | chan,100000,12,16,128,10000,9,28.315042
111 | chan,100000,12,16,128,10000,10,27.612150
112 | chan,100000,12,16,128,10000,11,27.724441
113 | chan,100000,12,16,128,10000,12,28.278531
114 | chan,100000,12,16,128,10000,13,27.564547
115 | chan,100000,12,16,128,10000,14,27.545289
116 | chan,100000,12,16,128,10000,15,28.498775
117 | chan,100000,12,16,128,10000,16,28.194233
118 | chan,100000,12,16,128,10000,17,28.042158
119 | chan,100000,12,16,128,10000,18,27.733760
120 | chan,100000,12,16,128,10000,19,23.489180
121 | chan,100000,12,16,128,10000,20,27.867968
122 | chan,100000,12,16,128,100000,1,28.688822
123 | chan,100000,12,16,128,100000,2,28.172282
124 | chan,100000,12,16,128,100000,3,28.431106
125 | chan,100000,12,16,128,100000,4,28.655976
126 | chan,100000,12,16,128,100000,5,23.034884
127 | chan,100000,12,16,128,100000,6,28.475143
128 | chan,100000,12,16,128,100000,7,27.949890
129 | chan,100000,12,16,128,100000,8,28.711835
130 | chan,100000,12,16,128,100000,9,28.264331
131 | chan,100000,12,16,128,100000,10,28.390331
132 | chan,100000,12,16,128,100000,11,27.687716
133 | chan,100000,12,16,128,100000,12,28.300156
134 | chan,100000,12,16,128,100000,13,28.252632
135 | chan,100000,12,16,128,100000,14,28.841992
136 | chan,100000,12,16,128,100000,15,29.123085
137 | chan,100000,12,16,128,100000,16,24.058373
138 | chan,100000,12,16,128,100000,17,28.251988
139 | chan,100000,12,16,128,100000,18,23.060083
140 | chan,100000,12,16,128,100000,19,28.803184
141 | chan,100000,12,16,128,100000,20,28.249340
142 | chan,100000,12,16,128,1000000,1,23.341275
143 | chan,100000,12,16,128,1000000,2,27.960480
144 | chan,100000,12,16,128,1000000,3,27.965699
145 | chan,100000,12,16,128,1000000,4,28.260114
146 | chan,100000,12,16,128,1000000,5,22.787201
147 | chan,100000,12,16,128,1000000,6,28.314806
148 | chan,100000,12,16,128,1000000,7,27.769908
149 | chan,100000,12,16,128,1000000,8,27.955299
150 | chan,100000,12,16,128,1000000,9,28.364142
151 | chan,100000,12,16,128,1000000,10,28.056026
152 | chan,100000,12,16,128,1000000,11,28.292778
153 | chan,100000,12,16,128,1000000,12,28.040546
154 | chan,100000,12,16,128,1000000,13,28.281895
155 | chan,100000,12,16,128,1000000,14,28.111810
156 | chan,100000,12,16,128,1000000,15,27.549470
157 | chan,100000,12,16,128,1000000,16,26.898413
158 | chan,100000,12,16,128,1000000,17,29.058733
159 | chan,100000,12,16,128,1000000,18,27.354946
160 | chan,100000,12,16,128,1000000,19,28.469475
161 | chan,100000,12,16,128,1000000,20,28.109963
162 |
--------------------------------------------------------------------------------
/bench/block_darwin_x86_64.csv:
--------------------------------------------------------------------------------
1 | workload,ops,goroutines,depth,blockprofilerate,run,ms
2 | mutex,100000,12,16,0,1,42.438360
3 | mutex,100000,12,16,0,2,56.538516
4 | mutex,100000,12,16,0,3,52.666698
5 | mutex,100000,12,16,0,4,50.588831
6 | mutex,100000,12,16,0,5,42.993016
7 | mutex,100000,12,16,0,6,47.262495
8 | mutex,100000,12,16,0,7,42.614335
9 | mutex,100000,12,16,0,8,50.589187
10 | mutex,100000,12,16,0,9,53.232434
11 | mutex,100000,12,16,0,10,64.352130
12 | mutex,100000,12,16,0,11,52.387284
13 | mutex,100000,12,16,0,12,65.223335
14 | mutex,100000,12,16,0,13,50.572637
15 | mutex,100000,12,16,0,14,49.732913
16 | mutex,100000,12,16,0,15,46.789851
17 | mutex,100000,12,16,0,16,42.016991
18 | mutex,100000,12,16,0,17,50.339128
19 | mutex,100000,12,16,0,18,52.286547
20 | mutex,100000,12,16,0,19,50.750858
21 | mutex,100000,12,16,0,20,62.743093
22 | mutex,100000,12,16,1,1,43.440872
23 | mutex,100000,12,16,1,2,49.936582
24 | mutex,100000,12,16,1,3,50.284105
25 | mutex,100000,12,16,1,4,58.646662
26 | mutex,100000,12,16,1,5,63.336982
27 | mutex,100000,12,16,1,6,63.688515
28 | mutex,100000,12,16,1,7,63.627604
29 | mutex,100000,12,16,1,8,39.704634
30 | mutex,100000,12,16,1,9,43.121722
31 | mutex,100000,12,16,1,10,46.115743
32 | mutex,100000,12,16,1,11,50.286809
33 | mutex,100000,12,16,1,12,47.840831
34 | mutex,100000,12,16,1,13,62.910580
35 | mutex,100000,12,16,1,14,51.057369
36 | mutex,100000,12,16,1,15,54.046803
37 | mutex,100000,12,16,1,16,48.035078
38 |
--------------------------------------------------------------------------------
/bench/block_linux_x86_64.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/bench/block_linux_x86_64.png
--------------------------------------------------------------------------------
/bench/csv.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "time"
6 | )
7 |
8 | type Record struct {
9 | Blockprofilerate int
10 | Bufsize int
11 | Depth int
12 | Duration time.Duration
13 | Goroutines int
14 | Ops int
15 | Run int
16 | Workload string
17 | }
18 |
19 | type Column struct {
20 | Name string
21 | MarshalValue func(*Record) (string, error)
22 | }
23 |
24 | var Columns = []Column{
25 | {"workload", func(r *Record) (string, error) {
26 | return fmt.Sprintf("%s", r.Workload), nil
27 | }},
28 | {"ops", func(r *Record) (string, error) {
29 | return fmt.Sprintf("%d", r.Ops), nil
30 | }},
31 | {"goroutines", func(r *Record) (string, error) {
32 | return fmt.Sprintf("%d", r.Goroutines), nil
33 | }},
34 | {"depth", func(r *Record) (string, error) {
35 | return fmt.Sprintf("%d", r.Depth), nil
36 | }},
37 | {"bufsize", func(r *Record) (string, error) {
38 | return fmt.Sprintf("%d", r.Bufsize), nil
39 | }},
40 | {"blockprofilerate", func(r *Record) (string, error) {
41 | return fmt.Sprintf("%d", r.Blockprofilerate), nil
42 | }},
43 | {"run", func(r *Record) (string, error) {
44 | return fmt.Sprintf("%d", r.Run), nil
45 | }},
46 | {"ms", func(r *Record) (string, error) {
47 | return fmt.Sprintf("%f", r.Duration.Seconds()*1000), nil
48 | }},
49 | }
50 |
51 | func (r *Record) MarshalRecord() ([]string, error) {
52 | record := make([]string, len(Columns))
53 | for i, col := range Columns {
54 | val, err := col.MarshalValue(r)
55 | if err != nil {
56 | return nil, err
57 | }
58 | record[i] = val
59 | }
60 | return record, nil
61 | }
62 |
63 | func Headers() []string {
64 | headers := make([]string, len(Columns))
65 | for i, col := range Columns {
66 | headers[i] = col.Name
67 | }
68 | return headers
69 | }
70 |
--------------------------------------------------------------------------------
/bench/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "encoding/csv"
6 | "flag"
7 | "fmt"
8 | "os"
9 | "os/exec"
10 | "runtime"
11 | "runtime/pprof"
12 | "strconv"
13 | "strings"
14 | "time"
15 | )
16 |
17 | func main() {
18 | if err := run(); err != nil {
19 | fmt.Fprintln(os.Stderr, err)
20 | os.Exit(1)
21 | }
22 | }
23 |
24 | func run() error {
25 | if os.Getenv("WORKER") == "" {
26 | return leader()
27 | } else {
28 | return worker()
29 | }
30 | }
31 |
32 | func leader() error {
33 | var (
34 | blockprofilerates = flagIntSlice("blockprofilerates", []int{0, 1, 10, 100, 1000, 10000, 100000, 1000000}, "The runtime.SetBlockProfileRate() values to benchmark.")
35 | bufsizes = flagIntSlice("bufsizes", []int{0, 64}, "The buffer sizes to use for channel operations (not applicable to all workloads).")
36 | depths = flagIntSlice("depths", []int{2, 4, 8, 16, 32}, "The different frame depths values to use for each workload.")
37 | goroutines = flagIntSlice("goroutines", []int{runtime.NumCPU()}, "The number of goroutine values to use for each workloads.")
38 | ops = flag.Int("ops", 1000, "The number of operations to perform for each workload.")
39 | runs = flag.Int("runs", 3, "The number of times to repeat the same benchmark to understand variance.")
40 | workloads = flagStringSlice("workloads", []string{"mutex", "chan"}, "The workloads to benchmark.")
41 | )
42 | flag.Parse()
43 |
44 | cw := csv.NewWriter(os.Stdout)
45 | cw.Write(Headers())
46 | cw.Flush()
47 |
48 | for _, workload := range *workloads {
49 | for _, goroutine := range *goroutines {
50 | for _, blockprofilerate := range *blockprofilerates {
51 | for _, depth := range *depths {
52 | for _, bufsize := range *bufsizes {
53 | for run := 1; run <= *runs; run++ {
54 | cmd := exec.Command(os.Args[0],
55 | "-run", fmt.Sprintf("%d", run),
56 | "-blockprofilerate", fmt.Sprintf("%d", blockprofilerate),
57 | "-ops", fmt.Sprintf("%d", *ops),
58 | "-goroutines", fmt.Sprintf("%d", goroutine),
59 | "-depth", fmt.Sprintf("%d", depth),
60 | "-bufsize", fmt.Sprintf("%d", bufsize),
61 | "-workload", workload,
62 | )
63 |
64 | buf := &bytes.Buffer{}
65 | cmd.Stdout = buf
66 | cmd.Stderr = os.Stderr
67 | cmd.Env = append(cmd.Env, "WORKER=yeah")
68 |
69 | if err := cmd.Run(); err != nil {
70 | return err
71 | }
72 |
73 | buf.WriteTo(os.Stdout)
74 | }
75 | }
76 | }
77 | }
78 | }
79 | }
80 |
81 | return nil
82 | }
83 |
84 | func worker() error {
85 | var (
86 | blockprofilerate = flag.Int("blockprofilerate", 1, "The block profile rate to use.")
87 | bufsize = flag.Int("bufsize", 0, "The buffer size to use for channel operations (not applicable to all workloads).")
88 | depth = flag.Int("depth", 16, "The stack depth at which to perform blocking events.")
89 | goroutines = flag.Int("goroutines", runtime.NumCPU(), "The number of goroutines to utilize.")
90 | ops = flag.Int("ops", 100000, "The number of operations to perform.")
91 | out = flag.String("blockprofile", "", "Path to a file for writing the block profile.")
92 | run = flag.Int("run", 1, "The number of run. Has no impact on the benchmark, but gets included in the csv output line.")
93 | workload = flag.String("workload", "mutex", "The workload to simulate.")
94 | )
95 | flag.Parse()
96 |
97 | if *blockprofilerate > 0 {
98 | runtime.SetBlockProfileRate(*blockprofilerate)
99 | }
100 |
101 | start := time.Now()
102 | switch *workload {
103 | case "mutex":
104 | if err := mutexWorkload(*goroutines, *ops, *depth); err != nil {
105 | return err
106 | }
107 | case "chan":
108 | if err := chanWorkload(*goroutines, *ops, *depth, *bufsize); err != nil {
109 | return err
110 | }
111 | default:
112 | return fmt.Errorf("unknown workload: %q", *workload)
113 | }
114 | duration := time.Since(start)
115 |
116 | if *blockprofilerate > 0 && *out != "" {
117 | f, err := os.Create(*out)
118 | if err != nil {
119 | return err
120 | }
121 | defer f.Close()
122 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil {
123 | return err
124 | }
125 | }
126 |
127 | cw := csv.NewWriter(os.Stdout)
128 | record, err := (&Record{
129 | Blockprofilerate: *blockprofilerate,
130 | Bufsize: *bufsize,
131 | Depth: *depth,
132 | Duration: duration,
133 | Goroutines: *goroutines,
134 | Ops: *ops,
135 | Run: *run,
136 | Workload: *workload,
137 | }).MarshalRecord()
138 | if err != nil {
139 | return err
140 | }
141 | cw.Write(record)
142 | cw.Flush()
143 | return cw.Error()
144 | }
145 |
146 | func atStackDepth(depth int, fn func()) {
147 | pcs := make([]uintptr, depth*10)
148 | n := runtime.Callers(1, pcs)
149 | if n > depth {
150 | panic("depth exceeded")
151 | } else if n < depth {
152 | atStackDepth(depth, fn)
153 | return
154 | }
155 |
156 | fn()
157 | }
158 |
159 | func flagIntSlice(name string, value []int, usage string) *[]int {
160 | val := &intSlice{vals: value}
161 | flag.Var(val, name, usage)
162 | return &val.vals
163 | }
164 |
165 | type intSlice struct {
166 | vals []int
167 | }
168 |
169 | func (i *intSlice) Set(val string) error {
170 | var vals []int
171 | for _, val := range strings.Split(val, ",") {
172 | num, err := strconv.Atoi(val)
173 | if err != nil {
174 | return err
175 | }
176 | vals = append(vals, num)
177 | }
178 | i.vals = vals
179 | return nil
180 | }
181 |
182 | func (i *intSlice) String() string {
183 | return fmt.Sprintf("%v", i.vals)
184 | }
185 |
186 | func flagStringSlice(name string, value []string, usage string) *[]string {
187 | val := &strSlice{vals: value}
188 | flag.Var(val, name, usage)
189 | return &val.vals
190 | }
191 |
192 | type strSlice struct {
193 | vals []string
194 | }
195 |
196 | func (s *strSlice) Set(val string) error {
197 | s.vals = strings.Split(val, ",")
198 | return nil
199 | }
200 |
201 | func (s *strSlice) String() string {
202 | return fmt.Sprintf("%v", s.vals)
203 | }
204 |
--------------------------------------------------------------------------------
/bench/run.sh:
--------------------------------------------------------------------------------
1 | os_arch() {
2 | echo "$(uname | tr '[:upper:]' '[:lower:]')_$(uname -m)"
3 | }
4 |
5 | #go run . \
6 | #-workloads mutex,chan \
7 | #-ops 100000 \
8 | #-blockprofilerates 0,1,10,100,1000,10000,100000,1000000 \
9 | #-runs 20 \
10 | #-bufsizes 0 \
11 | #-depths 16 \
12 | #> "block_$(os_arch).csv"
13 |
14 | go run . \
15 | -workloads chan \
16 | -ops 100000 \
17 | -blockprofilerates 0,1,10,100,1000,10000,100000,1000000 \
18 | -bufsizes 128 \
19 | -runs 20 \
20 | -depths 16 \
21 | > "block_bufchan_$(os_arch).csv"
22 |
--------------------------------------------------------------------------------
/bench/workload_chan.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | )
7 |
8 | func chanWorkload(goroutines, ops, depth, bufsize int) error {
9 | if goroutines%2 != 0 {
10 | return fmt.Errorf("bad goroutines: %d: must be a multiple of 2", goroutines)
11 | }
12 |
13 | wg := &sync.WaitGroup{}
14 | for j := 0; j < goroutines/2; j++ {
15 | ch := make(chan struct{}, bufsize)
16 | wg.Add(1)
17 | go atStackDepth(depth, func() {
18 | defer wg.Done()
19 | for i := 0; i < ops; i++ {
20 | ch <- struct{}{}
21 | }
22 | })
23 | wg.Add(1)
24 | go atStackDepth(depth, func() {
25 | defer wg.Done()
26 | for i := 0; i < ops; i++ {
27 | <-ch
28 | }
29 | })
30 | }
31 | wg.Wait()
32 | return nil
33 | }
34 |
--------------------------------------------------------------------------------
/bench/workload_mutex.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | )
7 |
8 | func mutexWorkload(goroutines, ops, depth int) error {
9 | if goroutines%2 != 0 {
10 | return fmt.Errorf("bad goroutines: %d: must be a multiple of 2", goroutines)
11 | }
12 |
13 | wg := &sync.WaitGroup{}
14 | for j := 0; j < goroutines/2; j++ {
15 | m := &sync.Mutex{}
16 | wg.Add(1)
17 | go atStackDepth(depth, func() {
18 | defer wg.Done()
19 | for i := 0; i < ops; i++ {
20 | m.Lock()
21 | m.Unlock()
22 | }
23 | })
24 | wg.Add(1)
25 | go atStackDepth(depth, func() {
26 | defer wg.Done()
27 | for i := 0; i < ops; i++ {
28 | m.Lock()
29 | m.Unlock()
30 | }
31 | })
32 | }
33 | wg.Wait()
34 | return nil
35 | }
36 |
--------------------------------------------------------------------------------
/block-bias.md:
--------------------------------------------------------------------------------
1 | ⬅ [Block Profiling in Go](./block.md)
2 |
3 | ⚠️This document describes a sampling bias issue I discovered while researching the block profiler for Go. I have since [landed a fix for it](https://go-review.googlesource.com/c/go/+/299991) that should appear in Go 1.17.
4 |
5 | # Block Profiler Sampling Bias
6 |
7 | **tl;dr:** Setting your sampling `rate` too high will bias your results towards infrequent long events over frequent short events.
8 |
9 | As described in the [Usage](#usage) section, the block profiler will sample as follows:
10 |
11 | - Events with `duration >= rate` will be sampled 100%
12 | - Events with `duration < rate` have a `duration / rate` chance of getting sampled.
13 |
14 | The [implementation](https://github.com/golang/go/blob/go1.15.7/src/runtime/mprof.go#L408) for this looks like that:
15 |
16 | ```go
17 | func blocksampled(cycles int64) bool {
18 | rate := int64(atomic.Load64(&blockprofilerate))
19 | if rate <= 0 || (rate > cycles && int64(fastrand())%rate > cycles) {
20 | return false
21 | }
22 | return true
23 | }
24 | ```
25 |
26 | This means that if you set your profiling `rate` low enough, you'll get very accurate results. However, if your `rate` is higher than the `duration` of some of the events you are sampling, the sampling process will exhibit a bias favoring infrequent events of higher `duration` over frequent events with lower `duration` even so they may contribute to the same amount of overall block duration in your program.
27 |
28 | ## Simple Example
29 |
30 | Let's say your `blockprofilerate` is `100ns` and your application produces the following events:
31 |
32 | - `A`: `1` event with a duration of `100ns`.
33 | - `B`: `10` events with a duration of `10ns` each.
34 |
35 | Given this scenario, the `blockprofiler` is guaranteed to catch and accurately report event `A` as `100ns` in the profile. For event `B` the most likely outcome is that the profiler will capture only a single event (10% of 10 events) and report `B` as `10ns` in the profile. So you might find yourself in a situation where you think event `A` is causing 10x more blocking than event `B`, which is not true.
36 |
37 | ## Simulation & Proposal for Improvement
38 |
39 | For an even better intuition about this, consider the [simulated example](./sim/block_sampling.ipynb) below. Here we have a histogram of all durations collected from 3 types of blocking events. As you can see, they all have different mean durations (`1000ns`, `2000ns`, `3000ns`) and they are occurring at different frequencies, with `count(a) > count(b) > count(c)`. What's more difficult to see, is that the cumulative durations of these events are the same, i.e. `sum(a) = sum(b) = sum(c)`, but you can trust me on that : ).
40 |
41 |
42 |
43 | So given that your application might produce events like this, how will they show up in your block profile as you try out different `blockprofilerate` values? As you can see below, all is well and fine until a `blockprofilerate` of `1000ns`. Each event shows up with the same total duration in the profile (the red and green dots are hidden below the blue ones). However starting at `1000ns` you see that event `a` starts to fade from our profile and at `2000ns` you'd already think that events `b` and `c` are causing twice as much blocking time as event `a`.
44 |
45 |
46 |
47 | So what can we do? Do we always need to live in fear of bias when working with block profiles? No! If the [Overhead](#overhead) for your workload allows it, the simplest solution is to use a low enough `blockprofilerate` in order to capture most blocking events.
48 |
49 | But perhaps there is an even better way. I'm thinking we could correct for the current bias by keeping the same logic of sampling `duration / rate` fraction of events when `duration < rate`. However, when this happens we could simply multiply the sampled duration by `rate/duration` like this:
50 |
51 | ```
52 | duration = duration * (rate/duration)
53 | # note: the expression above can be simplified to just `duration = rate`
54 | ```
55 |
56 | Doing so could be done with a [trivial patch](https://github.com/felixge/go/compare/master...debias-blockprofile-rate) to the go runtime and the picture below shows the results from simulating it.
57 |
58 |
59 |
60 | ## Disclaimers
61 |
62 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ).
63 |
64 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
65 |
--------------------------------------------------------------------------------
/cpu.md:
--------------------------------------------------------------------------------
1 | 🚧 This note is still work in progress, please come back later! 🚧
2 |
3 | # CPU Profiling in Go
4 |
5 | ## How it works
6 |
7 | Go's CPU profiler works by sampling the profiled program at a configurable rate of 100 Hz. A sample consists mostly of the stack trace of the goroutine that was interrupted. Below is an example of a program that has a function `A()` that sometimes calls function `C()` and `B()` which doesn't call anything.
8 |
9 | ```
10 | ------------------------------------> time
11 | | | | | | | | | | | | |
12 | A B A B A A A B A B A A
13 | C C C
14 |
15 | <-->
16 | 10ms
17 | ```
18 |
19 | The resulting [pprof output](./pprof.md) doesn't contain the raw sample events. Instead it aggregates all samples and reports the **samples/count** and **cpu/nanoseconds** values. The latter is [derived](https://github.com/golang/go/blob/go1.15.6/src/runtime/pprof/proto.go#L354) by multiplying the sample count with the sample period and is therefore redundant. I suspect it's included because pprof doesnt support the count to time conversion, but if anybody has the full story please let me know!
20 |
21 | Back to the example above, the pprof output file would containing the following:
22 |
23 | | stack trace | samples/count | cpu/nanoseconds |
24 | | ----------- | ------------- | --------------- |
25 | | A | 5 | 50000000 |
26 | | A C | 3 | 30000000 |
27 | | B | 4 | 40000000 |
28 |
29 | Please note that the **stack trace** is a bit more complex in reality as it includes the exact program counter address at which your program was sampled as well as the line number in your code that it maps to. This information is usually omitted when visualizing the data as a [Flame Graph](http://www.brendangregg.com/flamegraphs.html), but can be very useful when drilling down into the details. Speaking of flame graphs, here is what the data above looks like when visualized:
30 |
31 | 
32 |
33 | ## Usage
34 |
35 | The various ways one can record CPU profiles in Go are listed below.
36 |
37 | 1. The `go test` command has a `-cpuprofile` flag described that can be used to profile the benchmarks in your test suite like shown below. You can also [add a flag like this](https://golang.org/pkg/runtime/pprof/#hdr-Profiling_a_Go_program) to you own programs which might makes sense for command line tools.
38 |
39 | ```
40 | go test -cpuprofile benchmark.cpu.pb.gz -bench .
41 | go tool pprof -http=:6061 benchmark.cpu.pb.gz
42 | ```
43 |
44 | 2. The [net/http/pprof](https://golang.org/pkg/net/http/pprof/) allows you to setup http endpoints that can start/stop the CPU profiler via http requests on-demand and return the resulting pprof data file. You can directly pass a URL to such an endpoint to the pprof tool.
45 |
46 | ```
47 | go tool pprof -http=:6061 http://localhost:6060/debug/pprof/profile?seconds=30
48 | ```
49 |
50 | 3. You can programmatically start/stop the profiler via the [runtime/pprof API](https://golang.org/pkg/runtime/pprof/#StartCPUProfile).
51 |
52 | ```go
53 | pprof.StartCPUProfile(outputFile)
54 | defer pprof.StopCPUProfile()
55 | //
56 | ```
57 |
58 | 4. You can use a service such as [Datadog's Continious Profiler](https://www.datadoghq.com/product/code-profiling/) to automatically take and upload CPU profiles of your applications running in production. Disclaimer: I just started working for Datadog to take Go profiling to the next level.
59 |
60 | ## Use Cases
61 |
62 | - Break down CPU utilization by functions or even line numbers.
63 | - Break down CPU utilization by user-dimensions (endpoint, request_id, etc.) using [profiler labels](https://rakyll.org/profiler-labels/).
64 |
65 | ## Anti-Use Cases
66 |
67 | - Understanding [Off-CPU](http://www.brendangregg.com/offcpuanalysis.html) time your program spends while waiting on I/O, Locks, Timers, Scheduling, etc.
68 | - High-resolution sampling. The default rate is 100 Hz and you won't be able to go much higher for reasons outlined later on.
69 |
70 | ## Todo
71 |
72 | - Talk about multi threading.
73 | - setitimer
74 | - Profiler labels
75 | - bias
76 | - performance overhead (including negative!)
77 | - https://github.com/golang/go/issues/38325
78 | - accuracy
79 | - https://github.com/golang/go/issues/35057
80 | - Discuss [Proposal: hardware performance counters for CPU profiling.](https://go.googlesource.com/proposal/+/refs/changes/08/219508/2/design/36821-perf-counter-pprof.md)
81 |
82 | ## Disclaimers
83 |
84 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ).
85 |
86 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
87 |
--------------------------------------------------------------------------------
/datadog.md:
--------------------------------------------------------------------------------
1 | # Datadog's Go Profiler
2 |
3 | The profiler code is a sub-package in the [dd-trace-go](https://github.com/DataDog/dd-trace-go/tree/v1/profiler) repo. Basic integration is described in the [Getting Started](https://docs.datadoghq.com/tracing/profiler/getting_started/?tab=go) guide, and the [package docs](https://pkg.go.dev/gopkg.in/DataDog/dd-trace-go.v1/profiler#pkg-constants) explain additional API options. The minimum required Go version is 1.12.
4 |
5 | Users invoke the profiler by calling `profiler.Start()` with a few options, especially tags for identifying the source of the data. Every 60s the profiler then takes a 15s CPU profile as well as a heap profile and uploads it using the [payload format](#payload-format) described below.
6 |
7 | The data is sent to the local [datadog agent](https://docs.datadoghq.com/agent/) which forwards it to Datadog's backend. It's also possible to directly upload the profiles without an agent using an API key is given, but this method is deprecated and not supported.
8 |
9 | ## Operation Details
10 |
11 | The [`Start()`](https://pkg.go.dev/gopkg.in/DataDog/dd-trace-go.v1/profiler#Start) function invokes two background goroutines. The `collect()` routine captures profiling data as a `batch` struct and puts it into a Go channel for the `send()` routine to read. More details can be seen below.
12 |
13 | 
14 |
15 | A `batch` has a `start` and an `end` time as well as a slice of `profiles`. Each profile corresponds to one of the supported [profile type](https://pkg.go.dev/gopkg.in/DataDog/dd-trace-go.v1/profiler#ProfileType), e.g. CPU, Heap, etc..
16 |
17 | ## Payload Format
18 |
19 | TODO: This is outdated, needs updating [based on this PR](https://github.com/DataDog/dd-trace-go/pull/781).
20 |
21 | The payload uses `multipart/form-data` encoding and includes the following form fields for every `batch` that is being uploaded.
22 |
23 | - `format`: Always `pprof`
24 | - `runtime`: Always `go`
25 | - `recording-start`: The batch start time formatted as `2006-01-02T15:04:05Z07:00` in UTC.
26 | - `recording-end`: The batch end time formatted as `2006-01-02T15:04:05Z07:00` in UTC.
27 | - `tags[]`: The profiler's `p.cfg.tags` + `service:p.cfg.service` + `env:p.cfg.env` + `host:bat.host` (if set) + `runtime:go`
28 | - `types[0..n]`: The comma separates types included in each profile, e.g. `alloc_objects,alloc_space,inuse_objects,inuse_space`.
29 | - `data[0..n]`: One file field for each profile. The filename is always `pprof-data`, and the pprof data is compressed (by Go).
30 |
31 | TODO: Link to a sample payload file.
32 |
33 | ## Disclaimers
34 |
35 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ).
36 |
37 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
--------------------------------------------------------------------------------
/datadog.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/datadog.png
--------------------------------------------------------------------------------
/delve/stackannotate.star:
--------------------------------------------------------------------------------
1 | word_size = 8 # 64bit only for now
2 |
3 | def get_reg(name):
4 | for req in registers().Regs:
5 | if req.Name == name:
6 | return int(req.Value, 16)
7 |
8 | def read_word(addr):
9 | v = 0
10 | m = examine_memory(addr, word_size)
11 | s = 1
12 | for b in m.Mem:
13 | v += b * s
14 | s = s * 256
15 | return v
16 |
17 | def hex(d, n = 0):
18 | if d == 0:
19 | return lpad("0", n, "0")
20 |
21 | lookup = {10: "a", 11: "b", 12: "c", 13: "d", 14: "e", 15: "e", 16: "f"}
22 | s = ""
23 | while d > 0:
24 | r = d % 16
25 | d = d // 16
26 | if r >= 10:
27 | r = lookup[r]
28 | else:
29 | r = str(r)
30 | s = r + s
31 | return lpad(s, n, "0")
32 |
33 | def lpad(s, n, c = " "):
34 | while len(s) < n:
35 | s = c + s
36 | return s
37 |
38 | def rpad(s, n, c = " "):
39 | while len(s) < n:
40 | s = s + c
41 | return s
42 |
43 | def ascii_table(rows, align_right = {}):
44 | widths = []
45 | for row in rows:
46 | for i, col in enumerate(row):
47 | if len(widths) < i+1:
48 | widths.append(0)
49 | widths[i] = max(widths[i], len(col))
50 |
51 | s = ""
52 | for row in rows:
53 | for i, col in enumerate(row):
54 | width = widths[i]
55 | if align_right.get(i, False):
56 | col = lpad(col, width)
57 | else:
58 | col = rpad(col, width)
59 | s += col + " "
60 | s += "\n"
61 | return s
62 |
63 | def getg():
64 | # TODO(fg) there is probably a better way to implement this.
65 | g = raw_command("goroutine").State.SelectedGoroutine
66 | for gp in eval(None, "runtime.allgs").Variable.Value:
67 | if gp.goid == g.ID:
68 | return gp
69 |
70 | def stack():
71 | g = getg()
72 | bp = get_reg("Rbp")
73 | ip = get_reg("Rip")
74 | sp = get_reg("Rsp")
75 | regs = {"sp": sp, "bp": bp, "ip": ip}
76 |
77 | addr_list = []
78 | addr_dict = {}
79 | offset = 0
80 | while True:
81 | addr = g.stack.hi+offset-word_size
82 | addr_info = {
83 | "addr": addr,
84 | "val": read_word(addr),
85 | "offset": offset,
86 | "regs": [],
87 | "note": [],
88 | "func": None,
89 | "arg": None,
90 | "local": None,
91 | "fp": False,
92 | }
93 | for (name, val) in regs.items():
94 | if addr == val:
95 | addr_info["regs"].append(name)
96 |
97 | addr_list.append(addr_info)
98 | addr_dict[addr] = addr_info
99 | offset -= word_size
100 | if addr <= sp:
101 | break
102 |
103 | for f in stacktrace(g.goid, 128, True).Locations:
104 | fp_addr = g.stack.hi+f.FramePointerOffset
105 | if fp_addr > 0 and len(addr_dict[fp_addr]["note"]) == 0:
106 | addr_dict[fp_addr]["note"].append("frame pointer for "+f.Function.Name_)
107 | pc_addr = fp_addr+word_size
108 | pc = read_word(pc_addr)
109 | ins = disassemble(None, pc, pc+1).Disassemble[0]
110 | addr_dict[pc_addr]["note"].append("return addr to "+ins.Loc.Function.Name_)
111 |
112 | for arg in f.Arguments:
113 | addr_dict[arg.Addr]["note"].append("arg "+arg.Name+" "+arg.Type)
114 | for local in f.Locals:
115 | addr = local.Addr // 8 * 8
116 | if addr_dict.get(addr):
117 | addr_dict[addr]["note"].append("var "+local.Name+" "+local.Type)
118 |
119 | return addr_list
120 |
121 | # stackannotate (alias sa) will print an annotated stack dump.
122 | def command_stackannotate():
123 | rows = [["regs", "addr", "offset", "value", "explanation"]]
124 | for addr_info in stack():
125 | regs = ""
126 | if len(addr_info["regs"]) > 0:
127 | regs = ",".join(addr_info["regs"])+" -->"
128 |
129 | note = "?"
130 | if len(addr_info["note"]) > 0:
131 | note = ", ".join(addr_info["note"])
132 |
133 | rows.append([
134 | regs,
135 | hex(addr_info["addr"]),
136 | lpad(str(addr_info["offset"]), 6),
137 | lpad(hex(addr_info["val"]), word_size*2+2),
138 | note,
139 | ])
140 | print(ascii_table(rows))
141 |
142 | def main():
143 | dlv_command("config alias stackannotate sa")
144 |
--------------------------------------------------------------------------------
/examples/block-bias/block.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block-bias/block.pb.gz
--------------------------------------------------------------------------------
/examples/block-bias/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/block-bias
2 |
3 | go 1.15
4 |
5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
6 |
--------------------------------------------------------------------------------
/examples/block-bias/go.sum:
--------------------------------------------------------------------------------
1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
3 |
--------------------------------------------------------------------------------
/examples/block-bias/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "runtime"
7 | "runtime/pprof"
8 | "sync"
9 | "time"
10 | )
11 |
12 | func main() {
13 | if err := run(); err != nil {
14 | fmt.Fprintln(os.Stderr, err)
15 | os.Exit(1)
16 | }
17 | }
18 |
19 | var (
20 | fastEventDuration = 1 * time.Millisecond
21 | slowEventDuration = 1 * fastEventDuration
22 | )
23 |
24 | func run() error {
25 | runtime.SetBlockProfileRate(int(slowEventDuration.Nanoseconds()))
26 |
27 | var (
28 | done = make(chan struct{})
29 | wg = &sync.WaitGroup{}
30 | )
31 | wg.Add(1)
32 | go func() {
33 | defer wg.Done()
34 | slowEvent(done)
35 | }()
36 | wg.Add(1)
37 | go func() {
38 | defer wg.Done()
39 | fastEvent(done)
40 | }()
41 | time.Sleep(1 * time.Second)
42 | close(done)
43 | wg.Wait()
44 |
45 | f, err := os.Create("block.pb.gz")
46 | if err != nil {
47 | return err
48 | }
49 | defer f.Close()
50 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil {
51 | return err
52 | }
53 | return nil
54 | }
55 |
56 | func slowEvent(done chan struct{}) error {
57 | return simulateBlockEvents(slowEventDuration, done)
58 | }
59 |
60 | func fastEvent(done chan struct{}) error {
61 | return simulateBlockEvents(fastEventDuration, done)
62 | }
63 |
64 | func simulateBlockEvents(duration time.Duration, done chan struct{}) error {
65 | ticker := time.NewTicker(duration)
66 | defer ticker.Stop()
67 | for {
68 | select {
69 | case <-ticker.C:
70 | // do nothing
71 | case <-done:
72 | return nil
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/examples/block-net/README.md:
--------------------------------------------------------------------------------
1 | # block-net
2 |
3 | This [program](./main.go) explores the [question](https://twitter.com/rogpeppe/status/1359202847708037124) whether network i/o (e.g. waiting on socket read/write operations) will show up in the [block profiler](../../block.md) or not.
4 |
5 | The program does the following:
6 |
7 | 1. Start a TCP server via `net.Listen()` and wait for a client using `l.Accept()`.
8 | 2. Sleep for 1s
9 | 3. Connect to the TCP server with a client using `net.Dial()`.
10 | 4. Let the server wait for a message from the client using `conn.Read()`
11 | 4. Sleep for 1s
12 | 5. Send a `"hello world"` message from the client to the server using `conn.Write()`
13 | 6. Server prints the message it received
14 | 7. Program exits
15 |
16 | Given this program and assuming the block profiler captures network i/o, we'd expect to capture two major blocking events:
17 |
18 | 1. `l.Accept()` waiting for 1s before a client connects.
19 | 2. `conn.Read()` waiting for 1s before receiving a message from the client
20 |
21 | However, as you can see below, the block profiler [captures](./block.pb.gz) only `~12ms` of activity related to channel operations involved in the listen/dial process. The accept/read activity however remains completely invisible.
22 |
23 | 
24 |
25 | This means that [block profiler](../../block.md) is generally not able to give a good idea about goroutines that are waiting on network i/o.
26 |
27 |
--------------------------------------------------------------------------------
/examples/block-net/block-net.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block-net/block-net.png
--------------------------------------------------------------------------------
/examples/block-net/block.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block-net/block.pb.gz
--------------------------------------------------------------------------------
/examples/block-net/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/block-net
2 |
3 | go 1.15
4 |
5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
6 |
--------------------------------------------------------------------------------
/examples/block-net/go.sum:
--------------------------------------------------------------------------------
1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
3 |
--------------------------------------------------------------------------------
/examples/block-net/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net"
6 | "os"
7 | "runtime"
8 | "runtime/pprof"
9 | "time"
10 |
11 | "golang.org/x/sync/errgroup"
12 | )
13 |
14 | func main() {
15 | if err := run(); err != nil {
16 | fmt.Fprintln(os.Stderr, err)
17 | os.Exit(1)
18 | }
19 | }
20 |
21 | func run() error {
22 | runtime.SetBlockProfileRate(1)
23 |
24 | listening := make(chan struct{})
25 | g := &errgroup.Group{}
26 | g.Go(func() error {
27 | return server(listening)
28 | })
29 |
30 | <-listening
31 | time.Sleep(time.Second)
32 |
33 | if err := client(); err != nil {
34 | return err
35 | }
36 | if err := g.Wait(); err != nil {
37 | return err
38 | }
39 |
40 | f, err := os.Create("block.pb.gz")
41 | if err != nil {
42 | return err
43 | }
44 | defer f.Close()
45 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil {
46 | return err
47 | }
48 | return nil
49 | }
50 |
51 | func server(listening chan struct{}) error {
52 | l, err := net.Listen("tcp", "localhost:9090")
53 | if err != nil {
54 | return err
55 | }
56 |
57 | listening <- struct{}{}
58 |
59 | conn, err := l.Accept()
60 | if err != nil {
61 | return err
62 | }
63 | fmt.Printf("[server] accepted: %v\n", conn)
64 | buf := make([]byte, 1024)
65 | n, err := conn.Read(buf)
66 | if err != nil {
67 | return err
68 | }
69 | buf = buf[:n]
70 | fmt.Printf("[server] read: %s\n", buf)
71 | return nil
72 | }
73 |
74 | func client() error {
75 | conn, err := net.Dial("tcp", "localhost:9090")
76 | if err != nil {
77 | return err
78 | }
79 | defer conn.Close()
80 |
81 | time.Sleep(time.Second)
82 |
83 | fmt.Printf("[client] connected: %v\n", conn)
84 | _, err = conn.Write([]byte("hello world"))
85 | return err
86 | }
87 |
--------------------------------------------------------------------------------
/examples/block-sample/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/block-sample
2 |
3 | go 1.15
4 |
5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
6 |
--------------------------------------------------------------------------------
/examples/block-sample/go.sum:
--------------------------------------------------------------------------------
1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
3 |
--------------------------------------------------------------------------------
/examples/block-sample/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "os"
7 | "runtime"
8 | "runtime/pprof"
9 | "time"
10 |
11 | "golang.org/x/sync/errgroup"
12 | )
13 |
14 | func main() {
15 | if err := run(); err != nil {
16 | fmt.Fprintln(os.Stderr, err)
17 | os.Exit(1)
18 | }
19 | }
20 |
21 | func run() error {
22 | labels := pprof.Labels("test_label", "test_value")
23 | ctx := pprof.WithLabels(context.Background(), labels)
24 | pprof.SetGoroutineLabels(ctx)
25 |
26 | runtime.SetBlockProfileRate(int((40 * time.Microsecond).Nanoseconds()))
27 | done := make(chan struct{})
28 | g := errgroup.Group{}
29 | g.Go(func() error {
30 | return eventA(done)
31 | })
32 | g.Go(func() error {
33 | return eventB(done)
34 | })
35 | time.Sleep(time.Second)
36 | close(done)
37 | if err := g.Wait(); err != nil {
38 | return err
39 | }
40 |
41 | f, err := os.Create("block.pb.gz")
42 | if err != nil {
43 | return err
44 | }
45 | defer f.Close()
46 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil {
47 | return err
48 | }
49 | return nil
50 | }
51 |
52 | func eventA(done chan struct{}) error {
53 | return simulateBlockEvents(20*time.Microsecond, done)
54 | }
55 |
56 | func eventB(done chan struct{}) error {
57 | return simulateBlockEvents(40*time.Microsecond, done)
58 | }
59 |
60 | const tolerance = 1.1
61 |
62 | func simulateBlockEvents(meanDuration time.Duration, done chan struct{}) error {
63 | var (
64 | prev time.Time
65 | sum time.Duration
66 | count int
67 | ticker = time.NewTicker(meanDuration)
68 | )
69 | defer ticker.Stop()
70 | for {
71 | select {
72 | case <-ticker.C:
73 | now := time.Now()
74 | if !prev.IsZero() {
75 | sum += now.Sub(prev)
76 | count += 1
77 | if count > 1000 {
78 | actualMean := float64(sum) / float64(count)
79 | max := tolerance * float64(meanDuration)
80 | min := float64(meanDuration) / tolerance
81 | if actualMean <= min || actualMean >= max {
82 | return fmt.Errorf("low clock accuracy: got=%s want=%s", time.Duration(actualMean), meanDuration)
83 | }
84 | }
85 | }
86 | prev = now
87 | case <-done:
88 | return nil
89 | }
90 | }
91 | }
92 |
93 | /*
94 |
95 | Bias in current go version:
96 |
97 | $ go version
98 | go version go1.15.6 darwin/amd64
99 | $ go run . && go tool pprof -raw block.pb.gz
100 | PeriodType: contentions count
101 | Period: 1
102 | Time: 2021-02-05 15:27:01.371414 +0100 CET
103 | Samples:
104 | contentions/count delay/nanoseconds
105 | 23271 892063188: 1 2 3 4
106 | 22612 438270491: 1 2 5 4
107 | Locations
108 | 1: 0x10453af M=1 runtime.selectgo /usr/local/Cellar/go/1.15.6/libexec/src/runtime/select.go:511 s=0
109 | 2: 0x10cf56b M=1 main.simulateBlockEvents /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:66 s=0
110 | 3: 0x10cf8b2 M=1 main.eventB /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:52 s=0
111 | main.run.func2 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:28 s=0
112 | 4: 0x10cefd8 M=1 golang.org/x/sync/errgroup.(*Group).Go.func1 /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:57 s=0
113 | 5: 0x10cf852 M=1 main.eventA /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:48 s=0
114 | main.run.func1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:25 s=0
115 | Mappings
116 | 1: 0x0/0x0/0x0 [FN]
117 |
118 | After removing bias with this patch:
119 | https://github.com/felixge/go/commit/7c3f70c378d3f9a331a2079d17cd4cc420c70190
120 |
121 | $ sgo version
122 | go version devel +7c3f70c378 2021-02-05 15:19:48 +0100 darwin/amd64
123 | $ sgo run . && go tool pprof -raw block.pb.gz
124 | PeriodType: contentions count
125 | Period: 1
126 | Time: 2021-02-05 15:18:56.401244 +0100 CET
127 | Samples:
128 | contentions/count delay/nanoseconds
129 | 22833 931500628: 1 2 3 4
130 | 22453 902100036: 1 2 5 4
131 | 1 39999: 6 7 8 9 10
132 | Locations
133 | 1: 0x10471d9 M=1 runtime.selectgo /Users/felix.geisendoerfer/go/src/github.com/golang/go/src/runtime/select.go:492 s=0
134 | 2: 0x10d023e M=1 main.simulateBlockEvents /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:66 s=0
135 | 3: 0x10d0592 M=1 main.eventB /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:52 s=0
136 | main.run.func2 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:28 s=0
137 | 4: 0x10cfcd8 M=1 golang.org/x/sync/errgroup.(*Group).Go.func1 /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:57 s=0
138 | 5: 0x10d0532 M=1 main.eventA /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:48 s=0
139 | main.run.func1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:25 s=0
140 | 6: 0x107b024 M=1 sync.(*WaitGroup).Wait /Users/felix.geisendoerfer/go/src/github.com/golang/go/src/sync
141 | /waitgroup.go:130 s=0
142 | 7: 0x10cfb30 M=1 golang.org/x/sync/errgroup.(*Group).Wait /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:40 s=0
143 | 8: 0x10cff7a M=1 main.run /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:32 s=0
144 | 9: 0x10cfda5 M=1 main.main /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/block-sample/main.go:14 s=0
145 | 10: 0x10371b5 M=1 runtime.main /Users/felix.geisendoerfer/go/src/github.com/golang/go/src/runtime/proc.go:225 s=0
146 | Mappings
147 | 1: 0x0/0x0/0x0 [FN]
148 |
149 | */
150 |
--------------------------------------------------------------------------------
/examples/block-vs-mutex/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "runtime"
7 | "runtime/pprof"
8 | "sync"
9 | "time"
10 | )
11 |
12 | func main() {
13 | if err := run(); err != nil {
14 | fmt.Fprintln(os.Stderr, err)
15 | os.Exit(1)
16 | }
17 | }
18 |
19 | func run() error {
20 | runtime.SetBlockProfileRate(1)
21 | runtime.SetMutexProfileFraction(1)
22 |
23 | aquired := make(chan struct{})
24 | var m sync.Mutex
25 | m.Lock()
26 | go func() {
27 | <-aquired
28 | m.Lock()
29 | aquired <- struct{}{}
30 | }()
31 | aquired <- struct{}{}
32 | time.Sleep(time.Nanosecond)
33 | m.Unlock()
34 | <-aquired
35 |
36 | if err := writeProfile("block"); err != nil {
37 | return err
38 | } else if err := writeProfile("mutex"); err != nil {
39 | return err
40 | }
41 | return nil
42 | }
43 |
44 | func writeProfile(name string) error {
45 | f, err := os.Create(name + ".pb.gz")
46 | if err != nil {
47 | return err
48 | }
49 | defer f.Close()
50 |
51 | if err := pprof.Lookup(name).WriteTo(f, 0); err != nil {
52 | return err
53 | }
54 | return nil
55 | }
56 |
--------------------------------------------------------------------------------
/examples/block/main:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/block/main
--------------------------------------------------------------------------------
/examples/block/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "runtime"
6 | "runtime/pprof"
7 | "sync"
8 | "time"
9 | )
10 |
11 | var foo []string
12 |
13 | func main() {
14 | demonstrateSleep()
15 |
16 | f, err := os.Create("block.pb.gz")
17 | if err != nil {
18 | panic(err)
19 | }
20 | defer f.Close()
21 | if err := pprof.Lookup("block").WriteTo(f, 0); err != nil {
22 | panic(err)
23 | }
24 | }
25 |
26 | func demonstrateSleep() {
27 | runtime.SetBlockProfileRate(1)
28 | <-time.After(time.Millisecond)
29 | }
30 |
31 | func demonstrateSelect() {
32 | runtime.SetBlockProfileRate(1)
33 |
34 | ch1 := make(chan struct{}, 0)
35 | ch2 := make(chan struct{}, 1)
36 | ch3 := make(chan struct{}, 0)
37 |
38 | go func() {
39 | ch2 <- struct{}{}
40 | }()
41 |
42 | time.Sleep(20 * time.Millisecond)
43 |
44 | select {
45 | case <-ch1:
46 | case <-ch2:
47 | case <-ch3:
48 | }
49 | }
50 |
51 | func demonstrateSampling() {
52 | runtime.SetBlockProfileRate(int(40 * time.Microsecond.Nanoseconds()))
53 | for i := 0; i < 10000; i++ {
54 | blockMutex(10 * time.Microsecond)
55 | }
56 | }
57 |
58 | func blockMutex(d time.Duration) {
59 | m := &sync.Mutex{}
60 | m.Lock()
61 | go func() {
62 | spinSleep(d)
63 | m.Unlock()
64 | }()
65 | m.Lock()
66 | }
67 |
68 | // spinSleep is a more accurate version of time.Sleep() for short sleep
69 | // durations. Accuracy seems to be ~35ns.
70 | func spinSleep(d time.Duration) {
71 | start := time.Now()
72 | n := 0
73 | for time.Since(start) < d {
74 | n++
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/examples/callers-func-length/README.md:
--------------------------------------------------------------------------------
1 | # callers-func-length
2 |
3 | A simple benchmark that shows that the costs of stack unwinding in Go increase for larger functions. This is due to the way `gopclntab` based unwinding is implemented.
4 |
5 | ```
6 | go test -bench .
7 | goos: darwin
8 | goarch: amd64
9 | pkg: github.com/felixge/go-profiler-notes/examples/callers-func-length
10 | cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
11 | BenchmarkCallers/short-12 2623002 454.1 ns/op
12 | BenchmarkCallers/loop-12 2590384 466.8 ns/op
13 | BenchmarkCallers/long-12 638096 1862 ns/op
14 | ```
15 |
--------------------------------------------------------------------------------
/examples/callers-func-length/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/callers-func-length
2 |
3 | go 1.16
4 |
--------------------------------------------------------------------------------
/examples/callers-func-length/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | )
7 |
8 | func main() {
9 | if err := run(); err != nil {
10 | fmt.Fprintln(os.Stderr, err)
11 | os.Exit(1)
12 | }
13 | }
14 |
15 | func run() error {
16 | return nil
17 | }
18 |
--------------------------------------------------------------------------------
/examples/callers-func-length/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "runtime"
5 | "testing"
6 | )
7 |
8 | var callers = make([]uintptr, 32)
9 |
10 | func BenchmarkCallers(b *testing.B) {
11 | b.Run("short", func(b *testing.B) {
12 | short(func() {
13 | for i := 0; i < b.N; i++ {
14 | runtime.Callers(0, callers)
15 | }
16 | })
17 | })
18 |
19 | b.Run("loop", func(b *testing.B) {
20 | loop(func() {
21 | for i := 0; i < b.N; i++ {
22 | runtime.Callers(0, callers)
23 | }
24 | })
25 | })
26 |
27 | b.Run("long", func(b *testing.B) {
28 | long(func() {
29 | for i := 0; i < b.N; i++ {
30 | runtime.Callers(0, callers)
31 | }
32 | })
33 | })
34 | }
35 |
36 | func short(fn func()) {
37 | other()
38 | fn()
39 | }
40 |
41 | func loop(fn func()) {
42 | for i := 0; i < 1000; i++ {
43 | other()
44 | }
45 | fn()
46 | }
47 |
48 | func long(fn func()) {
49 | other()
50 | other()
51 | other()
52 | other()
53 | other()
54 | other()
55 | other()
56 | other()
57 | other()
58 | other()
59 | other()
60 | other()
61 | other()
62 | other()
63 | other()
64 | other()
65 | other()
66 | other()
67 | other()
68 | other()
69 | other()
70 | other()
71 | other()
72 | other()
73 | other()
74 | other()
75 | other()
76 | other()
77 | other()
78 | other()
79 | other()
80 | other()
81 | other()
82 | other()
83 | other()
84 | other()
85 | other()
86 | other()
87 | other()
88 | other()
89 | other()
90 | other()
91 | other()
92 | other()
93 | other()
94 | other()
95 | other()
96 | other()
97 | other()
98 | other()
99 | other()
100 | other()
101 | other()
102 | other()
103 | other()
104 | other()
105 | other()
106 | other()
107 | other()
108 | other()
109 | other()
110 | other()
111 | other()
112 | other()
113 | other()
114 | other()
115 | other()
116 | other()
117 | other()
118 | other()
119 | other()
120 | other()
121 | other()
122 | other()
123 | other()
124 | other()
125 | other()
126 | other()
127 | other()
128 | other()
129 | other()
130 | other()
131 | other()
132 | other()
133 | other()
134 | other()
135 | other()
136 | other()
137 | other()
138 | other()
139 | other()
140 | other()
141 | other()
142 | other()
143 | other()
144 | other()
145 | other()
146 | other()
147 | other()
148 | other()
149 | fn()
150 | }
151 |
152 | func other() int {
153 | m := map[int]string{}
154 | m[0] = "foo"
155 | m[100] = "bar"
156 | return len(m)
157 | }
158 |
--------------------------------------------------------------------------------
/examples/cpu/benchmark.cpu.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/cpu/benchmark.cpu.pb.gz
--------------------------------------------------------------------------------
/examples/cpu/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/cpu
2 |
3 | go 1.15
4 |
5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
6 |
--------------------------------------------------------------------------------
/examples/cpu/go.sum:
--------------------------------------------------------------------------------
1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
3 |
--------------------------------------------------------------------------------
/examples/cpu/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "log"
7 | "net/http"
8 | _ "net/http/pprof"
9 | "os"
10 | "time"
11 |
12 | "golang.org/x/sync/errgroup"
13 | )
14 |
15 | func main() {
16 | if err := run(); err != nil {
17 | fmt.Fprintln(os.Stderr, err)
18 | os.Exit(1)
19 | }
20 | }
21 |
22 | func run() error {
23 | g, _ := errgroup.WithContext(context.Background())
24 |
25 | g.Go(func() error {
26 | addr := "localhost:6060"
27 | log.Printf("Listening on %s", addr)
28 | return http.ListenAndServe(addr, nil)
29 | })
30 |
31 | g.Go(func() error { return computeSleepLoop(1000000, 10*time.Millisecond) })
32 |
33 | return g.Wait()
34 | }
35 |
36 | func computeSleepLoop(n int, sleep time.Duration) error {
37 | for {
38 | compute(n)
39 | time.Sleep(sleep)
40 | }
41 | return nil
42 | }
43 |
44 | func compute(n int) int64 {
45 | var sum int64
46 | for i := 0; i < n; i++ {
47 | sum += int64(i) / 2
48 | sum += int64(i) / 3
49 | sum += int64(i) / 4
50 | sum += int64(i) / 5
51 | sum += int64(i) / 6
52 | sum += int64(i) / 7
53 | sum += int64(i) / 8
54 | }
55 | return sum
56 | }
57 |
--------------------------------------------------------------------------------
/examples/cpu/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "testing"
5 | )
6 |
7 | func Benchmark_compute(b *testing.B) {
8 | compute(b.N)
9 | }
10 |
--------------------------------------------------------------------------------
/examples/cpu/pprof.samples.cpu.001.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/cpu/pprof.samples.cpu.001.pb.gz
--------------------------------------------------------------------------------
/examples/cpu/pprof.samples.cpu.001.pprof.txt:
--------------------------------------------------------------------------------
1 | PeriodType: cpu nanoseconds
2 | Period: 10000000
3 | Time: 2021-01-08 17:10:32.116825 +0100 CET
4 | Duration: 3.13
5 | Samples:
6 | samples/count cpu/nanoseconds
7 | 19 190000000: 1 2 3
8 | 5 50000000: 4 5 2 3
9 | 1 10000000: 6 7 8 9 10 11 12 13 14
10 | 1 10000000: 15 16 17 11 18 14
11 | 2 20000000: 6 7 8 9 10 11 18 14
12 | 7 70000000: 19 20 21 22 23 24 14
13 | 3 30000000: 25 26 27 28
14 | Locations
15 | 1: 0x1372f7f M=1 main.computeSum /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/cpu/main.go:39 s=0
16 | 2: 0x13730f2 M=1 main.run.func2 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/cpu/main.go:31 s=0
17 | 3: 0x1372cf8 M=1 golang.org/x/sync/errgroup.(*Group).Go.func1 /Users/felix.geisendoerfer/go/pkg/mod/golang.org/x/sync@v0.0.0-20201207232520-09787c993a3a/errgroup/errgroup.go:57 s=0
18 | 4: 0x10711c0 M=1 runtime.asyncPreempt /usr/local/Cellar/go/1.15.6/libexec/src/runtime/preempt_amd64.s:7 s=0
19 | 5: 0x1372f7e M=1 main.computeSum /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/cpu/main.go:37 s=0
20 | 6: 0x1058598 M=1 runtime.pthread_cond_wait /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:414 s=0
21 | 7: 0x10351ac M=1 runtime.semasleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/os_darwin.go:63 s=0
22 | 8: 0x100de06 M=1 runtime.notesleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/lock_sema.go:181 s=0
23 | 9: 0x103ee64 M=1 runtime.stopm /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1924 s=0
24 | 10: 0x104063e M=1 runtime.findrunnable /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2485 s=0
25 | 11: 0x10412b6 M=1 runtime.schedule /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2683 s=0
26 | 12: 0x1041a59 M=1 runtime.goschedImpl /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2866 s=0
27 | 13: 0x1041cd3 M=1 runtime.gopreempt_m /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2894 s=0
28 | 14: 0x106df3a M=1 runtime.mcall /usr/local/Cellar/go/1.15.6/libexec/src/runtime/asm_amd64.s:318 s=0
29 | 15: 0x1057ff7 M=1 runtime.nanotime1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:284 s=0
30 | 16: 0x1041704 M=1 runtime.nanotime /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time_nofake.go:19 s=0
31 | runtime.checkTimers /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2757 s=0
32 | 17: 0x103fd69 M=1 runtime.findrunnable /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2289 s=0
33 | 18: 0x104185c M=1 runtime.park_m /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2851 s=0
34 | 19: 0x1057f58 M=1 runtime.write1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:270 s=0
35 | 20: 0x1034be4 M=1 runtime.write /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time_nofake.go:30 s=0
36 | runtime.netpollBreak /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll_kqueue.go:89 s=0
37 | 21: 0x1040c37 M=1 runtime.wakeNetPoller /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2521 s=0
38 | 22: 0x10593e9 M=1 runtime.modtimer /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:480 s=0
39 | 23: 0x1058744 M=1 runtime.resettimer /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:529 s=0
40 | runtime.resetForSleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:197 s=0
41 | 24: 0x1041881 M=1 runtime.park_m /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2840 s=0
42 | 25: 0x1057f10 M=1 runtime.usleep /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:263 s=0
43 | 26: 0x1045cac M=1 runtime.sysmon /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:4660 s=0
44 | 27: 0x103d8a7 M=1 runtime.mstart1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1172 s=0
45 | 28: 0x103d7c5 M=1 runtime.mstart /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1137 s=0
46 | Mappings
47 | 1: 0x0/0x0/0x0 [FN]
48 |
--------------------------------------------------------------------------------
/examples/datadog/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/datadog
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/DataDog/datadog-go v4.2.0+incompatible // indirect
7 | github.com/google/uuid v1.1.4 // indirect
8 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0
9 | )
10 |
--------------------------------------------------------------------------------
/examples/datadog/go.sum:
--------------------------------------------------------------------------------
1 | github.com/DataDog/datadog-go v4.2.0+incompatible h1:Q73jzyKHwyA04Gf4SSukRF+KR4wJEimU6tAuU0B8Y4Y=
2 | github.com/DataDog/datadog-go v4.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
3 | github.com/google/uuid v1.1.4 h1:0ecGp3skIrHWPNGPJDaBIghfA6Sp7Ruo2Io8eLKzWm0=
4 | github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
5 | github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
6 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 h1:EmglUJuykRsTwsQDcKaAo3CmOunWU6Dqk7U2lo7Pjss=
7 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0/go.mod h1:Sp1lku8WJMvNV0kjDI4Ni/T7J/U3BO5ct5kEaoVU8+I=
8 |
--------------------------------------------------------------------------------
/examples/datadog/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | _ "net/http/pprof"
6 | "os"
7 | "time"
8 |
9 | "gopkg.in/DataDog/dd-trace-go.v1/profiler"
10 | )
11 |
12 | func main() {
13 | if err := run(); err != nil {
14 | fmt.Fprintln(os.Stderr, err)
15 | os.Exit(1)
16 | }
17 | }
18 |
19 | func run() error {
20 | if err := startProfiler(); err != nil {
21 | return err
22 | }
23 | defer profiler.Stop()
24 |
25 | for {
26 | time.Sleep(10 * time.Millisecond)
27 | }
28 | }
29 |
30 | func startProfiler() error {
31 | return profiler.Start(
32 | profiler.WithService("datadog-example"),
33 | profiler.WithEnv("localhost"),
34 | profiler.WithVersion("1.0"),
35 | profiler.WithProfileTypes(
36 | profiler.CPUProfile,
37 | profiler.HeapProfile,
38 | profiler.MutexProfile,
39 | profiler.GoroutineProfile,
40 | profiler.BlockProfile,
41 | ),
42 | )
43 | }
44 |
--------------------------------------------------------------------------------
/examples/delve-trace/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/delve-trace
2 |
3 | go 1.16
4 |
--------------------------------------------------------------------------------
/examples/delve-trace/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | )
7 |
8 | func main() {
9 | if err := run(); err != nil {
10 | fmt.Fprintln(os.Stderr, err)
11 | os.Exit(1)
12 | }
13 | }
14 |
15 | func run() error {
16 | for i := 0; i < 10; i++ {
17 | foo(i)
18 | }
19 | return nil
20 | }
21 |
22 | func foo(i int) int {
23 | return i * 2
24 | }
25 |
--------------------------------------------------------------------------------
/examples/frame-order/frames.txt:
--------------------------------------------------------------------------------
1 | goroutine 1 [running]:
2 | runtime/pprof.writeGoroutineStacks(0x111e660, 0xc0000a0018, 0x30, 0x12b3aa8)
3 | /usr/local/Cellar/go/1.15.7_1/libexec/src/runtime/pprof/pprof.go:693 +0x9f
4 | runtime/pprof.writeGoroutine(0x111e660, 0xc0000a0018, 0x2, 0x0, 0x602e6537)
5 | /usr/local/Cellar/go/1.15.7_1/libexec/src/runtime/pprof/pprof.go:682 +0x45
6 | runtime/pprof.(*Profile).WriteTo(0x11ae4e0, 0x111e660, 0xc0000a0018, 0x2, 0xc0000a0020, 0x0)
7 | /usr/local/Cellar/go/1.15.7_1/libexec/src/runtime/pprof/pprof.go:331 +0x3f2
8 | main.bar()
9 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/frame-order/main.go:19 +0xd5
10 | main.foo(...)
11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/frame-order/main.go:13
12 | main.main()
13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/frame-order/main.go:9 +0x25
14 |
--------------------------------------------------------------------------------
/examples/frame-order/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/frame-order
2 |
3 | go 1.15
4 |
--------------------------------------------------------------------------------
/examples/frame-order/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "runtime/pprof"
6 | )
7 |
8 | func main() {
9 | foo()
10 | }
11 |
12 | func foo() {
13 | bar()
14 | }
15 | func bar() {
16 | debug2, _ := os.Create("frames.txt")
17 | debug0, _ := os.Create("frames.pb.gz")
18 |
19 | pprof.Lookup("goroutine").WriteTo(debug2, 2)
20 | pprof.Lookup("goroutine").WriteTo(debug0, 0)
21 | }
22 |
--------------------------------------------------------------------------------
/examples/goroutine/1.net.http.pprof.goroutine.debug0.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/1.net.http.pprof.goroutine.debug0.pb.gz
--------------------------------------------------------------------------------
/examples/goroutine/1.net.http.pprof.goroutine.debug1.txt:
--------------------------------------------------------------------------------
1 | goroutine profile: total 10
2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81
3 | # labels: {"test_label":"test_value"}
4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165
6 |
7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81
8 | # labels: {"test_label":"test_value"}
9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177
10 |
11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12ff937 0x11707c5 0x117092f 0x13005e8 0x106fd81
12 | # labels: {"test_label":"test_value"}
13 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
14 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
15 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
16 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159
17 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55
18 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182
19 | # 0x12ff936 net/http.(*persistConn).Read+0x76 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887
20 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101
21 | # 0x117092e bufio.(*Reader).Peek+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139
22 | # 0x13005e7 net/http.(*persistConn).readLoop+0x1a7 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040
23 |
24 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81
25 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
26 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
27 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
28 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394
29 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172
30 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139
31 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261
32 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937
33 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866
34 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
35 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123
36 |
37 | 1 @ 0x103b125 0x104afcf 0x130223c 0x106fd81
38 | # labels: {"test_label":"test_value"}
39 | # 0x130223b net/http.(*persistConn).writeLoop+0x11b /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340
40 |
41 | 1 @ 0x103b125 0x104afcf 0x1302f59 0x12f74a5 0x12dd435 0x129e853 0x129e23f 0x12a025f 0x129fbbe 0x129fbce 0x13abba5 0x13abb8a 0x13acafe 0x13abe67 0x13ac247 0x103ad29 0x106fd81
42 | # labels: {"test_label":"test_value"}
43 | # 0x1302f58 net/http.(*persistConn).roundTrip+0x778 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2565
44 | # 0x12f74a4 net/http.(*Transport).roundTrip+0xa64 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:582
45 | # 0x12dd434 net/http.(*Transport).RoundTrip+0x34 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/roundtrip.go:17
46 | # 0x129e852 net/http.send+0x452 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:252
47 | # 0x129e23e net/http.(*Client).send+0xfe /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:176
48 | # 0x12a025e net/http.(*Client).do+0x45e /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:718
49 | # 0x129fbbd net/http.(*Client).Do+0xbd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:586
50 | # 0x129fbcd net/http.(*Client).Get+0xcd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:475
51 | # 0x13abba4 net/http.Get+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:447
52 | # 0x13abb89 main.writeHttpProfile+0xe9 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:92
53 | # 0x13acafd main.glob..func7+0x3d /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:79
54 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106
55 | # 0x13ac246 main.main+0x2c6 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142
56 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204
57 |
58 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81
59 | # labels: {"test_label":"test_value"}
60 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
61 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171
62 |
63 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13a9865 0x13ab145 0x12e6424 0x12e834d 0x12e9a23 0x12e522d 0x106fd81
64 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716
65 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724
66 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684
67 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331
68 | # 0x13a9864 net/http/pprof.handler.ServeHTTP+0x384 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:256
69 | # 0x13ab144 net/http/pprof.Index+0x944 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:367
70 | # 0x12e6423 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2042
71 | # 0x12e834c net/http.(*ServeMux).ServeHTTP+0x1ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2417
72 | # 0x12e9a22 net/http.serverHandler.ServeHTTP+0xa2 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2843
73 | # 0x12e522c net/http.(*conn).serve+0x8ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1925
74 |
75 | 1 @ 0x12def61 0x106fd81
76 | # 0x12def60 net/http.(*connReader).backgroundRead+0x0 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:689
77 |
78 |
--------------------------------------------------------------------------------
/examples/goroutine/1.net.http.pprof.goroutine.debug2.txt:
--------------------------------------------------------------------------------
1 | goroutine 41 [running]:
2 | runtime/pprof.writeGoroutineStacks(0x14e5f60, 0xc0001d6000, 0xc0001b6270, 0x0)
3 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:693 +0x9f
4 | runtime/pprof.writeGoroutine(0x14e5f60, 0xc0001d6000, 0x2, 0x1714f40, 0xc0001ba420)
5 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:682 +0x45
6 | runtime/pprof.(*Profile).WriteTo(0x17179e0, 0x14e5f60, 0xc0001d6000, 0x2, 0xc0001d6000, 0xc0003379d8)
7 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 +0x3f2
8 | net/http/pprof.handler.ServeHTTP(0xc0001ca071, 0x9, 0x14ec9a0, 0xc0001d6000, 0xc000110300)
9 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:256 +0x385
10 | net/http/pprof.Index(0x14ec9a0, 0xc0001d6000, 0xc000110300)
11 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:367 +0x945
12 | net/http.HandlerFunc.ServeHTTP(0x1486d90, 0x14ec9a0, 0xc0001d6000, 0xc000110300)
13 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2042 +0x44
14 | net/http.(*ServeMux).ServeHTTP(0x1725ba0, 0x14ec9a0, 0xc0001d6000, 0xc000110300)
15 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2417 +0x1ad
16 | net/http.serverHandler.ServeHTTP(0xc00019c000, 0x14ec9a0, 0xc0001d6000, 0xc000110300)
17 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2843 +0xa3
18 | net/http.(*conn).serve(0xc0000c6320, 0x14ed4a0, 0xc000322000)
19 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1925 +0x8ad
20 | created by net/http.(*Server).Serve
21 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2969 +0x36c
22 |
23 | goroutine 1 [select]:
24 | net/http.(*persistConn).roundTrip(0xc0000cea20, 0xc0001b40c0, 0x0, 0x0, 0x0)
25 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2565 +0x779
26 | net/http.(*Transport).roundTrip(0x171d020, 0xc0001ce000, 0x30, 0x30, 0x180de98)
27 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:582 +0xa65
28 | net/http.(*Transport).RoundTrip(0x171d020, 0xc0001ce000, 0x171d020, 0x0, 0x0)
29 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/roundtrip.go:17 +0x35
30 | net/http.send(0xc0001ce000, 0x14e5d80, 0x171d020, 0x0, 0x0, 0x0, 0xc0001c8018, 0x203000, 0x1, 0x0)
31 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:252 +0x453
32 | net/http.(*Client).send(0x17259e0, 0xc0001ce000, 0x0, 0x0, 0x0, 0xc0001c8018, 0x0, 0x1, 0xf8)
33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:176 +0xff
34 | net/http.(*Client).do(0x17259e0, 0xc0001ce000, 0x0, 0x0, 0x0)
35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:718 +0x45f
36 | net/http.(*Client).Do(...)
37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:586
38 | net/http.(*Client).Get(0x17259e0, 0xc0001cc000, 0x33, 0x2, 0x2, 0xc0001cc000)
39 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:475 +0xbe
40 | net/http.Get(...)
41 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:447
42 | main.writeHttpProfile(0x14e5940, 0xc0001b6090, 0x2, 0x0, 0x0)
43 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:92 +0x105
44 | main.glob..func8(0x14e5940, 0xc0001b6090, 0xc000213eb0, 0x2)
45 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:85 +0x3e
46 | main.writeProfiles(0x1, 0xc0000c4008, 0x146641d)
47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187
48 | main.main()
49 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 +0x2c7
50 |
51 | goroutine 22 [sleep]:
52 | time.Sleep(0x3b9aca00)
53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
54 | main.shortSleepLoop()
55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
56 | created by main.indirectShortSleepLoop2
57 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35
58 |
59 | goroutine 3 [IO wait]:
60 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0)
61 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
62 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786)
63 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
64 | internal/poll.(*pollDesc).waitRead(...)
65 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
66 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
67 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc
68 | net.(*netFD).accept(0xc00019e000, 0x7d667d63cbbded3e, 0x1789ccbbded3e, 0x100000001)
69 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45
70 | net.(*TCPListener).accept(0xc000188060, 0x60006709, 0xc000196da8, 0x109abe6)
71 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32
72 | net.(*TCPListener).Accept(0xc000188060, 0xc000196df8, 0x18, 0xc000001200, 0x12e9eec)
73 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65
74 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0)
75 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266
76 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536)
77 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7
78 | net/http.ListenAndServe(...)
79 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
80 | main.main.func1(0xc000032120)
81 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126
82 | created by main.main
83 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5
84 |
85 | goroutine 4 [sleep]:
86 | time.Sleep(0x3b9aca00)
87 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
88 | main.shortSleepLoop()
89 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
90 | created by main.main
91 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195
92 |
93 | goroutine 5 [sleep]:
94 | time.Sleep(0x34630b8a000)
95 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
96 | main.sleepLoop(0x34630b8a000)
97 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b
98 | created by main.main
99 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc
100 |
101 | goroutine 6 [chan receive]:
102 | main.chanReceiveForever()
103 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d
104 | created by main.main
105 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4
106 |
107 | goroutine 24 [select]:
108 | net/http.(*persistConn).writeLoop(0xc0000cea20)
109 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 +0x11c
110 | created by net/http.(*Transport).dialConn
111 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1709 +0xcdc
112 |
113 | goroutine 23 [IO wait]:
114 | internal/poll.runtime_pollWait(0x1e91da0, 0x72, 0x14e6ca0)
115 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
116 | internal/poll.(*pollDesc).wait(0xc00010e198, 0x72, 0x14e6c00, 0x16db878, 0x0)
117 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
118 | internal/poll.(*pollDesc).waitRead(...)
119 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
120 | internal/poll.(*FD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
121 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5
122 | net.(*netFD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x103b1dc, 0xc000199b58, 0x10680e0)
123 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f
124 | net.(*conn).Read(0xc000010008, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
125 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e
126 | net/http.(*persistConn).Read(0xc0000cea20, 0xc000256000, 0x1000, 0x1000, 0xc00009e300, 0xc000199c58, 0x10074b5)
127 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 +0x77
128 | bufio.(*Reader).fill(0xc0001801e0)
129 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105
130 | bufio.(*Reader).Peek(0xc0001801e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc000030180)
131 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 +0x4f
132 | net/http.(*persistConn).readLoop(0xc0000cea20)
133 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 +0x1a8
134 | created by net/http.(*Transport).dialConn
135 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1708 +0xcb7
136 |
137 | goroutine 67 [IO wait]:
138 | internal/poll.runtime_pollWait(0x1e91cb8, 0x72, 0x14e6ca0)
139 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
140 | internal/poll.(*pollDesc).wait(0xc00019e098, 0x72, 0x14e6c00, 0x16db878, 0x0)
141 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
142 | internal/poll.(*pollDesc).waitRead(...)
143 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
144 | internal/poll.(*FD).Read(0xc00019e080, 0xc00007c311, 0x1, 0x1, 0x0, 0x0, 0x0)
145 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5
146 | net.(*netFD).Read(0xc00019e080, 0xc00007c311, 0x1, 0x1, 0x0, 0x0, 0x0)
147 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f
148 | net.(*conn).Read(0xc000186028, 0xc00007c311, 0x1, 0x1, 0x0, 0x0, 0x0)
149 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e
150 | net/http.(*connReader).backgroundRead(0xc00007c300)
151 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:690 +0x58
152 | created by net/http.(*connReader).startBackgroundRead
153 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:686 +0xd5
154 |
--------------------------------------------------------------------------------
/examples/goroutine/1.pprof.lookup.goroutine.debug0.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/1.pprof.lookup.goroutine.debug0.pb.gz
--------------------------------------------------------------------------------
/examples/goroutine/1.pprof.lookup.goroutine.debug1.txt:
--------------------------------------------------------------------------------
1 | goroutine profile: total 6
2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81
3 | # labels: {"test_label":"test_value"}
4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165
6 |
7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81
8 | # labels: {"test_label":"test_value"}
9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177
10 |
11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81
12 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
13 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
14 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
15 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394
16 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172
17 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139
18 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261
19 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937
20 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866
21 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
22 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123
23 |
24 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81
25 | # labels: {"test_label":"test_value"}
26 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
27 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171
28 |
29 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13ac965 0x13abe67 0x13ac247 0x103ad29 0x106fd81
30 | # labels: {"test_label":"test_value"}
31 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716
32 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724
33 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684
34 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331
35 | # 0x13ac964 main.glob..func4+0x64 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:60
36 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106
37 | # 0x13ac246 main.main+0x2c6 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142
38 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204
39 |
40 |
--------------------------------------------------------------------------------
/examples/goroutine/1.pprof.lookup.goroutine.debug2.txt:
--------------------------------------------------------------------------------
1 | goroutine 1 [running]:
2 | runtime/pprof.writeGoroutineStacks(0x14e5940, 0xc0000abb00, 0x101b8a5, 0xc000213d20)
3 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:693 +0x9f
4 | runtime/pprof.writeGoroutine(0x14e5940, 0xc0000abb00, 0x2, 0xc000213dc0, 0x10ee5c8)
5 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:682 +0x45
6 | runtime/pprof.(*Profile).WriteTo(0x17179e0, 0x14e5940, 0xc0000abb00, 0x2, 0xc00002c210, 0xc0002041a0)
7 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 +0x3f2
8 | main.glob..func5(0x14e5940, 0xc0000abb00, 0xc000213eb0, 0x2)
9 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:67 +0x65
10 | main.writeProfiles(0x1, 0xc0000c4008, 0x146641d)
11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187
12 | main.main()
13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 +0x2c7
14 |
15 | goroutine 22 [sleep]:
16 | time.Sleep(0x3b9aca00)
17 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
18 | main.shortSleepLoop()
19 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
20 | created by main.indirectShortSleepLoop2
21 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35
22 |
23 | goroutine 3 [IO wait]:
24 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0)
25 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
26 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786)
27 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
28 | internal/poll.(*pollDesc).waitRead(...)
29 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
30 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
31 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc
32 | net.(*netFD).accept(0xc00019e000, 0xc0001822d0, 0x1010038, 0xc000088000)
33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45
34 | net.(*TCPListener).accept(0xc000188060, 0xc000196da8, 0x1010038, 0x30)
35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32
36 | net.(*TCPListener).Accept(0xc000188060, 0x1436c40, 0xc0001822d0, 0x13f4620, 0x1714f50)
37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65
38 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0)
39 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266
40 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536)
41 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7
42 | net/http.ListenAndServe(...)
43 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
44 | main.main.func1(0xc000032120)
45 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126
46 | created by main.main
47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5
48 |
49 | goroutine 4 [sleep]:
50 | time.Sleep(0x3b9aca00)
51 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
52 | main.shortSleepLoop()
53 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
54 | created by main.main
55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195
56 |
57 | goroutine 5 [sleep]:
58 | time.Sleep(0x34630b8a000)
59 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
60 | main.sleepLoop(0x34630b8a000)
61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b
62 | created by main.main
63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc
64 |
65 | goroutine 6 [chan receive]:
66 | main.chanReceiveForever()
67 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d
68 | created by main.main
69 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4
70 |
--------------------------------------------------------------------------------
/examples/goroutine/1.runtime.goroutineprofile.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "Stack0": [
4 | 20629256,
5 | 20629212,
6 | 20627047,
7 | 20628039,
8 | 17018153,
9 | 17235329,
10 | 0,
11 | 0,
12 | 0,
13 | 0,
14 | 0,
15 | 0,
16 | 0,
17 | 0,
18 | 0,
19 | 0,
20 | 0,
21 | 0,
22 | 0,
23 | 0,
24 | 0,
25 | 0,
26 | 0,
27 | 0,
28 | 0,
29 | 0,
30 | 0,
31 | 0,
32 | 0,
33 | 0,
34 | 0,
35 | 0
36 | ]
37 | },
38 | {
39 | "Stack0": [
40 | 17019173,
41 | 17222943,
42 | 20628554,
43 | 17235329,
44 | 0,
45 | 0,
46 | 0,
47 | 0,
48 | 0,
49 | 0,
50 | 0,
51 | 0,
52 | 0,
53 | 0,
54 | 0,
55 | 0,
56 | 0,
57 | 0,
58 | 0,
59 | 0,
60 | 0,
61 | 0,
62 | 0,
63 | 0,
64 | 0,
65 | 0,
66 | 0,
67 | 0,
68 | 0,
69 | 0,
70 | 0,
71 | 0
72 | ]
73 | },
74 | {
75 | "Stack0": [
76 | 17019173,
77 | 16990811,
78 | 17211861,
79 | 17662341,
80 | 17673596,
81 | 17673566,
82 | 18587653,
83 | 18694226,
84 | 18689765,
85 | 19832294,
86 | 19831575,
87 | 20630758,
88 | 20630672,
89 | 17235329,
90 | 0,
91 | 0,
92 | 0,
93 | 0,
94 | 0,
95 | 0,
96 | 0,
97 | 0,
98 | 0,
99 | 0,
100 | 0,
101 | 0,
102 | 0,
103 | 0,
104 | 0,
105 | 0,
106 | 0,
107 | 0
108 | ]
109 | },
110 | {
111 | "Stack0": [
112 | 17019173,
113 | 17222943,
114 | 20628554,
115 | 17235329,
116 | 0,
117 | 0,
118 | 0,
119 | 0,
120 | 0,
121 | 0,
122 | 0,
123 | 0,
124 | 0,
125 | 0,
126 | 0,
127 | 0,
128 | 0,
129 | 0,
130 | 0,
131 | 0,
132 | 0,
133 | 0,
134 | 0,
135 | 0,
136 | 0,
137 | 0,
138 | 0,
139 | 0,
140 | 0,
141 | 0,
142 | 0,
143 | 0
144 | ]
145 | },
146 | {
147 | "Stack0": [
148 | 17019173,
149 | 17222943,
150 | 20628619,
151 | 17235329,
152 | 0,
153 | 0,
154 | 0,
155 | 0,
156 | 0,
157 | 0,
158 | 0,
159 | 0,
160 | 0,
161 | 0,
162 | 0,
163 | 0,
164 | 0,
165 | 0,
166 | 0,
167 | 0,
168 | 0,
169 | 0,
170 | 0,
171 | 0,
172 | 0,
173 | 0,
174 | 0,
175 | 0,
176 | 0,
177 | 0,
178 | 0,
179 | 0
180 | ]
181 | },
182 | {
183 | "Stack0": [
184 | 17019173,
185 | 16810991,
186 | 16810027,
187 | 20628717,
188 | 17235329,
189 | 0,
190 | 0,
191 | 0,
192 | 0,
193 | 0,
194 | 0,
195 | 0,
196 | 0,
197 | 0,
198 | 0,
199 | 0,
200 | 0,
201 | 0,
202 | 0,
203 | 0,
204 | 0,
205 | 0,
206 | 0,
207 | 0,
208 | 0,
209 | 0,
210 | 0,
211 | 0,
212 | 0,
213 | 0,
214 | 0,
215 | 0
216 | ]
217 | }
218 | ]
219 |
--------------------------------------------------------------------------------
/examples/goroutine/1.runtime.stack.txt:
--------------------------------------------------------------------------------
1 | goroutine 1 [running]:
2 | main.glob..func1(0x14e5940, 0xc0000aa7b0, 0xc000064eb0, 0x2)
3 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:29 +0x6f
4 | main.writeProfiles(0x1, 0xc0000c4008, 0x146641d)
5 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187
6 | main.main()
7 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:142 +0x2c7
8 |
9 | goroutine 22 [sleep]:
10 | time.Sleep(0x3b9aca00)
11 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
12 | main.shortSleepLoop()
13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
14 | created by main.indirectShortSleepLoop2
15 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35
16 |
17 | goroutine 3 [IO wait]:
18 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0)
19 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
20 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786)
21 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
22 | internal/poll.(*pollDesc).waitRead(...)
23 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
24 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
25 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc
26 | net.(*netFD).accept(0xc00019e000, 0xc0001822d0, 0x1010038, 0xc000088000)
27 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45
28 | net.(*TCPListener).accept(0xc000188060, 0xc000196da8, 0x1010038, 0x30)
29 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32
30 | net.(*TCPListener).Accept(0xc000188060, 0x1436c40, 0xc0001822d0, 0x13f4620, 0x1714f50)
31 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65
32 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0)
33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266
34 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536)
35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7
36 | net/http.ListenAndServe(...)
37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
38 | main.main.func1(0xc000032120)
39 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126
40 | created by main.main
41 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5
42 |
43 | goroutine 4 [sleep]:
44 | time.Sleep(0x3b9aca00)
45 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
46 | main.shortSleepLoop()
47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
48 | created by main.main
49 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195
50 |
51 | goroutine 5 [sleep]:
52 | time.Sleep(0x34630b8a000)
53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
54 | main.sleepLoop(0x34630b8a000)
55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b
56 | created by main.main
57 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc
58 |
59 | goroutine 6 [chan receive]:
60 | main.chanReceiveForever()
61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d
62 | created by main.main
63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4
64 |
--------------------------------------------------------------------------------
/examples/goroutine/2.net.http.pprof.goroutine.debug0.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/2.net.http.pprof.goroutine.debug0.pb.gz
--------------------------------------------------------------------------------
/examples/goroutine/2.net.http.pprof.goroutine.debug1.txt:
--------------------------------------------------------------------------------
1 | goroutine profile: total 10
2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81
3 | # labels: {"test_label":"test_value"}
4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165
6 |
7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81
8 | # labels: {"test_label":"test_value"}
9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177
10 |
11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12ff937 0x11707c5 0x117092f 0x13005e8 0x106fd81
12 | # labels: {"test_label":"test_value"}
13 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
14 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
15 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
16 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159
17 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55
18 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182
19 | # 0x12ff936 net/http.(*persistConn).Read+0x76 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887
20 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101
21 | # 0x117092e bufio.(*Reader).Peek+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139
22 | # 0x13005e7 net/http.(*persistConn).readLoop+0x1a7 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040
23 |
24 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81
25 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
26 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
27 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
28 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394
29 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172
30 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139
31 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261
32 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937
33 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866
34 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
35 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123
36 |
37 | 1 @ 0x103b125 0x104afcf 0x130223c 0x106fd81
38 | # labels: {"test_label":"test_value"}
39 | # 0x130223b net/http.(*persistConn).writeLoop+0x11b /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340
40 |
41 | 1 @ 0x103b125 0x104afcf 0x1302f59 0x12f74a5 0x12dd435 0x129e853 0x129e23f 0x12a025f 0x129fbbe 0x129fbce 0x13abba5 0x13abb8a 0x13acafe 0x13abe67 0x13ac352 0x103ad29 0x106fd81
42 | # labels: {"test_label":"test_value"}
43 | # 0x1302f58 net/http.(*persistConn).roundTrip+0x778 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2565
44 | # 0x12f74a4 net/http.(*Transport).roundTrip+0xa64 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:582
45 | # 0x12dd434 net/http.(*Transport).RoundTrip+0x34 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/roundtrip.go:17
46 | # 0x129e852 net/http.send+0x452 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:252
47 | # 0x129e23e net/http.(*Client).send+0xfe /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:176
48 | # 0x12a025e net/http.(*Client).do+0x45e /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:718
49 | # 0x129fbbd net/http.(*Client).Do+0xbd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:586
50 | # 0x129fbcd net/http.(*Client).Get+0xcd /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:475
51 | # 0x13abba4 net/http.Get+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/client.go:447
52 | # 0x13abb89 main.writeHttpProfile+0xe9 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:92
53 | # 0x13acafd main.glob..func7+0x3d /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:79
54 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106
55 | # 0x13ac351 main.main+0x3d1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152
56 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204
57 |
58 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81
59 | # labels: {"test_label":"test_value"}
60 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
61 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171
62 |
63 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13a9865 0x13ab145 0x12e6424 0x12e834d 0x12e9a23 0x12e522d 0x106fd81
64 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716
65 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724
66 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684
67 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331
68 | # 0x13a9864 net/http/pprof.handler.ServeHTTP+0x384 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:256
69 | # 0x13ab144 net/http/pprof.Index+0x944 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/pprof/pprof.go:367
70 | # 0x12e6423 net/http.HandlerFunc.ServeHTTP+0x43 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2042
71 | # 0x12e834c net/http.(*ServeMux).ServeHTTP+0x1ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2417
72 | # 0x12e9a22 net/http.serverHandler.ServeHTTP+0xa2 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2843
73 | # 0x12e522c net/http.(*conn).serve+0x8ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1925
74 |
75 | 1 @ 0x12def61 0x106fd81
76 | # 0x12def60 net/http.(*connReader).backgroundRead+0x0 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:689
77 |
78 |
--------------------------------------------------------------------------------
/examples/goroutine/2.pprof.lookup.goroutine.debug0.pb.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/goroutine/2.pprof.lookup.goroutine.debug0.pb.gz
--------------------------------------------------------------------------------
/examples/goroutine/2.pprof.lookup.goroutine.debug1.txt:
--------------------------------------------------------------------------------
1 | goroutine profile: total 9
2 | 2 @ 0x103b125 0x106cd1f 0x13ac44a 0x106fd81
3 | # labels: {"test_label":"test_value"}
4 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
5 | # 0x13ac449 main.shortSleepLoop+0x29 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165
6 |
7 | 1 @ 0x103b125 0x10083ef 0x100802b 0x13ac4ed 0x106fd81
8 | # labels: {"test_label":"test_value"}
9 | # 0x13ac4ec main.chanReceiveForever+0x4c /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177
10 |
11 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12df52d 0x11707c5 0x117151d 0x1171754 0x1263c2c 0x12d96ca 0x12d96f9 0x12e09ba 0x12e5085 0x106fd81
12 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
13 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
14 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
15 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159
16 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55
17 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182
18 | # 0x12df52c net/http.(*connReader).Read+0x1ac /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:798
19 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101
20 | # 0x117151c bufio.(*Reader).ReadSlice+0x3c /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:360
21 | # 0x1171753 bufio.(*Reader).ReadLine+0x33 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:389
22 | # 0x1263c2b net/textproto.(*Reader).readLineSlice+0x6b /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:58
23 | # 0x12d96c9 net/textproto.(*Reader).ReadLine+0xa9 /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:39
24 | # 0x12d96f8 net/http.readRequest+0xd8 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/request.go:1012
25 | # 0x12e09b9 net/http.(*conn).readRequest+0x199 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:984
26 | # 0x12e5084 net/http.(*conn).serve+0x704 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1851
27 |
28 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10d91c5 0x10d91a3 0x11b8a8f 0x11cb72e 0x12ff937 0x11707c5 0x117092f 0x13005e8 0x106fd81
29 | # labels: {"test_label":"test_value"}
30 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
31 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
32 | # 0x10d91c4 internal/poll.(*pollDesc).waitRead+0x1a4 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
33 | # 0x10d91a2 internal/poll.(*FD).Read+0x182 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159
34 | # 0x11b8a8e net.(*netFD).Read+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55
35 | # 0x11cb72d net.(*conn).Read+0x8d /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182
36 | # 0x12ff936 net/http.(*persistConn).Read+0x76 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887
37 | # 0x11707c4 bufio.(*Reader).fill+0x104 /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101
38 | # 0x117092e bufio.(*Reader).Peek+0x4e /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139
39 | # 0x13005e7 net/http.(*persistConn).readLoop+0x1a7 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040
40 |
41 | 1 @ 0x103b125 0x103425b 0x106a1d5 0x10d8185 0x10dad7c 0x10dad5e 0x11ba005 0x11d4052 0x11d2ee5 0x12e9de6 0x12e9b17 0x13acce6 0x13acc90 0x106fd81
42 | # 0x106a1d4 internal/poll.runtime_pollWait+0x54 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222
43 | # 0x10d8184 internal/poll.(*pollDesc).wait+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87
44 | # 0x10dad7b internal/poll.(*pollDesc).waitRead+0x1fb /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
45 | # 0x10dad5d internal/poll.(*FD).Accept+0x1dd /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394
46 | # 0x11ba004 net.(*netFD).accept+0x44 /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172
47 | # 0x11d4051 net.(*TCPListener).accept+0x31 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139
48 | # 0x11d2ee4 net.(*TCPListener).Accept+0x64 /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261
49 | # 0x12e9de5 net/http.(*Server).Serve+0x265 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937
50 | # 0x12e9b16 net/http.(*Server).ListenAndServe+0xb6 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866
51 | # 0x13acce5 net/http.ListenAndServe+0x125 /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
52 | # 0x13acc8f main.main.func1+0xcf /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123
53 |
54 | 1 @ 0x103b125 0x104afcf 0x130223c 0x106fd81
55 | # labels: {"test_label":"test_value"}
56 | # 0x130223b net/http.(*persistConn).writeLoop+0x11b /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340
57 |
58 | 1 @ 0x103b125 0x106cd1f 0x13ac48b 0x106fd81
59 | # labels: {"test_label":"test_value"}
60 | # 0x106cd1e time.Sleep+0xbe /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188
61 | # 0x13ac48a main.sleepLoop+0x2a /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171
62 |
63 | 1 @ 0x1069e1d 0x139efc2 0x139ed85 0x139b952 0x13ac965 0x13abe67 0x13ac352 0x103ad29 0x106fd81
64 | # labels: {"test_label":"test_value"}
65 | # 0x1069e1c runtime/pprof.runtime_goroutineProfileWithLabels+0x5c /usr/local/Cellar/go/1.15.6/libexec/src/runtime/mprof.go:716
66 | # 0x139efc1 runtime/pprof.writeRuntimeProfile+0xe1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:724
67 | # 0x139ed84 runtime/pprof.writeGoroutine+0xa4 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:684
68 | # 0x139b951 runtime/pprof.(*Profile).WriteTo+0x3f1 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331
69 | # 0x13ac964 main.glob..func4+0x64 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:60
70 | # 0x13abe66 main.writeProfiles+0x186 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106
71 | # 0x13ac351 main.main+0x3d1 /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152
72 | # 0x103ad28 runtime.main+0x208 /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:204
73 |
74 |
--------------------------------------------------------------------------------
/examples/goroutine/2.pprof.lookup.goroutine.debug2.txt:
--------------------------------------------------------------------------------
1 | goroutine 1 [running]:
2 | runtime/pprof.writeGoroutineStacks(0x14e5940, 0xc0000abc50, 0x101b8a5, 0xc000333d20)
3 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:693 +0x9f
4 | runtime/pprof.writeGoroutine(0x14e5940, 0xc0000abc50, 0x2, 0xc000333dc0, 0x10ee5c8)
5 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:682 +0x45
6 | runtime/pprof.(*Profile).WriteTo(0x17179e0, 0x14e5940, 0xc0000abc50, 0x2, 0xc00002c210, 0xc0000ac4e0)
7 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/pprof/pprof.go:331 +0x3f2
8 | main.glob..func5(0x14e5940, 0xc0000abc50, 0xc000333eb0, 0x2)
9 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:67 +0x65
10 | main.writeProfiles(0x2, 0xc0000c4008, 0x1466424)
11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187
12 | main.main()
13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152 +0x3d2
14 |
15 | goroutine 22 [sleep, 1 minutes]:
16 | time.Sleep(0x3b9aca00)
17 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
18 | main.shortSleepLoop()
19 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
20 | created by main.indirectShortSleepLoop2
21 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35
22 |
23 | goroutine 3 [IO wait, 1 minutes]:
24 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0)
25 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
26 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786)
27 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
28 | internal/poll.(*pollDesc).waitRead(...)
29 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
30 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
31 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc
32 | net.(*netFD).accept(0xc00019e000, 0x7d667d63cbbded3e, 0x1789ccbbded3e, 0x100000001)
33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45
34 | net.(*TCPListener).accept(0xc000188060, 0x60006709, 0xc000196da8, 0x109abe6)
35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32
36 | net.(*TCPListener).Accept(0xc000188060, 0xc000196df8, 0x18, 0xc000001200, 0x12e9eec)
37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65
38 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0)
39 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266
40 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536)
41 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7
42 | net/http.ListenAndServe(...)
43 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
44 | main.main.func1(0xc000032120)
45 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126
46 | created by main.main
47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5
48 |
49 | goroutine 4 [sleep, 1 minutes]:
50 | time.Sleep(0x3b9aca00)
51 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
52 | main.shortSleepLoop()
53 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
54 | created by main.main
55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195
56 |
57 | goroutine 5 [sleep, 1 minutes]:
58 | time.Sleep(0x34630b8a000)
59 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
60 | main.sleepLoop(0x34630b8a000)
61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b
62 | created by main.main
63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc
64 |
65 | goroutine 6 [chan receive, 1 minutes]:
66 | main.chanReceiveForever()
67 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d
68 | created by main.main
69 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4
70 |
71 | goroutine 24 [select, 1 minutes]:
72 | net/http.(*persistConn).writeLoop(0xc0000cea20)
73 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 +0x11c
74 | created by net/http.(*Transport).dialConn
75 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1709 +0xcdc
76 |
77 | goroutine 23 [IO wait, 1 minutes]:
78 | internal/poll.runtime_pollWait(0x1e91da0, 0x72, 0x14e6ca0)
79 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
80 | internal/poll.(*pollDesc).wait(0xc00010e198, 0x72, 0x14e6c00, 0x16db878, 0x0)
81 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
82 | internal/poll.(*pollDesc).waitRead(...)
83 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
84 | internal/poll.(*FD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
85 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5
86 | net.(*netFD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x103b1dc, 0xc000199b58, 0x10680e0)
87 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f
88 | net.(*conn).Read(0xc000010008, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
89 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e
90 | net/http.(*persistConn).Read(0xc0000cea20, 0xc000256000, 0x1000, 0x1000, 0xc00009e300, 0xc000199c58, 0x10074b5)
91 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 +0x77
92 | bufio.(*Reader).fill(0xc0001801e0)
93 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105
94 | bufio.(*Reader).Peek(0xc0001801e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0001d0060)
95 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 +0x4f
96 | net/http.(*persistConn).readLoop(0xc0000cea20)
97 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 +0x1a8
98 | created by net/http.(*Transport).dialConn
99 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1708 +0xcb7
100 |
101 | goroutine 41 [IO wait, 1 minutes]:
102 | internal/poll.runtime_pollWait(0x1e91cb8, 0x72, 0x14e6ca0)
103 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
104 | internal/poll.(*pollDesc).wait(0xc00019e098, 0x72, 0x14e6c00, 0x16db878, 0x0)
105 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
106 | internal/poll.(*pollDesc).waitRead(...)
107 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
108 | internal/poll.(*FD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
109 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5
110 | net.(*netFD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x203000, 0x203000, 0x203000)
111 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f
112 | net.(*conn).Read(0xc000186028, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
113 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e
114 | net/http.(*connReader).Read(0xc00007c300, 0xc000326000, 0x1000, 0x1000, 0x100000006, 0x10, 0x1819408)
115 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:798 +0x1ad
116 | bufio.(*Reader).fill(0xc000290060)
117 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105
118 | bufio.(*Reader).ReadSlice(0xc000290060, 0xa, 0x1819408, 0xc000337988, 0x100f6d0, 0xc000110000, 0x100)
119 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:360 +0x3d
120 | bufio.(*Reader).ReadLine(0xc000290060, 0xc000110000, 0x1079694, 0xc0001a4000, 0x0, 0x1010038, 0x30)
121 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:389 +0x34
122 | net/textproto.(*Reader).readLineSlice(0xc000182300, 0xc000110000, 0x10d7c4d, 0xc00019e080, 0x1068000, 0xc000282900)
123 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:58 +0x6c
124 | net/textproto.(*Reader).ReadLine(...)
125 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:39
126 | net/http.readRequest(0xc000290060, 0x0, 0xc000110000, 0x0, 0x0)
127 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/request.go:1012 +0xaa
128 | net/http.(*conn).readRequest(0xc0000c6320, 0x14ed4a0, 0xc000322000, 0x0, 0x0, 0x0)
129 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:984 +0x19a
130 | net/http.(*conn).serve(0xc0000c6320, 0x14ed4a0, 0xc000322000)
131 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1851 +0x705
132 | created by net/http.(*Server).Serve
133 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2969 +0x36c
134 |
--------------------------------------------------------------------------------
/examples/goroutine/2.runtime.goroutineprofile.json:
--------------------------------------------------------------------------------
1 | [
2 | {
3 | "Stack0": [
4 | 20629256,
5 | 20629212,
6 | 20627047,
7 | 20628306,
8 | 17018153,
9 | 17235329,
10 | 0,
11 | 0,
12 | 0,
13 | 0,
14 | 0,
15 | 0,
16 | 0,
17 | 0,
18 | 0,
19 | 0,
20 | 0,
21 | 0,
22 | 0,
23 | 0,
24 | 0,
25 | 0,
26 | 0,
27 | 0,
28 | 0,
29 | 0,
30 | 0,
31 | 0,
32 | 0,
33 | 0,
34 | 0,
35 | 0
36 | ]
37 | },
38 | {
39 | "Stack0": [
40 | 17019173,
41 | 17222943,
42 | 20628554,
43 | 17235329,
44 | 0,
45 | 0,
46 | 0,
47 | 0,
48 | 0,
49 | 0,
50 | 0,
51 | 0,
52 | 0,
53 | 0,
54 | 0,
55 | 0,
56 | 0,
57 | 0,
58 | 0,
59 | 0,
60 | 0,
61 | 0,
62 | 0,
63 | 0,
64 | 0,
65 | 0,
66 | 0,
67 | 0,
68 | 0,
69 | 0,
70 | 0,
71 | 0
72 | ]
73 | },
74 | {
75 | "Stack0": [
76 | 17019173,
77 | 16990811,
78 | 17211861,
79 | 17662341,
80 | 17673596,
81 | 17673566,
82 | 18587653,
83 | 18694226,
84 | 18689765,
85 | 19832294,
86 | 19831575,
87 | 20630758,
88 | 20630672,
89 | 17235329,
90 | 0,
91 | 0,
92 | 0,
93 | 0,
94 | 0,
95 | 0,
96 | 0,
97 | 0,
98 | 0,
99 | 0,
100 | 0,
101 | 0,
102 | 0,
103 | 0,
104 | 0,
105 | 0,
106 | 0,
107 | 0
108 | ]
109 | },
110 | {
111 | "Stack0": [
112 | 17019173,
113 | 17222943,
114 | 20628554,
115 | 17235329,
116 | 0,
117 | 0,
118 | 0,
119 | 0,
120 | 0,
121 | 0,
122 | 0,
123 | 0,
124 | 0,
125 | 0,
126 | 0,
127 | 0,
128 | 0,
129 | 0,
130 | 0,
131 | 0,
132 | 0,
133 | 0,
134 | 0,
135 | 0,
136 | 0,
137 | 0,
138 | 0,
139 | 0,
140 | 0,
141 | 0,
142 | 0,
143 | 0
144 | ]
145 | },
146 | {
147 | "Stack0": [
148 | 17019173,
149 | 17222943,
150 | 20628619,
151 | 17235329,
152 | 0,
153 | 0,
154 | 0,
155 | 0,
156 | 0,
157 | 0,
158 | 0,
159 | 0,
160 | 0,
161 | 0,
162 | 0,
163 | 0,
164 | 0,
165 | 0,
166 | 0,
167 | 0,
168 | 0,
169 | 0,
170 | 0,
171 | 0,
172 | 0,
173 | 0,
174 | 0,
175 | 0,
176 | 0,
177 | 0,
178 | 0,
179 | 0
180 | ]
181 | },
182 | {
183 | "Stack0": [
184 | 17019173,
185 | 16810991,
186 | 16810027,
187 | 20628717,
188 | 17235329,
189 | 0,
190 | 0,
191 | 0,
192 | 0,
193 | 0,
194 | 0,
195 | 0,
196 | 0,
197 | 0,
198 | 0,
199 | 0,
200 | 0,
201 | 0,
202 | 0,
203 | 0,
204 | 0,
205 | 0,
206 | 0,
207 | 0,
208 | 0,
209 | 0,
210 | 0,
211 | 0,
212 | 0,
213 | 0,
214 | 0,
215 | 0
216 | ]
217 | },
218 | {
219 | "Stack0": [
220 | 17019173,
221 | 17084367,
222 | 19931708,
223 | 17235329,
224 | 0,
225 | 0,
226 | 0,
227 | 0,
228 | 0,
229 | 0,
230 | 0,
231 | 0,
232 | 0,
233 | 0,
234 | 0,
235 | 0,
236 | 0,
237 | 0,
238 | 0,
239 | 0,
240 | 0,
241 | 0,
242 | 0,
243 | 0,
244 | 0,
245 | 0,
246 | 0,
247 | 0,
248 | 0,
249 | 0,
250 | 0,
251 | 0
252 | ]
253 | },
254 | {
255 | "Stack0": [
256 | 17019173,
257 | 16990811,
258 | 17211861,
259 | 17662341,
260 | 17666501,
261 | 17666467,
262 | 18582159,
263 | 18659118,
264 | 19921207,
265 | 18286533,
266 | 18286895,
267 | 19924456,
268 | 17235329,
269 | 0,
270 | 0,
271 | 0,
272 | 0,
273 | 0,
274 | 0,
275 | 0,
276 | 0,
277 | 0,
278 | 0,
279 | 0,
280 | 0,
281 | 0,
282 | 0,
283 | 0,
284 | 0,
285 | 0,
286 | 0,
287 | 0
288 | ]
289 | },
290 | {
291 | "Stack0": [
292 | 17019173,
293 | 16990811,
294 | 17211861,
295 | 17662341,
296 | 17666501,
297 | 17666467,
298 | 18582159,
299 | 18659118,
300 | 19789101,
301 | 18286533,
302 | 18289949,
303 | 18290516,
304 | 19282988,
305 | 19764938,
306 | 19764985,
307 | 19794362,
308 | 19812485,
309 | 17235329,
310 | 0,
311 | 0,
312 | 0,
313 | 0,
314 | 0,
315 | 0,
316 | 0,
317 | 0,
318 | 0,
319 | 0,
320 | 0,
321 | 0,
322 | 0,
323 | 0
324 | ]
325 | }
326 | ]
327 |
--------------------------------------------------------------------------------
/examples/goroutine/2.runtime.stack.txt:
--------------------------------------------------------------------------------
1 | goroutine 1 [running]:
2 | main.glob..func1(0x14e5940, 0xc0000aa7b0, 0xc000064eb0, 0x2)
3 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:29 +0x6f
4 | main.writeProfiles(0x2, 0xc0000c4008, 0x1466424)
5 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:106 +0x187
6 | main.main()
7 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:152 +0x3d2
8 |
9 | goroutine 22 [sleep, 1 minutes]:
10 | time.Sleep(0x3b9aca00)
11 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
12 | main.shortSleepLoop()
13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
14 | created by main.indirectShortSleepLoop2
15 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:185 +0x35
16 |
17 | goroutine 3 [IO wait, 1 minutes]:
18 | internal/poll.runtime_pollWait(0x1e91e88, 0x72, 0x0)
19 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
20 | internal/poll.(*pollDesc).wait(0xc00019e018, 0x72, 0x0, 0x0, 0x1465786)
21 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
22 | internal/poll.(*pollDesc).waitRead(...)
23 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
24 | internal/poll.(*FD).Accept(0xc00019e000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
25 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:394 +0x1fc
26 | net.(*netFD).accept(0xc00019e000, 0x7d667d63cbbded3e, 0x1789ccbbded3e, 0x100000001)
27 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_unix.go:172 +0x45
28 | net.(*TCPListener).accept(0xc000188060, 0x60006709, 0xc000196da8, 0x109abe6)
29 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock_posix.go:139 +0x32
30 | net.(*TCPListener).Accept(0xc000188060, 0xc000196df8, 0x18, 0xc000001200, 0x12e9eec)
31 | /usr/local/Cellar/go/1.15.6/libexec/src/net/tcpsock.go:261 +0x65
32 | net/http.(*Server).Serve(0xc00019c000, 0x14ec6e0, 0xc000188060, 0x0, 0x0)
33 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2937 +0x266
34 | net/http.(*Server).ListenAndServe(0xc00019c000, 0xc00019c000, 0x1475536)
35 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2866 +0xb7
36 | net/http.ListenAndServe(...)
37 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:3120
38 | main.main.func1(0xc000032120)
39 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:123 +0x126
40 | created by main.main
41 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:121 +0xc5
42 |
43 | goroutine 4 [sleep, 1 minutes]:
44 | time.Sleep(0x3b9aca00)
45 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
46 | main.shortSleepLoop()
47 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:165 +0x2a
48 | created by main.main
49 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:130 +0x195
50 |
51 | goroutine 5 [sleep, 1 minutes]:
52 | time.Sleep(0x34630b8a000)
53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
54 | main.sleepLoop(0x34630b8a000)
55 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:171 +0x2b
56 | created by main.main
57 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:131 +0x1bc
58 |
59 | goroutine 6 [chan receive, 1 minutes]:
60 | main.chanReceiveForever()
61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:177 +0x4d
62 | created by main.main
63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/goroutine/main.go:132 +0x1d4
64 |
65 | goroutine 24 [select, 1 minutes]:
66 | net/http.(*persistConn).writeLoop(0xc0000cea20)
67 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2340 +0x11c
68 | created by net/http.(*Transport).dialConn
69 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1709 +0xcdc
70 |
71 | goroutine 23 [IO wait, 1 minutes]:
72 | internal/poll.runtime_pollWait(0x1e91da0, 0x72, 0x14e6ca0)
73 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
74 | internal/poll.(*pollDesc).wait(0xc00010e198, 0x72, 0x14e6c00, 0x16db878, 0x0)
75 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
76 | internal/poll.(*pollDesc).waitRead(...)
77 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
78 | internal/poll.(*FD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
79 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5
80 | net.(*netFD).Read(0xc00010e180, 0xc000256000, 0x1000, 0x1000, 0x103b1dc, 0xc000199b58, 0x10680e0)
81 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f
82 | net.(*conn).Read(0xc000010008, 0xc000256000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
83 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e
84 | net/http.(*persistConn).Read(0xc0000cea20, 0xc000256000, 0x1000, 0x1000, 0xc00009e300, 0xc000199c58, 0x10074b5)
85 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1887 +0x77
86 | bufio.(*Reader).fill(0xc0001801e0)
87 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105
88 | bufio.(*Reader).Peek(0xc0001801e0, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc0001d0060)
89 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:139 +0x4f
90 | net/http.(*persistConn).readLoop(0xc0000cea20)
91 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:2040 +0x1a8
92 | created by net/http.(*Transport).dialConn
93 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/transport.go:1708 +0xcb7
94 |
95 | goroutine 41 [IO wait, 1 minutes]:
96 | internal/poll.runtime_pollWait(0x1e91cb8, 0x72, 0x14e6ca0)
97 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/netpoll.go:222 +0x55
98 | internal/poll.(*pollDesc).wait(0xc00019e098, 0x72, 0x14e6c00, 0x16db878, 0x0)
99 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
100 | internal/poll.(*pollDesc).waitRead(...)
101 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_poll_runtime.go:92
102 | internal/poll.(*FD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
103 | /usr/local/Cellar/go/1.15.6/libexec/src/internal/poll/fd_unix.go:159 +0x1a5
104 | net.(*netFD).Read(0xc00019e080, 0xc000326000, 0x1000, 0x1000, 0x203000, 0x203000, 0x203000)
105 | /usr/local/Cellar/go/1.15.6/libexec/src/net/fd_posix.go:55 +0x4f
106 | net.(*conn).Read(0xc000186028, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
107 | /usr/local/Cellar/go/1.15.6/libexec/src/net/net.go:182 +0x8e
108 | net/http.(*connReader).Read(0xc00007c300, 0xc000326000, 0x1000, 0x1000, 0x100000006, 0x10, 0x1819408)
109 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:798 +0x1ad
110 | bufio.(*Reader).fill(0xc000290060)
111 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:101 +0x105
112 | bufio.(*Reader).ReadSlice(0xc000290060, 0xa, 0x1819408, 0xc000337988, 0x100f6d0, 0xc000110000, 0x100)
113 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:360 +0x3d
114 | bufio.(*Reader).ReadLine(0xc000290060, 0xc000110000, 0x1079694, 0xc0001a4000, 0x0, 0x1010038, 0x30)
115 | /usr/local/Cellar/go/1.15.6/libexec/src/bufio/bufio.go:389 +0x34
116 | net/textproto.(*Reader).readLineSlice(0xc000182300, 0xc000110000, 0x10d7c4d, 0xc00019e080, 0x1068000, 0xc000282900)
117 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:58 +0x6c
118 | net/textproto.(*Reader).ReadLine(...)
119 | /usr/local/Cellar/go/1.15.6/libexec/src/net/textproto/reader.go:39
120 | net/http.readRequest(0xc000290060, 0x0, 0xc000110000, 0x0, 0x0)
121 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/request.go:1012 +0xaa
122 | net/http.(*conn).readRequest(0xc0000c6320, 0x14ed4a0, 0xc000322000, 0x0, 0x0, 0x0)
123 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:984 +0x19a
124 | net/http.(*conn).serve(0xc0000c6320, 0x14ed4a0, 0xc000322000)
125 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:1851 +0x705
126 | created by net/http.(*Server).Serve
127 | /usr/local/Cellar/go/1.15.6/libexec/src/net/http/server.go:2969 +0x36c
128 |
--------------------------------------------------------------------------------
/examples/goroutine/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/goroutine
2 |
3 | go 1.16
4 |
--------------------------------------------------------------------------------
/examples/goroutine/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "bytes"
5 | "context"
6 | "encoding/json"
7 | "errors"
8 | "flag"
9 | "fmt"
10 | "io"
11 | "io/ioutil"
12 | "net/http"
13 | _ "net/http/pprof"
14 | "runtime"
15 | "runtime/pprof"
16 | "time"
17 | )
18 |
19 | type Profile struct {
20 | Name string
21 | WriteTo func(w io.Writer) error
22 | }
23 |
24 | var profiles = []Profile{
25 | {
26 | Name: "runtime.stack.txt",
27 | WriteTo: func(w io.Writer) error {
28 | buf := make([]byte, 1024*1024)
29 | n := runtime.Stack(buf, true)
30 | buf = buf[:n]
31 | _, err := w.Write(buf)
32 | return err
33 | },
34 | },
35 | {
36 | Name: "runtime.goroutineprofile.json",
37 | WriteTo: func(w io.Writer) error {
38 | p := make([]runtime.StackRecord, 1000)
39 | n, ok := runtime.GoroutineProfile(p)
40 | if !ok {
41 | return errors.New("runtime.GoroutineProfile: not ok")
42 | }
43 | p = p[0:n]
44 | e := json.NewEncoder(w)
45 | e.SetIndent("", " ")
46 | return e.Encode(p)
47 | },
48 | },
49 | {
50 | Name: "pprof.lookup.goroutine.debug0.pb.gz",
51 | WriteTo: func(w io.Writer) error {
52 | profile := pprof.Lookup("goroutine")
53 | return profile.WriteTo(w, 0)
54 | },
55 | },
56 | {
57 | Name: "pprof.lookup.goroutine.debug1.txt",
58 | WriteTo: func(w io.Writer) error {
59 | profile := pprof.Lookup("goroutine")
60 | return profile.WriteTo(w, 1)
61 | },
62 | },
63 | {
64 | Name: "pprof.lookup.goroutine.debug2.txt",
65 | WriteTo: func(w io.Writer) error {
66 | profile := pprof.Lookup("goroutine")
67 | return profile.WriteTo(w, 2)
68 | },
69 | },
70 | {
71 | Name: "net.http.pprof.goroutine.debug0.pb.gz",
72 | WriteTo: func(w io.Writer) error {
73 | return writeHttpProfile(w, 0)
74 | },
75 | },
76 | {
77 | Name: "net.http.pprof.goroutine.debug1.txt",
78 | WriteTo: func(w io.Writer) error {
79 | return writeHttpProfile(w, 1)
80 | },
81 | },
82 | {
83 | Name: "net.http.pprof.goroutine.debug2.txt",
84 | WriteTo: func(w io.Writer) error {
85 | return writeHttpProfile(w, 2)
86 | },
87 | },
88 | }
89 |
90 | func writeHttpProfile(w io.Writer, debug int) error {
91 | url := fmt.Sprintf("http://%s/debug/pprof/goroutine?debug=%d", listenAddr, debug)
92 | res, err := http.Get(url)
93 | if err != nil {
94 | return err
95 | }
96 | defer res.Body.Close()
97 |
98 | _, err = io.Copy(w, res.Body)
99 | return err
100 | }
101 |
102 | func writeProfiles(n int) error {
103 | for _, profile := range profiles {
104 | buf := &bytes.Buffer{}
105 | filename := fmt.Sprintf("%d.%s", n, profile.Name)
106 | if err := profile.WriteTo(buf); err != nil {
107 | return err
108 | } else if err := ioutil.WriteFile(filename, buf.Bytes(), 0666); err != nil {
109 | return err
110 | }
111 | }
112 | return nil
113 | }
114 |
115 | var listenAddr = "127.0.0.1:8080"
116 |
117 | func main() {
118 | flag.Parse()
119 |
120 | errCh := make(chan error, 1)
121 | go func() {
122 | fmt.Printf("Listening for pprof requests on %s\n", listenAddr)
123 | errCh <- http.ListenAndServe(listenAddr, nil)
124 | }()
125 |
126 | labels := pprof.Labels("test_label", "test_value")
127 | ctx := pprof.WithLabels(context.Background(), labels)
128 | pprof.SetGoroutineLabels(ctx)
129 |
130 | go shortSleepLoop()
131 | go sleepLoop(time.Hour)
132 | go chanReceiveForever()
133 | go indirectShortSleepLoop()
134 |
135 | sleep := time.Second
136 | fmt.Printf("Sleeping for %s followed by gc\n", sleep)
137 |
138 | time.Sleep(time.Second)
139 | runtime.GC()
140 |
141 | fmt.Printf("Dump 1\n")
142 | if err := writeProfiles(1); err != nil {
143 | panic(err)
144 | }
145 |
146 | sleep = 1*time.Minute + 10*time.Second
147 | fmt.Printf("Sleeping for %s followed by gc\n", sleep)
148 | time.Sleep(sleep)
149 | runtime.GC()
150 |
151 | fmt.Printf("Dump 2\n")
152 | if err := writeProfiles(2); err != nil {
153 | panic(err)
154 | }
155 |
156 | fmt.Printf("Waiting forever\n")
157 |
158 | //fmt.Printf("waiting indefinitely so you can press ctrl+\\ to compare the output\n")
159 | //runtime.GC()
160 | <-errCh
161 | }
162 |
163 | func shortSleepLoop() {
164 | for {
165 | time.Sleep(time.Second)
166 | }
167 | }
168 |
169 | func sleepLoop(d time.Duration) {
170 | for {
171 | time.Sleep(d)
172 | }
173 | }
174 |
175 | func chanReceiveForever() {
176 | forever := make(chan struct{})
177 | <-forever
178 | }
179 |
180 | func indirectShortSleepLoop() {
181 | indirectShortSleepLoop2()
182 | }
183 |
184 | func indirectShortSleepLoop2() {
185 | go shortSleepLoop()
186 | }
187 |
--------------------------------------------------------------------------------
/examples/goroutine/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "runtime"
6 | "testing"
7 | "time"
8 | )
9 |
10 | func BenchmarkProfilerGoroutines(b *testing.B) {
11 | max := 1024 * 1024
12 | buf := make([]runtime.StackRecord, max*2)
13 | for g := 1; g <= max; g = g * 2 {
14 | g := g
15 | name := fmt.Sprintf("%d goroutines", g)
16 |
17 | b.Run(name, func(b *testing.B) {
18 | initalRoutines := runtime.NumGoroutine()
19 |
20 | readyCh := make(chan struct{})
21 | stopCh := make(chan struct{})
22 | for i := 0; i < g; i++ {
23 | go atStackDepth(16, func() {
24 | defer func() { stopCh <- struct{}{} }()
25 | readyCh <- struct{}{}
26 | })
27 | <-readyCh
28 | }
29 |
30 | b.ResetTimer()
31 | for i := 0; i < b.N; i++ {
32 | n, ok := runtime.GoroutineProfile(buf)
33 | if !ok {
34 | b.Logf("GoroutineProfile not ok")
35 | } else if gotRoutines := n - initalRoutines; gotRoutines != g {
36 | b.Logf("want %d goroutines, but got %d on iteration %d", g, gotRoutines, i)
37 | }
38 | }
39 | b.StopTimer()
40 | for i := 0; i < g; i++ {
41 | <-stopCh
42 | }
43 | start := time.Now()
44 | for i := 0; ; i++ {
45 | if runtime.NumGoroutine() == initalRoutines {
46 | break
47 | }
48 | time.Sleep(20 * time.Millisecond)
49 | if time.Since(start) > 10*time.Second {
50 | b.Fatalf("%d goroutines still running, want %d", runtime.NumGoroutine(), initalRoutines)
51 | }
52 | }
53 | })
54 | }
55 | }
56 |
57 | func atStackDepth(depth int, fn func()) {
58 | pcs := make([]uintptr, depth+10)
59 | n := runtime.Callers(1, pcs)
60 | if n > depth {
61 | panic("depth exceeded")
62 | } else if n < depth {
63 | atStackDepth(depth, fn)
64 | return
65 | }
66 |
67 | fn()
68 | }
69 |
--------------------------------------------------------------------------------
/examples/guide/main:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/guide/main
--------------------------------------------------------------------------------
/examples/guide/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "time"
5 | )
6 |
7 | func main() {
8 | start := time.Now()
9 | for time.Since(start) < time.Second {
10 | }
11 | //time.Sleep(time.Second)
12 | }
13 |
--------------------------------------------------------------------------------
/examples/guide/main2.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "os"
5 | "runtime/trace"
6 | "time"
7 | )
8 |
9 | func main() {
10 | trace.Start(os.Stdout)
11 | defer trace.Stop()
12 |
13 | go b()
14 | a()
15 | }
16 |
17 | func a() {
18 | start := time.Now()
19 | for time.Since(start) < time.Second {
20 | }
21 | }
22 |
23 | func b() {
24 | start := time.Now()
25 | for time.Since(start) < time.Second {
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/examples/guide/main3.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "net/http"
6 | "os"
7 | "runtime/trace"
8 | )
9 |
10 | func main() {
11 | trace.Start(os.Stderr)
12 | defer trace.Stop()
13 |
14 | res, err := http.Get("https://example.org/")
15 | if err != nil {
16 | panic(err)
17 | }
18 | fmt.Printf("%d\n", res.StatusCode)
19 | }
20 |
--------------------------------------------------------------------------------
/examples/guide/trace.bin:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/examples/guide/trace.bin
--------------------------------------------------------------------------------
/examples/memory/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/memory
2 |
3 | go 1.15
4 |
5 | require golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
6 |
--------------------------------------------------------------------------------
/examples/memory/go.sum:
--------------------------------------------------------------------------------
1 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs=
2 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
3 |
--------------------------------------------------------------------------------
/examples/memory/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "fmt"
6 | "log"
7 | "net/http"
8 | _ "net/http/pprof"
9 | "os"
10 | "runtime"
11 | "time"
12 |
13 | "golang.org/x/sync/errgroup"
14 | )
15 |
16 | func main() {
17 | if err := run(); err != nil {
18 | fmt.Fprintln(os.Stderr, err)
19 | os.Exit(1)
20 | }
21 | }
22 |
23 | func run() error {
24 | runtime.MemProfileRate = 1
25 |
26 | g, _ := errgroup.WithContext(context.Background())
27 |
28 | g.Go(func() error {
29 | addr := "localhost:6060"
30 | log.Printf("Listening on %s", addr)
31 | return http.ListenAndServe(addr, nil)
32 | })
33 |
34 | //g.Go(allocStuff)
35 |
36 | // Memory leak that leaks 1 kB / sec at a rate of 10 Hz.
37 | g.Go(func() error { return leakStuff(1024, 100*time.Millisecond) })
38 | g.Go(func() error { return allocStuff(1024, 100*time.Millisecond) })
39 | g.Go(func() error { return forceGc(time.Second) })
40 |
41 | return g.Wait()
42 | }
43 |
44 | var leak []*Data
45 |
46 | // leakStuff leaks ~bytes every interval.
47 | func leakStuff(bytes int, interval time.Duration) error {
48 | for {
49 | leak = append(leak, newData(bytes))
50 | time.Sleep(interval)
51 | }
52 | }
53 |
54 | // allocStuff is allocating things but not leaking them.
55 | func allocStuff(bytes int, interval time.Duration) error {
56 | for {
57 | newData(bytes)
58 | time.Sleep(interval)
59 | }
60 | }
61 |
62 | // forceGc forces a GC to occur every interval.
63 | func forceGc(interval time.Duration) error {
64 | for {
65 | time.Sleep(interval)
66 | runtime.GC()
67 | }
68 | }
69 |
70 | func newData(size int) *Data {
71 | return &Data{data: make([]byte, size)}
72 | }
73 |
74 | type Data struct {
75 | data []byte
76 | }
77 |
--------------------------------------------------------------------------------
/examples/pclnttab/darwin.go:
--------------------------------------------------------------------------------
1 | //go:build darwin
2 | // +build darwin
3 |
4 | package main
5 |
6 | import (
7 | "debug/gosym"
8 | "debug/macho"
9 | "os"
10 | )
11 |
12 | // from https://github.com/lizrice/debugger-from-scratch/blob/master/symbols.go
13 | func goSymTable() (*gosym.Table, error) {
14 | exe, err := macho.Open(os.Args[0])
15 | if err != nil {
16 | return nil, nil
17 | }
18 | defer exe.Close()
19 |
20 | addr := exe.Section("__text").Addr
21 |
22 | lineTableData, err := exe.Section("__gopclntab").Data()
23 | if err != nil {
24 | return nil, nil
25 | }
26 | lineTable := gosym.NewLineTable(lineTableData, addr)
27 | if err != nil {
28 | return nil, nil
29 | }
30 |
31 | symTableData, err := exe.Section("__gosymtab").Data()
32 | if err != nil {
33 | return nil, nil
34 | }
35 |
36 | return gosym.NewTable(symTableData, lineTable)
37 | }
38 |
--------------------------------------------------------------------------------
/examples/pclnttab/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/pclnttab
2 |
3 | go 1.16
4 |
--------------------------------------------------------------------------------
/examples/pclnttab/linux.go:
--------------------------------------------------------------------------------
1 | //go:build linux
2 | // +build linux
3 |
4 | package main
5 |
6 | import (
7 | "debug/elf"
8 | "errors"
9 | "fmt"
10 | "os"
11 | )
12 |
13 | // from https://github.com/lizrice/debugger-from-scratch/blob/master/symbols.go
14 | func goSymTable() (*gosym.Table, error) {
15 | exe, err := elf.Open(os.Args[0])
16 | if err != nil {
17 | return nil, nil
18 | }
19 | defer exe.Close()
20 |
21 | addr := exe.Section(".text").Addr
22 |
23 | lineTableData, err := exe.Section(".gopclntab").Data()
24 | if err != nil {
25 | return nil, nil
26 | }
27 | lineTable := gosym.NewLineTable(lineTableData, addr)
28 | if err != nil {
29 | return nil, nil
30 | }
31 |
32 | symTableData, err := exe.Section(".gosymtab").Data()
33 | if err != nil {
34 | return nil, nil
35 | }
36 |
37 | return gosym.NewTable(symTableData, lineTable)
38 | }
39 |
--------------------------------------------------------------------------------
/examples/pclnttab/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "runtime"
7 | )
8 |
9 | func main() {
10 | if err := run(); err != nil {
11 | fmt.Fprintln(os.Stderr, err)
12 | os.Exit(1)
13 | }
14 | }
15 |
16 | func run() error {
17 | symTable, err := goSymTable()
18 | if err != nil {
19 | return err
20 | }
21 | for _, pc := range callers() {
22 | file, line, fn := symTable.PCToLine(uint64(pc))
23 | fmt.Printf("%x: %s() %s:%d\n", pc, fn.Name, file, line)
24 | }
25 | return nil
26 | }
27 |
28 | func callers() []uintptr {
29 | pcs := make([]uintptr, 32)
30 | n := runtime.Callers(2, pcs)
31 | pcs = pcs[0:n]
32 | return pcs
33 | }
34 |
--------------------------------------------------------------------------------
/examples/presentation/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | )
6 |
7 | func main() {
8 | foo()
9 | foobar()
10 | }
11 |
12 | func foo() {
13 | bar()
14 | }
15 |
16 | func bar() {
17 | fmt.Printf("Hello from bar\n")
18 | panic("oh no")
19 | }
20 |
21 | func foobar() {
22 | fmt.Printf("Hello from foobar\n")
23 | }
24 |
--------------------------------------------------------------------------------
/examples/presentation/main.simple.txt:
--------------------------------------------------------------------------------
1 | 0x10a46a0 mov rcx, qword ptr gs:[0x30]
2 | 0x10a46a9 cmp rsp, qword ptr [rcx+0x10]
3 | 0x10a46ad jbe 0x10a46d3
4 | 0x10a46af sub rsp, 0x8
5 | 0x10a46b3 mov qword ptr [rsp], rbp
6 | 0x10a46b7 lea rbp, ptr [rsp]
7 | 0x10a46bb nop dword ptr [rax+rax*1], eax
8 | 0x10a46c0 call $main.foo
9 | 0x10a46c5 call $main.foobar
10 | 0x10a46ca mov rbp, qword ptr [rsp]
11 | 0x10a46ce add rsp, 0x8
12 | 0x10a46d2 ret
13 | 0x10a46d3 call $runtime.morestack_noctxt
14 | 0x10a46d8 jmp $main.main
15 | 0x10a46e0 mov rcx, qword ptr gs:[0x30]
16 | 0x10a46e9 cmp rsp, qword ptr [rcx+0x10]
17 | 0x10a46ed jbe 0x10a470e
18 | 0x10a46ef sub rsp, 0x8
19 | 0x10a46f3 mov qword ptr [rsp], rbp
20 | 0x10a46f7 lea rbp, ptr [rsp]
21 | 0x10a46fb nop dword ptr [rax+rax*1], eax
22 | 0x10a4700 call $main.bar
23 | 0x10a4705 mov rbp, qword ptr [rsp]
24 | 0x10a4709 add rsp, 0x8
25 | 0x10a470d ret
26 | 0x10a470e call $runtime.morestack_noctxt
27 | 0x10a4713 jmp $main.foo
28 | 0x10a4720 mov rcx, qword ptr gs:[0x30]
29 | 0x10a4729 cmp rsp, qword ptr [rcx+0x10]
30 | 0x10a472d jbe 0x10a4771
31 | 0x10a472f sub rsp, 0x48
32 | 0x10a4733 mov qword ptr [rsp+0x40], rbp
33 | 0x10a4738 lea rbp, ptr [rsp+0x40]
34 | 0x10a473d lea rax, ptr [rip+0x25c23]
35 | 0x10a4744 mov qword ptr [rsp], rax
36 | 0x10a4748 mov qword ptr [rsp+0x8], 0xf
37 | 0x10a4751 xorps xmm0, xmm0
38 | 0x10a4754 movups xmmword ptr [rsp+0x10], xmm0
39 | 0x10a4759 mov qword ptr [rsp+0x20], 0x0
40 | 0x10a4762 call $fmt.Printf
41 | 0x10a4767 mov rbp, qword ptr [rsp+0x40]
42 | 0x10a476c add rsp, 0x48
43 | 0x10a4770 ret
44 | 0x10a4771 call $runtime.morestack_noctxt
45 | 0x10a4776 jmp $main.bar
46 | 0x10a4780 mov rcx, qword ptr gs:[0x30]
47 | 0x10a4789 cmp rsp, qword ptr [rcx+0x10]
48 | 0x10a478d jbe 0x10a47d1
49 | 0x10a478f sub rsp, 0x48
50 | 0x10a4793 mov qword ptr [rsp+0x40], rbp
51 | 0x10a4798 lea rbp, ptr [rsp+0x40]
52 | 0x10a479d lea rax, ptr [rip+0x263ed]
53 | 0x10a47a4 mov qword ptr [rsp], rax
54 | 0x10a47a8 mov qword ptr [rsp+0x8], 0x12
55 | 0x10a47b1 xorps xmm0, xmm0
56 | 0x10a47b4 movups xmmword ptr [rsp+0x10], xmm0
57 | 0x10a47b9 mov qword ptr [rsp+0x20], 0x0
58 | 0x10a47c2 call $fmt.Printf
59 | 0x10a47c7 mov rbp, qword ptr [rsp+0x40]
60 | 0x10a47cc add rsp, 0x48
61 | 0x10a47d0 ret
62 | 0x10a47d1 call $runtime.morestack_noctxt
63 | 0x10a47d6 jmp $main.foobar
64 |
--------------------------------------------------------------------------------
/examples/presentation/main.txt:
--------------------------------------------------------------------------------
1 | (dlv) disassemble -l main.main
2 | TEXT main.main(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go
3 | main.go:7 0x10a46a0 65488b0c2530000000 mov rcx, qword ptr gs:[0x30]
4 | main.go:7 0x10a46a9 483b6110 cmp rsp, qword ptr [rcx+0x10]
5 | main.go:7 0x10a46ad 7624 jbe 0x10a46d3
6 | main.go:7 0x10a46af 4883ec08 sub rsp, 0x8
7 | main.go:7 0x10a46b3 48892c24 mov qword ptr [rsp], rbp
8 | main.go:7 0x10a46b7 488d2c24 lea rbp, ptr [rsp]
9 | main.go:8 0x10a46bb 0f1f440000 nop dword ptr [rax+rax*1], eax
10 | main.go:8 0x10a46c0 e81b000000 call $main.foo
11 | main.go:9 0x10a46c5 e8b6000000 call $main.foobar
12 | main.go:10 0x10a46ca 488b2c24 mov rbp, qword ptr [rsp]
13 | main.go:10 0x10a46ce 4883c408 add rsp, 0x8
14 | main.go:10 0x10a46d2 c3 ret
15 | main.go:7 0x10a46d3 e8a8ddfbff call $runtime.morestack_noctxt
16 | .:0 0x10a46d8 ebc6 jmp $main.main
17 | (dlv) disassemble -l main.foo
18 | TEXT main.foo(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go
19 | main.go:12 0x10a46e0 65488b0c2530000000 mov rcx, qword ptr gs:[0x30]
20 | main.go:12 0x10a46e9 483b6110 cmp rsp, qword ptr [rcx+0x10]
21 | main.go:12 0x10a46ed 761f jbe 0x10a470e
22 | main.go:12 0x10a46ef 4883ec08 sub rsp, 0x8
23 | main.go:12 0x10a46f3 48892c24 mov qword ptr [rsp], rbp
24 | main.go:12 0x10a46f7 488d2c24 lea rbp, ptr [rsp]
25 | main.go:13 0x10a46fb 0f1f440000 nop dword ptr [rax+rax*1], eax
26 | main.go:13 0x10a4700 e81b000000 call $main.bar
27 | main.go:14 0x10a4705 488b2c24 mov rbp, qword ptr [rsp]
28 | main.go:14 0x10a4709 4883c408 add rsp, 0x8
29 | main.go:14 0x10a470d c3 ret
30 | main.go:12 0x10a470e e86dddfbff call $runtime.morestack_noctxt
31 | .:0 0x10a4713 ebcb jmp $main.foo
32 | (dlv) disassemble -l main.bar
33 | TEXT main.bar(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go
34 | main.go:16 0x10a4720 65488b0c2530000000 mov rcx, qword ptr gs:[0x30]
35 | main.go:16 0x10a4729 483b6110 cmp rsp, qword ptr [rcx+0x10]
36 | main.go:16 0x10a472d 7642 jbe 0x10a4771
37 | main.go:16 0x10a472f 4883ec48 sub rsp, 0x48
38 | main.go:16 0x10a4733 48896c2440 mov qword ptr [rsp+0x40], rbp
39 | main.go:16 0x10a4738 488d6c2440 lea rbp, ptr [rsp+0x40]
40 | main.go:17 0x10a473d 488d05235c0200 lea rax, ptr [rip+0x25c23]
41 | main.go:17 0x10a4744 48890424 mov qword ptr [rsp], rax
42 | main.go:17 0x10a4748 48c74424080f000000 mov qword ptr [rsp+0x8], 0xf
43 | main.go:17 0x10a4751 0f57c0 xorps xmm0, xmm0
44 | main.go:17 0x10a4754 0f11442410 movups xmmword ptr [rsp+0x10], xmm0
45 | main.go:17 0x10a4759 48c744242000000000 mov qword ptr [rsp+0x20], 0x0
46 | main.go:17 0x10a4762 e8b982ffff call $fmt.Printf
47 | main.go:18 0x10a4767 488b6c2440 mov rbp, qword ptr [rsp+0x40]
48 | main.go:18 0x10a476c 4883c448 add rsp, 0x48
49 | main.go:18 0x10a4770 c3 ret
50 | main.go:16 0x10a4771 e80addfbff call $runtime.morestack_noctxt
51 | .:0 0x10a4776 eba8 jmp $main.bar
52 | (dlv) disassemble -l main.foobar
53 | TEXT main.foobar(SB) /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/presentation/main.go
54 | main.go:20 0x10a4780 65488b0c2530000000 mov rcx, qword ptr gs:[0x30]
55 | main.go:20 0x10a4789 483b6110 cmp rsp, qword ptr [rcx+0x10]
56 | main.go:20 0x10a478d 7642 jbe 0x10a47d1
57 | main.go:20 0x10a478f 4883ec48 sub rsp, 0x48
58 | main.go:20 0x10a4793 48896c2440 mov qword ptr [rsp+0x40], rbp
59 | main.go:20 0x10a4798 488d6c2440 lea rbp, ptr [rsp+0x40]
60 | main.go:21 0x10a479d 488d05ed630200 lea rax, ptr [rip+0x263ed]
61 | main.go:21 0x10a47a4 48890424 mov qword ptr [rsp], rax
62 | main.go:21 0x10a47a8 48c744240812000000 mov qword ptr [rsp+0x8], 0x12
63 | main.go:21 0x10a47b1 0f57c0 xorps xmm0, xmm0
64 | main.go:21 0x10a47b4 0f11442410 movups xmmword ptr [rsp+0x10], xmm0
65 | main.go:21 0x10a47b9 48c744242000000000 mov qword ptr [rsp+0x20], 0x0
66 | main.go:21 0x10a47c2 e85982ffff call $fmt.Printf
67 | main.go:22 0x10a47c7 488b6c2440 mov rbp, qword ptr [rsp+0x40]
68 | main.go:22 0x10a47cc 4883c448 add rsp, 0x48
69 | main.go:22 0x10a47d0 c3 ret
70 | main.go:20 0x10a47d1 e8aadcfbff call $runtime.morestack_noctxt
71 | .:0 0x10a47d6 eba8 jmp $main.foobar
72 |
--------------------------------------------------------------------------------
/examples/presentation/stack.txt:
--------------------------------------------------------------------------------
1 | panic: oh no
2 |
3 | goroutine 1 [running]:
4 | main.bar(...)
5 | /go/src/presentation/main.go:18
6 | main.foo()
7 | /go/src/presentation/main.go:13 +0x85
8 | main.main()
9 | /go/src/presentation/main.go:8 +0x25
10 |
--------------------------------------------------------------------------------
/examples/pseudo-code/pseudo-block.txt:
--------------------------------------------------------------------------------
1 | func chansend(channel, msg):
2 | // non-blocking send
3 | if ready(channel):
4 | send(channel, msg)
5 | return
6 |
7 | t0 = now()
8 | // let scheduler run another goroutine
9 | // until the channel is ready
10 | wait_ready(channel)
11 | duration = now() - t0
12 | send(channel, msg)
13 | // sample a fraction of blocking events
14 | if random_sample(duration):
15 | s = stacktrace()
16 | profile[s].count++
17 | profile[s].duration += duration
18 |
--------------------------------------------------------------------------------
/examples/pseudo-code/pseudo-heap.txt:
--------------------------------------------------------------------------------
1 | func malloc(size):
2 | object = ... // alloc magic
3 |
4 | if random_sample():
5 | s = stacktrace()
6 | profile[s].allocs++
7 | profile[s].alloc_bytes += sizeof(object)
8 | track_profiled(object, s)
9 |
10 | return object
11 |
12 | func sweep(object):
13 | // do gc stuff to free object
14 |
15 | if is_profiled(object)
16 | s = alloc_stacktrace(object)
17 | profile[s].frees++
18 | profile[s].free_bytes += sizeof(object)
19 |
20 | return object
21 |
--------------------------------------------------------------------------------
/examples/runtime-stack/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/runtime-stack
2 |
3 | go 1.16
4 |
--------------------------------------------------------------------------------
/examples/runtime-stack/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "runtime"
6 | "time"
7 | )
8 |
9 | func main() {
10 | sleep := 1*time.Minute + 10*time.Second
11 |
12 | go sleepLoop(sleep)
13 | go sleepLoop(time.Second)
14 | go chanReceiveForever()
15 | go indirectSleepLoop(time.Second)
16 |
17 | time.Sleep(time.Second)
18 | // g.wait
19 | runtime.GC()
20 | fmt.Printf("sleeping for %s before showing stack traces\n", sleep)
21 | time.Sleep(sleep)
22 | runtime.GC()
23 |
24 | fmt.Printf("%s has passed, dumping all stacks", sleep)
25 | buf := make([]byte, 1024*1024)
26 | n := runtime.Stack(buf, true)
27 | buf = buf[:n]
28 | fmt.Printf("%s\n", buf)
29 |
30 | fmt.Printf("waiting indefinitely so you can press ctrl+\\ to compare the output\n")
31 | runtime.GC()
32 | chanReceiveForever()
33 | }
34 |
35 | func sleepLoop(d time.Duration) {
36 | for {
37 | time.Sleep(d)
38 | }
39 | }
40 |
41 | func chanReceiveForever() {
42 | forever := make(chan struct{})
43 | <-forever
44 | }
45 |
46 | func indirectSleepLoop(d time.Duration) {
47 | indirectSleepLoop2(d)
48 | }
49 |
50 | func indirectSleepLoop2(d time.Duration) {
51 | go sleepLoop(d)
52 | }
53 |
--------------------------------------------------------------------------------
/examples/runtime-stack/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "runtime"
6 | "testing"
7 | "time"
8 | )
9 |
10 | func BenchmarkRuntimeStack(b *testing.B) {
11 | buf := make([]byte, 1024*1024*64)
12 | for g := 1; g <= 1024*1024; g = g * 2 {
13 | g := g
14 | name := fmt.Sprintf("%d goroutines", g)
15 |
16 | b.Run(name, func(b *testing.B) {
17 | initalRoutines := runtime.NumGoroutine()
18 |
19 | readyCh := make(chan struct{})
20 | stopCh := make(chan struct{})
21 | for i := 0; i < g; i++ {
22 | go atStackDepth(16, func() {
23 | defer func() { stopCh <- struct{}{} }()
24 | readyCh <- struct{}{}
25 | })
26 | <-readyCh
27 | }
28 |
29 | gotRoutines := runtime.NumGoroutine() - initalRoutines
30 | if gotRoutines != g {
31 | b.Logf("want %d goroutines, but got %d", g, gotRoutines)
32 | }
33 |
34 | b.ResetTimer()
35 | for i := 0; i < b.N; i++ {
36 | runtime.Stack(buf, true)
37 | }
38 | b.StopTimer()
39 | for i := 0; i < g; i++ {
40 | <-stopCh
41 | }
42 | start := time.Now()
43 | for i := 0; ; i++ {
44 | if runtime.NumGoroutine() == initalRoutines {
45 | break
46 | }
47 | time.Sleep(20 * time.Millisecond)
48 | if time.Since(start) > 10*time.Second {
49 | b.Fatalf("%d goroutines still running, want %d", runtime.NumGoroutine(), initalRoutines)
50 | }
51 | }
52 | })
53 | }
54 | }
55 |
56 | func atStackDepth(depth int, fn func()) {
57 | pcs := make([]uintptr, depth*10)
58 | n := runtime.Callers(1, pcs)
59 | if n > depth {
60 | panic("depth exceeded")
61 | } else if n < depth {
62 | atStackDepth(depth, fn)
63 | return
64 | }
65 |
66 | fn()
67 | }
68 |
--------------------------------------------------------------------------------
/examples/runtime-stack/runtime-stack.txt:
--------------------------------------------------------------------------------
1 | $ go run examples/runtime-stack/main.go
2 | sleeping for 1m10s before showing stack traces
3 | 1m10s has passed, dumping all stacksgoroutine 1 [running]:
4 | main.main()
5 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:26 +0x228
6 |
7 | goroutine 18 [sleep, 1 minutes]:
8 | time.Sleep(0x104c533c00)
9 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
10 | main.sleepLoop(0x104c533c00)
11 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b
12 | created by main.main
13 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:12 +0x56
14 |
15 | goroutine 19 [sleep, 1 minutes]:
16 | time.Sleep(0x3b9aca00)
17 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
18 | main.sleepLoop(0x3b9aca00)
19 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b
20 | created by main.main
21 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:13 +0x77
22 |
23 | goroutine 20 [chan receive, 1 minutes]:
24 | main.chanReceiveForever()
25 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:43 +0x4d
26 | created by main.main
27 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:14 +0x8f
28 |
29 | goroutine 22 [sleep, 1 minutes]:
30 | time.Sleep(0x3b9aca00)
31 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
32 | main.sleepLoop(0x3b9aca00)
33 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b
34 | created by main.indirectSleepLoop2
35 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:51 +0x3f
36 |
37 | waiting indefinitely so you can press ctrl+\ to compare the output
38 | ^\SIGQUIT: quit
39 | PC=0x7fff739f9882 m=0 sigcode=0
40 |
41 | goroutine 0 [idle]:
42 | runtime.pthread_cond_wait(0x1172620, 0x11725e0, 0x7ffe00000000)
43 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/sys_darwin.go:414 +0x39
44 | runtime.semasleep(0xffffffffffffffff, 0x103af45)
45 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/os_darwin.go:63 +0x8d
46 | runtime.notesleep(0x11723e8)
47 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/lock_sema.go:181 +0xe7
48 | runtime.stopm()
49 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:1924 +0xc5
50 | runtime.findrunnable(0xc000029800, 0x0)
51 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2485 +0xa7f
52 | runtime.schedule()
53 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2683 +0x2d7
54 | runtime.park_m(0xc000082600)
55 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/proc.go:2851 +0x9d
56 | runtime.mcall(0x10613d6)
57 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/asm_amd64.s:318 +0x5b
58 |
59 | goroutine 1 [chan receive, 3 minutes]:
60 | main.chanReceiveForever(...)
61 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:43
62 | main.main()
63 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:32 +0x33d
64 |
65 | goroutine 18 [sleep]:
66 | time.Sleep(0x104c533c00)
67 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
68 | main.sleepLoop(0x104c533c00)
69 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b
70 | created by main.main
71 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:12 +0x56
72 |
73 | goroutine 19 [sleep]:
74 | time.Sleep(0x3b9aca00)
75 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
76 | main.sleepLoop(0x3b9aca00)
77 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b
78 | created by main.main
79 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:13 +0x77
80 |
81 | goroutine 20 [chan receive, 4 minutes]:
82 | main.chanReceiveForever()
83 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:43 +0x4d
84 | created by main.main
85 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:14 +0x8f
86 |
87 | goroutine 22 [sleep]:
88 | time.Sleep(0x3b9aca00)
89 | /usr/local/Cellar/go/1.15.6/libexec/src/runtime/time.go:188 +0xbf
90 | main.sleepLoop(0x3b9aca00)
91 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:37 +0x2b
92 | created by main.indirectSleepLoop2
93 | /Users/felix.geisendoerfer/go/src/github.com/felixge/go-profiler-notes/examples/runtime-stack/main.go:51 +0x3f
94 |
95 | rax 0x104
96 | rbx 0x2
97 | rcx 0x7ffeefbff248
98 | rdx 0xec00
99 | rdi 0x1172620
100 | rsi 0xec010000ed00
101 | rbp 0x7ffeefbff2e0
102 | rsp 0x7ffeefbff248
103 | r8 0x0
104 | r9 0xa0
105 | r10 0x0
106 | r11 0x202
107 | r12 0x1172620
108 | r13 0x16
109 | r14 0xec010000ed00
110 | r15 0x167fdc0
111 | rip 0x7fff739f9882
112 | rflags 0x203
113 | cs 0x7
114 | fs 0x0
115 | gs 0x0
116 | exit status 2
117 |
--------------------------------------------------------------------------------
/examples/stack-unwind-overhead/README.md:
--------------------------------------------------------------------------------
1 | # stack-unwind-overhead
2 |
3 | This directory contains benchmarks to explore which factors impact stack unwinding in Go. It's informed by an analysis of the `gopclntab` unwinding implementation.
4 |
5 | ## Result 1: O(N) Stack Depth
6 |
7 | The benchmark below shows that stack unwinding has O(N) complexity with regard to the number of call frames on the stack:
8 |
9 | ```
10 | BenchmarkStackDepth/8-12 1968214 612.2 ns/op
11 | BenchmarkStackDepth/16-12 975457 1184 ns/op
12 | BenchmarkStackDepth/32-12 572706 2101 ns/op
13 | BenchmarkStackDepth/64-12 333598 3596 ns/op
14 | BenchmarkStackDepth/128-12 182450 6561 ns/op
15 | BenchmarkStackDepth/256-12 94783 12548 ns/op
16 | BenchmarkStackDepth/512-12 48439 24471 ns/op
17 | BenchmarkStackDepth/1024-12 24884 48310 ns/op
18 | ```
19 |
20 | Tests were done on my local machine:
21 |
22 | ```
23 | go test -bench .
24 | goos: darwin
25 | goarch: amd64
26 | pkg: github.com/felixge/go-profiler-notes/examples/stack-unwind-overhead
27 | cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
28 | ```
29 |
--------------------------------------------------------------------------------
/examples/stack-unwind-overhead/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/stack-unwind-overhead
2 |
3 | go 1.16
4 |
--------------------------------------------------------------------------------
/examples/stack-unwind-overhead/main.go:
--------------------------------------------------------------------------------
1 | // Package main is a code generator that creates functions of different size.
2 | package main
3 |
4 | import (
5 | "fmt"
6 | "os"
7 | )
8 |
9 | //go:generate go run .
10 |
11 | func main() {
12 | if err := run(); err != nil {
13 | fmt.Fprintln(os.Stderr, err)
14 | os.Exit(1)
15 | }
16 | }
17 |
18 | func run() error {
19 | file, err := os.Create("generated.go")
20 | if err != nil {
21 | return err
22 | }
23 | defer file.Close()
24 |
25 | fmt.Fprint(file, `// Code generated by go generate; DO NOT EDIT.
26 |
27 | package main
28 |
29 | //go:noinline
30 | func stackdepth1(fn func()) { fn() }
31 | `)
32 |
33 | max := 1024
34 | for i := 2; i <= max; i++ {
35 | fmt.Fprintf(
36 | file,
37 | "//go:noinline\nfunc stackdepth%d(fn func()) { stackdepth%d(fn) }\n",
38 | i,
39 | i-1,
40 | )
41 | }
42 |
43 | fmt.Fprint(file, "\nvar stackdepth = map[int]func(func()) {\n")
44 | for i := 1; i <= max; i++ {
45 | fmt.Fprintf(file, "\t%d: stackdepth%d,\n", i, i)
46 | }
47 | fmt.Fprint(file, "}\n")
48 |
49 | // Disabled: Turned out to be a deadend
50 | //for i := 1; i <= max; i *= 2 {
51 | //fmt.Fprintf(file, "//go:noinline\n")
52 | //fmt.Fprintf(file, "func funcsize%d(v0, n int, fn func()) int {\n", i)
53 | //for n := 1; n <= i; n++ {
54 | //fmt.Fprintf(file, "\tv%d := v%d\n", n, n-1)
55 | //fmt.Fprintf(file, "\tv%d = (^v%d + %d)\n", n, n, n)
56 | //fmt.Fprintf(file, "\tif v%d == 23 {\n", n)
57 | //sum := []string{}
58 | //for j := 0; j <= n; j++ {
59 | //sum = append(sum, fmt.Sprintf("v%d", j))
60 | //}
61 | //fmt.Fprintf(file, "\t\treturn %s\n", strings.Join(sum, " + "))
62 | //fmt.Fprintf(file, "\t}\n")
63 | //}
64 | //fmt.Fprintf(file, "\tfn()\n")
65 | //fmt.Fprintf(file, "\treturn -1\n")
66 | //fmt.Fprintf(file, "}\n\n")
67 | //}
68 |
69 | //fmt.Fprint(file, "\nvar funcsize = map[int]func(int, int, func()) int {\n")
70 | //for i := 1; i <= max; i *= 2 {
71 | //fmt.Fprintf(file, "\t%d: funcsize%d,\n", i, i)
72 | //}
73 | //fmt.Fprint(file, "}\n")
74 |
75 | return file.Close()
76 | }
77 |
--------------------------------------------------------------------------------
/examples/stack-unwind-overhead/main_test.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "runtime"
6 | "testing"
7 | )
8 |
9 | // BenchmarkStackDepth measures the impact of stack depth on the time it takes
10 | // to create a stack trace.
11 | func BenchmarkStackDepth(b *testing.B) {
12 | for d := 8; d <= 1024; d = d * 2 {
13 | b.Run(fmt.Sprintf("%d", d), func(b *testing.B) {
14 | callers := make([]uintptr, d*2)
15 | atStackDepth(d, func() {
16 | for i := 0; i < b.N; i++ {
17 | if n := runtime.Callers(1, callers); n != d {
18 | b.Fatalf("got=%d want=%d", n, d)
19 | }
20 | }
21 | })
22 | })
23 | }
24 | }
25 |
26 | // atStackDepth calls functions that call each other until there are depth-1
27 | // functions on the stack and then calls fn.
28 | //go:noinline
29 | func atStackDepth(depth int, fn func()) {
30 | pcs := make([]uintptr, depth+10)
31 | remaining := depth - runtime.Callers(1, pcs) - 1
32 | if remaining < 1 {
33 | panic("can't simulate desired stack depth: too low")
34 | } else if remaining == 1 {
35 | fn()
36 | } else if f, ok := stackdepth[remaining]; !ok {
37 | panic("can't simulate desired stack depth: no map entry")
38 | } else {
39 | f(fn)
40 | }
41 | }
42 |
43 | // disabled: turned out to be a deadend
44 | //func BenchmarkFunctionSize(b *testing.B) {
45 | //var callers = make([]uintptr, 32)
46 | //for s := 1; s <= 1024; s = s * 2 {
47 | //b.Run(fmt.Sprintf("%d", s), func(b *testing.B) {
48 | //called := false
49 | //funcsize[s](0, 0, func() {
50 | //for i := 0; i < b.N; i++ {
51 | //runtime.Callers(0, callers)
52 | //}
53 | //called = true
54 | //})
55 | //if !called {
56 | //b.Fatal("not called")
57 | //}
58 | //})
59 | //}
60 | //}
61 |
--------------------------------------------------------------------------------
/examples/stackannotate/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | func main() {
4 | var n int
5 | for i := 0; i < 10; i++ {
6 | n += foo(1)
7 | }
8 | println(n)
9 | <-chan struct{}(nil)
10 | }
11 |
12 | func foo(a int) int {
13 | return bar(a, 2)
14 | }
15 |
16 | func bar(a int, b int) int {
17 | s := 3
18 | for i := 0; i < 100; i++ {
19 | s += a * b
20 | }
21 | return s
22 | }
23 |
--------------------------------------------------------------------------------
/examples/stuck-deadlock/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/stuck-program
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/DataDog/datadog-go v4.4.0+incompatible // indirect
7 | github.com/Microsoft/go-winio v0.4.16 // indirect
8 | github.com/google/uuid v1.2.0 // indirect
9 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0
10 | )
11 |
12 | replace gopkg.in/DataDog/dd-trace-go.v1 => /Users/felix.geisendoerfer/go/src/github.com/DataDog/dd-trace-go
13 |
--------------------------------------------------------------------------------
/examples/stuck-deadlock/go.sum:
--------------------------------------------------------------------------------
1 | github.com/DataDog/datadog-go v4.4.0+incompatible h1:R7WqXWP4fIOAqWJtUKmSfuc7eDsBT58k9AY5WSHVosk=
2 | github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
3 | github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
4 | github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
5 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
6 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
7 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998 h1:ruQkWz0PK91vTVrWtzAgv3VqTMgIN1FAIvwWr5MY+GQ=
11 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
12 | github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
13 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
14 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
15 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
16 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
18 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
20 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
21 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
22 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
23 | github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
24 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
25 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
26 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
27 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
28 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 h1:EmglUJuykRsTwsQDcKaAo3CmOunWU6Dqk7U2lo7Pjss=
29 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0/go.mod h1:Sp1lku8WJMvNV0kjDI4Ni/T7J/U3BO5ct5kEaoVU8+I=
30 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
31 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
32 |
--------------------------------------------------------------------------------
/examples/stuck-deadlock/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "os/signal"
7 | "runtime"
8 | "sync"
9 | "time"
10 |
11 | "gopkg.in/DataDog/dd-trace-go.v1/profiler"
12 | )
13 |
14 | func main() {
15 | if err := run(); err != nil {
16 | fmt.Fprintln(os.Stderr, err)
17 | os.Exit(1)
18 | }
19 | }
20 | func run() error {
21 | if err := os.Setenv("DD_PROFILING_WAIT_PROFILE", "yes"); err != nil {
22 | return err
23 | } else if err := profiler.Start(
24 | profiler.WithService("stuck-deadlock"),
25 | profiler.WithVersion(time.Now().String()),
26 | profiler.WithPeriod(60*time.Second),
27 | profiler.WithProfileTypes(
28 | profiler.GoroutineProfile,
29 | ),
30 | ); err != nil {
31 | return err
32 | }
33 |
34 | var (
35 | a = &sync.Mutex{}
36 | b = &sync.Mutex{}
37 | )
38 | go bob(a, b)
39 | go alice(a, b)
40 |
41 | c := make(chan os.Signal, 1)
42 | signal.Notify(c, os.Interrupt)
43 | ticker := time.NewTicker(10 * time.Second)
44 | defer ticker.Stop()
45 | for {
46 | select {
47 | case <-ticker.C:
48 | // force gc on a regular basis to make sure g.waitsince gets populated.
49 | fmt.Printf("gc\n")
50 | runtime.GC()
51 | case sig := <-c:
52 | fmt.Printf("sig: %s\n", sig)
53 | return nil
54 | }
55 | }
56 | return nil
57 | }
58 |
59 | func bob(a, b *sync.Mutex) {
60 | for {
61 | fmt.Println("bob is okay")
62 | a.Lock()
63 | b.Lock()
64 | // do stuff
65 | a.Unlock()
66 | b.Unlock()
67 | }
68 | }
69 |
70 | func alice(a, b *sync.Mutex) {
71 | for {
72 | fmt.Println("alice is okay")
73 | b.Lock()
74 | a.Lock()
75 | // do stuff
76 | b.Unlock()
77 | a.Unlock()
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/examples/stuck-producer-consumer/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/felixge/go-profiler-notes/examples/stuck-producer-consumer
2 |
3 | go 1.15
4 |
5 | require (
6 | github.com/DataDog/datadog-go v4.4.0+incompatible // indirect
7 | github.com/Microsoft/go-winio v0.4.16 // indirect
8 | github.com/google/uuid v1.2.0 // indirect
9 | gopkg.in/DataDog/dd-trace-go.v1 v1.29.0-alpha.1.0.20210216140755-2d091eca40bb
10 | )
11 |
12 | replace gopkg.in/DataDog/dd-trace-go.v1 => /Users/felix.geisendoerfer/go/src/github.com/DataDog/dd-trace-go
13 |
--------------------------------------------------------------------------------
/examples/stuck-producer-consumer/go.sum:
--------------------------------------------------------------------------------
1 | github.com/DataDog/datadog-go v4.4.0+incompatible h1:R7WqXWP4fIOAqWJtUKmSfuc7eDsBT58k9AY5WSHVosk=
2 | github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
3 | github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
4 | github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
5 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
6 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
7 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998 h1:ruQkWz0PK91vTVrWtzAgv3VqTMgIN1FAIvwWr5MY+GQ=
11 | github.com/google/pprof v0.0.0-20210125172800-10e9aeb4a998/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
12 | github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
13 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
14 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
15 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
16 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
18 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
19 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
20 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
21 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
22 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
23 | github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
24 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
25 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=
26 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
27 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e h1:9vRrk9YW2BTzLP0VCB9ZDjU4cPqkg+IDWL7XgxA1yxQ=
28 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
29 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0 h1:EmglUJuykRsTwsQDcKaAo3CmOunWU6Dqk7U2lo7Pjss=
30 | gopkg.in/DataDog/dd-trace-go.v1 v1.28.0/go.mod h1:Sp1lku8WJMvNV0kjDI4Ni/T7J/U3BO5ct5kEaoVU8+I=
31 | gopkg.in/DataDog/dd-trace-go.v1 v1.29.0-alpha.1.0.20210216140755-2d091eca40bb h1:BQMy4tHVzHLWsGeQMmYrgWbHkVPp0ibHM/IDREdchyk=
32 | gopkg.in/DataDog/dd-trace-go.v1 v1.29.0-alpha.1.0.20210216140755-2d091eca40bb/go.mod h1:FLwUDeuH0z5hkvgvd04/M3MHQN4AF5pQDnedeWRWvok=
33 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
34 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
35 |
--------------------------------------------------------------------------------
/examples/stuck-producer-consumer/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "math/rand"
6 | "os"
7 | "os/signal"
8 | "runtime"
9 | "time"
10 |
11 | "gopkg.in/DataDog/dd-trace-go.v1/profiler"
12 | )
13 |
14 | func main() {
15 | if err := run(); err != nil {
16 | fmt.Fprintln(os.Stderr, err)
17 | os.Exit(1)
18 | }
19 | }
20 |
21 | func run() error {
22 | if err := os.Setenv("DD_PROFILING_WAIT_PROFILE", "yes"); err != nil {
23 | return err
24 | } else if err := profiler.Start(
25 | profiler.WithService("stuck-producer-consumer"),
26 | profiler.WithVersion(time.Now().String()),
27 | profiler.WithPeriod(60*time.Second),
28 | profiler.WithProfileTypes(
29 | profiler.GoroutineProfile,
30 | ),
31 | ); err != nil {
32 | return err
33 | }
34 |
35 | rand.Seed(time.Now().UnixNano())
36 |
37 | workCh := make(chan struct{})
38 | go consumer(workCh)
39 | go producer(workCh)
40 |
41 | c := make(chan os.Signal, 1)
42 | signal.Notify(c, os.Interrupt)
43 | ticker := time.NewTicker(10 * time.Second)
44 | defer ticker.Stop()
45 | for {
46 | select {
47 | case <-ticker.C:
48 | // force gc on a regular basis to make sure g.waitsince gets populated.
49 | fmt.Printf("gc\n")
50 | runtime.GC()
51 | case sig := <-c:
52 | fmt.Printf("sig: %s\n", sig)
53 | return nil
54 | }
55 | }
56 | }
57 |
58 | func producer(workCh chan<- struct{}) {
59 | for {
60 | select {
61 | case workCh <- struct{}{}:
62 | if rand.Int63n(10) == 0 {
63 | takeNap()
64 | }
65 | default:
66 | }
67 | }
68 | }
69 |
70 | func consumer(workCh <-chan struct{}) {
71 | for {
72 | <-workCh
73 | if rand.Int63n(10) == 0 {
74 | takeNap()
75 | }
76 | }
77 | }
78 |
79 | func takeNap() {
80 | fmt.Printf("taking a nap\n")
81 | var forever chan struct{}
82 | <-forever
83 | }
84 |
--------------------------------------------------------------------------------
/examples/threadcreate/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "runtime"
7 | "runtime/pprof"
8 | "syscall"
9 | "time"
10 | )
11 |
12 | func main() {
13 | if err := run(); err != nil {
14 | fmt.Fprintln(os.Stderr, err)
15 | os.Exit(1)
16 | }
17 | }
18 |
19 | func run() error {
20 | runtime.GOMAXPROCS(1)
21 | n := 16
22 | ch := make(chan int)
23 | fmt.Printf("start work\n")
24 | for i := 0; i < n; i++ {
25 | go func() {
26 | buf := make([]byte, 1024)
27 | fmt.Printf("syscall\n")
28 | n, err := syscall.Read(1, buf)
29 | if err != nil {
30 | panic(err)
31 | }
32 | fmt.Printf("%d\n", n)
33 | }()
34 | go dowork(ch)
35 | }
36 | runtime.GOMAXPROCS(8)
37 | time.Sleep(50 * time.Millisecond)
38 | runtime.GC()
39 | if err := writeProfile("threadcreate"); err != nil {
40 | return err
41 | }
42 | fmt.Printf("profile\n")
43 | for i := 0; i < n; i++ {
44 | fmt.Printf("%d\n", <-ch)
45 | }
46 | fmt.Printf("done\n")
47 |
48 | return nil
49 | }
50 |
51 | func dowork(done chan int) {
52 | s := 0
53 | for i := 0; i < 1000000000; i++ {
54 | if i%2 == 0 {
55 | s++
56 | }
57 | }
58 | done <- s
59 | }
60 |
61 | func writeProfile(name string) error {
62 | f, err := os.Create(name + ".pb.gz")
63 | if err != nil {
64 | return err
65 | }
66 | defer f.Close()
67 |
68 | if err := pprof.Lookup(name).WriteTo(f, 0); err != nil {
69 | return err
70 | }
71 | return nil
72 | }
73 |
--------------------------------------------------------------------------------
/flame-abc.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/flame-abc.png
--------------------------------------------------------------------------------
/flame-abc.txt:
--------------------------------------------------------------------------------
1 | A 5
2 | A;C 3
3 | B 4
4 |
--------------------------------------------------------------------------------
/go-binary.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/go-binary.png
--------------------------------------------------------------------------------
/goroutine-matrix.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/goroutine-matrix.png
--------------------------------------------------------------------------------
/goroutine-stack.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/goroutine-stack.png
--------------------------------------------------------------------------------
/guide/Makefile:
--------------------------------------------------------------------------------
1 | .PHONY: scheduler.gif
2 | scheduler.gif:
3 | convert -delay 100 scheduler.*.png scheduler.gif
4 |
5 | .PHONY: stack.gif
6 | stack.gif:
7 | convert -delay 200 stack*.png stack.gif
8 |
9 | .PHONY: heap-gc.gif
10 | heap-gc.gif:
11 | convert -delay 200 heap-gc.*.png heap-gc.gif
12 |
--------------------------------------------------------------------------------
/guide/cpu-max-stack-depth.go:
--------------------------------------------------------------------------------
1 | //go:build ignore
2 | // +build ignore
3 |
4 | package main
5 |
6 | import (
7 | "os"
8 | "runtime/pprof"
9 | )
10 |
11 | func main() {
12 | pprof.StartCPUProfile(os.Stdout)
13 | defer pprof.StopCPUProfile()
14 |
15 | belowLimit()
16 | aboveLimit()
17 | }
18 |
19 | func belowLimit() {
20 | atDepth(32, cpuHog)
21 | }
22 |
23 | func aboveLimit() {
24 | atDepth(64, cpuHog)
25 | }
26 |
27 | func cpuHog() {
28 | for i := 0; i < 5000*1000*1000; i++ {
29 | }
30 | }
31 |
32 | func atDepth(n int, fn func()) {
33 | if n > 0 {
34 | atDepth(n-1, fn)
35 | } else {
36 | fn()
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/guide/cpu-max-stack-depth.pprof:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-max-stack-depth.pprof
--------------------------------------------------------------------------------
/guide/cpu-profiler-labels.go:
--------------------------------------------------------------------------------
1 | // +build ignore
2 |
3 | package main
4 |
5 | import (
6 | "context"
7 | "os"
8 | "runtime/pprof"
9 | "time"
10 | )
11 |
12 | func main() {
13 | pprof.StartCPUProfile(os.Stdout)
14 | defer pprof.StopCPUProfile()
15 |
16 | go work(context.Background(), "alice")
17 | go work(context.Background(), "bob")
18 |
19 | time.Sleep(50 * time.Millisecond)
20 | }
21 |
22 | func work(ctx context.Context, user string) {
23 | labels := pprof.Labels("user", user)
24 | pprof.Do(ctx, labels, func(_ context.Context) {
25 | go backgroundWork()
26 | directWork()
27 | })
28 | }
29 |
30 | func directWork() {
31 | for {
32 | }
33 | }
34 |
35 | func backgroundWork() {
36 | for {
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/guide/cpu-profiler-labels.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-profiler-labels.png
--------------------------------------------------------------------------------
/guide/cpu-profiler-labels.pprof:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-profiler-labels.pprof
--------------------------------------------------------------------------------
/guide/cpu-rate.go:
--------------------------------------------------------------------------------
1 | // +build ignore
2 |
3 | package main
4 |
5 | import (
6 | "os"
7 | "runtime"
8 | "runtime/pprof"
9 | "time"
10 | )
11 |
12 | func main() {
13 | runtime.SetCPUProfileRate(800)
14 |
15 | pprof.StartCPUProfile(os.Stdout)
16 | defer pprof.StopCPUProfile()
17 |
18 | go cpuHog()
19 | time.Sleep(time.Second)
20 | }
21 |
22 | func cpuHog() {
23 | for {
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/guide/cpu-rate.pprof:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-rate.pprof
--------------------------------------------------------------------------------
/guide/cpu-utilization.go:
--------------------------------------------------------------------------------
1 | // +build ignore
2 |
3 | package main
4 |
5 | import (
6 | "os"
7 | "runtime/pprof"
8 | "time"
9 | )
10 |
11 | func main() {
12 | file, _ := os.Create("./cpu-utilization.pprof")
13 | pprof.StartCPUProfile(file)
14 | defer pprof.StopCPUProfile()
15 |
16 | go cpuHog()
17 | go cpuHog()
18 |
19 | time.Sleep(time.Second)
20 | }
21 |
22 | func cpuHog() {
23 | for {
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/guide/cpu-utilization.pprof:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/cpu-utilization.pprof
--------------------------------------------------------------------------------
/guide/heap-gc.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-gc.1.png
--------------------------------------------------------------------------------
/guide/heap-gc.2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-gc.2.png
--------------------------------------------------------------------------------
/guide/heap-gc.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-gc.gif
--------------------------------------------------------------------------------
/guide/heap-simple.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap-simple.png
--------------------------------------------------------------------------------
/guide/heap.go:
--------------------------------------------------------------------------------
1 | // +build ignore
2 |
3 | package main
4 |
5 | import (
6 | "fmt"
7 | )
8 |
9 | func main() {
10 | fmt.Println(*add(23, 42))
11 | }
12 |
13 | func add(a, b int) *int {
14 | sum := a + b
15 | return &sum
16 | }
17 |
--------------------------------------------------------------------------------
/guide/heap.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/heap.png
--------------------------------------------------------------------------------
/guide/http-get.go:
--------------------------------------------------------------------------------
1 | //go:build ignore
2 |
3 | package main
4 |
5 | import (
6 | "fmt"
7 | "log"
8 | "net/http"
9 | "os"
10 | "runtime/trace"
11 | "time"
12 | )
13 |
14 | func main() {
15 | traceF, _ := os.Create("trace.out")
16 | trace.Start(traceF)
17 | defer trace.Stop()
18 |
19 | start := time.Now()
20 | res, err := http.Get("https://example.org/")
21 | if err != nil {
22 | panic(err)
23 | }
24 | fmt.Printf("%d\n", res.StatusCode)
25 | log.Printf("main() took: %s\n", time.Since(start))
26 | }
27 |
--------------------------------------------------------------------------------
/guide/memory-profiler.go:
--------------------------------------------------------------------------------
1 | //go:build ignore
2 | // +build ignore
3 |
4 | package main
5 |
6 | import (
7 | "fmt"
8 | "os"
9 | "runtime"
10 | "runtime/pprof"
11 | "time"
12 | )
13 |
14 | func main() {
15 | file, _ := os.Create("./memory-profiler.pprof")
16 | defer pprof.Lookup("allocs").WriteTo(file, 0)
17 | defer runtime.GC()
18 |
19 | go allocSmall()
20 | go allocBig()
21 |
22 | time.Sleep(1 * time.Second)
23 |
24 | var m runtime.MemStats
25 | runtime.ReadMemStats(&m)
26 | fmt.Printf("%#v\n", m)
27 | }
28 |
29 | //go:noinline
30 | func allocSmall() {
31 | for i := 0; ; i++ {
32 | _ = alloc(32)
33 | }
34 | }
35 |
36 | //go:noinline
37 | func allocBig() {
38 | for i := 0; ; i++ {
39 | _ = alloc(256)
40 | }
41 | }
42 |
43 | //go:noinline
44 | func alloc(size int) []byte {
45 | return make([]byte, size)
46 | }
47 |
--------------------------------------------------------------------------------
/guide/memory-profiler.pprof:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/memory-profiler.pprof
--------------------------------------------------------------------------------
/guide/profiler-venn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/profiler-venn.png
--------------------------------------------------------------------------------
/guide/scheduler-complete.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler-complete.png
--------------------------------------------------------------------------------
/guide/scheduler.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.1.png
--------------------------------------------------------------------------------
/guide/scheduler.2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.2.png
--------------------------------------------------------------------------------
/guide/scheduler.3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.3.png
--------------------------------------------------------------------------------
/guide/scheduler.4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.4.png
--------------------------------------------------------------------------------
/guide/scheduler.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/scheduler.gif
--------------------------------------------------------------------------------
/guide/stack.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack.gif
--------------------------------------------------------------------------------
/guide/stack.go:
--------------------------------------------------------------------------------
1 | // +build ignore
2 |
3 | package main
4 |
5 | import "fmt"
6 |
7 | func main() {
8 | sum := 0
9 | sum = add(23, 42)
10 | fmt.Println(sum)
11 | }
12 |
13 | func add(a, b int) int {
14 | return a + b
15 | }
16 |
--------------------------------------------------------------------------------
/guide/stack1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack1.png
--------------------------------------------------------------------------------
/guide/stack2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack2.png
--------------------------------------------------------------------------------
/guide/stack3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/stack3.png
--------------------------------------------------------------------------------
/guide/time.1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/time.1.png
--------------------------------------------------------------------------------
/guide/time.2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/time.2.png
--------------------------------------------------------------------------------
/guide/time.3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/time.3.png
--------------------------------------------------------------------------------
/guide/timeline.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/timeline.png
--------------------------------------------------------------------------------
/guide/trace.out:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/guide/trace.out
--------------------------------------------------------------------------------
/heap.md:
--------------------------------------------------------------------------------
1 | # Go's Heap Profiler
2 |
3 | Totally a work in progress ... !!! Please come back later : ).
4 |
5 | https://twitter.com/felixge/status/1355846360562589696
6 |
7 |
8 |
9 | ## How does it work?
10 |
11 | Look at the go source code to understand how the data is captured, what role runtime.MemProfileRate plays, etc.
12 |
13 | ## How does it fail?
14 |
15 | Think through sampling rate issues, alloc size classes, etc.
16 |
17 | ## Performance Overhead
18 |
19 | Talk about performance overhead ...
20 |
21 | ## Data Format
22 |
23 | Figure out how the data ends up in the pprof file.
24 |
25 | ## GC Control
26 |
27 | ```
28 | # turn of gc
29 | GOGC=off go run
30 | # print gc events to stdout
31 | GODEBUG=gctrace=1 go run
32 | ```
33 |
34 | ## Questions
35 |
36 | - What are the [docs](https://golang.org/pkg/runtime/pprof/#Profile) talking about here? How do I actually use this?
37 |
38 | > Pprof's -inuse_space, -inuse_objects, -alloc_space, and -alloc_objects flags select which to display, defaulting to -inuse_space (live objects, scaled by size).
39 |
40 | A: Those flags are deprecated. Easiest way to select this stuff is via the pprof web ui's sample drop down.
41 |
42 | - The [docs](https://golang.org/pkg/runtime/pprof/#Profile) say I should get some kind of data, even if there is no GC. I can reproduce that, but the data seems to not change?
43 |
44 | > If there has been no garbage collection at all, the heap profile reports all known allocations. This exception helps mainly in programs running without garbage collection enabled, usually for debugging purposes.
45 |
46 | ## Disclaimers
47 |
48 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ).
49 |
50 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
--------------------------------------------------------------------------------
/hello.trace:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/hello.trace
--------------------------------------------------------------------------------
/mutex.md:
--------------------------------------------------------------------------------
1 | 🚧 This note is still work in progress, please come back later! 🚧
2 |
3 | # Mutex Profiling in Go
4 |
5 | ## History
6 |
7 | Mutex profiling was [implemented](https://go-review.googlesource.com/c/go/+/29650/) by [Peter J. Weinberger](https://en.wikipedia.org/wiki/Peter_J._Weinberger) and first appeared in the [go1.8](https://golang.org/doc/go1.8#mutex_prof) release (2017-02-16).
8 |
9 | ## Links
10 |
11 | - https://rakyll.org/mutexprofile/
12 | - https://talks.golang.org/2017/state-of-go.slide#23
13 |
14 | ## Disclaimers
15 |
16 | I'm [felixge](https://github.com/felixge) and work at [Datadog](https://www.datadoghq.com/) on [Continuous Profiling](https://www.datadoghq.com/product/code-profiling/) for Go. You should check it out. We're also [hiring](https://www.datadoghq.com/jobs-engineering/#all&all_locations) : ).
17 |
18 | The information on this page is believed to be correct, but no warranty is provided. Feedback is welcome!
--------------------------------------------------------------------------------
/profile.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/profile.png
--------------------------------------------------------------------------------
/profile.proto:
--------------------------------------------------------------------------------
1 | // Copyright 2016 Google Inc. All Rights Reserved.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | // Profile is a common stacktrace profile format.
16 | //
17 | // Measurements represented with this format should follow the
18 | // following conventions:
19 | //
20 | // - Consumers should treat unset optional fields as if they had been
21 | // set with their default value.
22 | //
23 | // - When possible, measurements should be stored in "unsampled" form
24 | // that is most useful to humans. There should be enough
25 | // information present to determine the original sampled values.
26 | //
27 | // - On-disk, the serialized proto must be gzip-compressed.
28 | //
29 | // - The profile is represented as a set of samples, where each sample
30 | // references a sequence of locations, and where each location belongs
31 | // to a mapping.
32 | // - There is a N->1 relationship from sample.location_id entries to
33 | // locations. For every sample.location_id entry there must be a
34 | // unique Location with that id.
35 | // - There is an optional N->1 relationship from locations to
36 | // mappings. For every nonzero Location.mapping_id there must be a
37 | // unique Mapping with that id.
38 |
39 | syntax = "proto3";
40 |
41 | package perftools.profiles;
42 |
43 | option java_package = "com.google.perftools.profiles";
44 | option java_outer_classname = "ProfileProto";
45 |
46 | message Profile {
47 | // A description of the samples associated with each Sample.value.
48 | // For a cpu profile this might be:
49 | // [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]]
50 | // For a heap profile, this might be:
51 | // [["allocations","count"], ["space","bytes"]],
52 | // If one of the values represents the number of events represented
53 | // by the sample, by convention it should be at index 0 and use
54 | // sample_type.unit == "count".
55 | repeated ValueType sample_type = 1;
56 | // The set of samples recorded in this profile.
57 | repeated Sample sample = 2;
58 | // Mapping from address ranges to the image/binary/library mapped
59 | // into that address range. mapping[0] will be the main binary.
60 | repeated Mapping mapping = 3;
61 | // Useful program location
62 | repeated Location location = 4;
63 | // Functions referenced by locations
64 | repeated Function function = 5;
65 | // A common table for strings referenced by various messages.
66 | // string_table[0] must always be "".
67 | repeated string string_table = 6;
68 | // frames with Function.function_name fully matching the following
69 | // regexp will be dropped from the samples, along with their successors.
70 | int64 drop_frames = 7; // Index into string table.
71 | // frames with Function.function_name fully matching the following
72 | // regexp will be kept, even if it matches drop_functions.
73 | int64 keep_frames = 8; // Index into string table.
74 |
75 | // The following fields are informational, do not affect
76 | // interpretation of results.
77 |
78 | // Time of collection (UTC) represented as nanoseconds past the epoch.
79 | int64 time_nanos = 9;
80 | // Duration of the profile, if a duration makes sense.
81 | int64 duration_nanos = 10;
82 | // The kind of events between sampled ocurrences.
83 | // e.g [ "cpu","cycles" ] or [ "heap","bytes" ]
84 | ValueType period_type = 11;
85 | // The number of events between sampled occurrences.
86 | int64 period = 12;
87 | // Freeform text associated to the profile.
88 | repeated int64 comment = 13; // Indices into string table.
89 | // Index into the string table of the type of the preferred sample
90 | // value. If unset, clients should default to the last sample value.
91 | int64 default_sample_type = 14;
92 | }
93 |
94 | // ValueType describes the semantics and measurement units of a value.
95 | message ValueType {
96 | int64 type = 1; // Index into string table.
97 | int64 unit = 2; // Index into string table.
98 | }
99 |
100 | // Each Sample records values encountered in some program
101 | // context. The program context is typically a stack trace, perhaps
102 | // augmented with auxiliary information like the thread-id, some
103 | // indicator of a higher level request being handled etc.
104 | message Sample {
105 | // The ids recorded here correspond to a Profile.location.id.
106 | // The leaf is at location_id[0].
107 | repeated uint64 location_id = 1;
108 | // The type and unit of each value is defined by the corresponding
109 | // entry in Profile.sample_type. All samples must have the same
110 | // number of values, the same as the length of Profile.sample_type.
111 | // When aggregating multiple samples into a single sample, the
112 | // result has a list of values that is the elemntwise sum of the
113 | // lists of the originals.
114 | repeated int64 value = 2;
115 | // label includes additional context for this sample. It can include
116 | // things like a thread id, allocation size, etc
117 | repeated Label label = 3;
118 | }
119 |
120 | message Label {
121 | int64 key = 1; // Index into string table
122 |
123 | // At most one of the following must be present
124 | int64 str = 2; // Index into string table
125 | int64 num = 3;
126 |
127 | // Should only be present when num is present.
128 | // Specifies the units of num.
129 | // Use arbitrary string (for example, "requests") as a custom count unit.
130 | // If no unit is specified, consumer may apply heuristic to deduce the unit.
131 | // Consumers may also interpret units like "bytes" and "kilobytes" as memory
132 | // units and units like "seconds" and "nanoseconds" as time units,
133 | // and apply appropriate unit conversions to these.
134 | int64 num_unit = 4; // Index into string table
135 | }
136 |
137 | message Mapping {
138 | // Unique nonzero id for the mapping.
139 | uint64 id = 1;
140 | // Address at which the binary (or DLL) is loaded into memory.
141 | uint64 memory_start = 2;
142 | // The limit of the address range occupied by this mapping.
143 | uint64 memory_limit = 3;
144 | // Offset in the binary that corresponds to the first mapped address.
145 | uint64 file_offset = 4;
146 | // The object this entry is loaded from. This can be a filename on
147 | // disk for the main binary and shared libraries, or virtual
148 | // abstractions like "[vdso]".
149 | int64 filename = 5; // Index into string table
150 | // A string that uniquely identifies a particular program version
151 | // with high probability. E.g., for binaries generated by GNU tools,
152 | // it could be the contents of the .note.gnu.build-id field.
153 | int64 build_id = 6; // Index into string table
154 |
155 | // The following fields indicate the resolution of symbolic info.
156 | bool has_functions = 7;
157 | bool has_filenames = 8;
158 | bool has_line_numbers = 9;
159 | bool has_inline_frames = 10;
160 | }
161 |
162 | // Describes function and line table debug information.
163 | message Location {
164 | // Unique nonzero id for the location. A profile could use
165 | // instruction addresses or any integer sequence as ids.
166 | uint64 id = 1;
167 | // The id of the corresponding profile.Mapping for this location.
168 | // It can be unset if the mapping is unknown or not applicable for
169 | // this profile type.
170 | uint64 mapping_id = 2;
171 | // The instruction address for this location, if available. It
172 | // should be within [Mapping.memory_start...Mapping.memory_limit]
173 | // for the corresponding mapping. A non-leaf address may be in the
174 | // middle of a call instruction. It is up to display tools to find
175 | // the beginning of the instruction if necessary.
176 | uint64 address = 3;
177 | // Multiple line indicates this location has inlined functions,
178 | // where the last entry represents the caller into which the
179 | // preceding entries were inlined.
180 | //
181 | // E.g., if memcpy() is inlined into printf:
182 | // line[0].function_name == "memcpy"
183 | // line[1].function_name == "printf"
184 | repeated Line line = 4;
185 | // Provides an indication that multiple symbols map to this location's
186 | // address, for example due to identical code folding by the linker. In that
187 | // case the line information above represents one of the multiple
188 | // symbols. This field must be recomputed when the symbolization state of the
189 | // profile changes.
190 | bool is_folded = 5;
191 | }
192 |
193 | message Line {
194 | // The id of the corresponding profile.Function for this line.
195 | uint64 function_id = 1;
196 | // Line number in source code.
197 | int64 line = 2;
198 | }
199 |
200 | message Function {
201 | // Unique nonzero id for the function.
202 | uint64 id = 1;
203 | // Name of the function, in human-readable form if available.
204 | int64 name = 2; // Index into string table
205 | // Name of the function, as identified by the system.
206 | // For instance, it can be a C++ mangled name.
207 | int64 system_name = 3; // Index into string table
208 | // Source file containing the function.
209 | int64 filename = 4; // Index into string table
210 | // Line number in source file.
211 | int64 start_line = 5;
212 | }
--------------------------------------------------------------------------------
/scheduler.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/scheduler.png
--------------------------------------------------------------------------------
/sim/block_sampling_biased.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/sim/block_sampling_biased.png
--------------------------------------------------------------------------------
/sim/block_sampling_debiased.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/sim/block_sampling_debiased.png
--------------------------------------------------------------------------------
/sim/block_sampling_hist.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/DataDog/go-profiler-notes/65dd611ec7b225a8c843a284f755e3cfe0593176/sim/block_sampling_hist.png
--------------------------------------------------------------------------------