├── .github └── workflows │ ├── codecov.yml │ ├── lint.yml │ └── test.yml ├── LICENSE ├── README.md ├── cmd └── collect │ └── main.go ├── go.mod ├── go.sum └── profiles ├── profiles.go └── profiles_test.go /.github/workflows/codecov.yml: -------------------------------------------------------------------------------- 1 | name: coverage 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | with: 11 | fetch-depth: 2 12 | - uses: actions/setup-go@v2 13 | with: 14 | go-version: '1.16' 15 | - name: Run coverage 16 | run: go test ./... -race -coverprofile=coverage.out -covermode=atomic 17 | - name: Upload coverage to Codecov 18 | run: bash <(curl -s https://codecov.io/bash) 19 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | golangci: 7 | name: lint 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v2 11 | - name: golangci-lint 12 | uses: golangci/golangci-lint-action@v2 13 | with: 14 | version: v1.53.3 15 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | strategy: 8 | matrix: 9 | go-version: [1.16.x] 10 | platform: [ubuntu-latest, macos-latest, windows-latest] 11 | 12 | runs-on: ${{ matrix.platform }} 13 | 14 | steps: 15 | - name: Install Go 16 | uses: actions/setup-go@v2 17 | with: 18 | go-version: ${{ matrix.go-version }} 19 | 20 | - name: Checkout code 21 | uses: actions/checkout@v2 22 | 23 | - name: Test 24 | run: go test ./... 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Denis Maximov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Collect 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/tommsawyer/collect)](https://goreportcard.com/report/github.com/tommsawyer/collect) 3 | [![codecov](https://codecov.io/gh/tommsawyer/collect/branch/main/graph/badge.svg?token=63GFZ0O3OR)](https://codecov.io/gh/tommsawyer/collect) 4 | 5 | Allows you to collect all pprof profiles with one command. 6 | 7 | ## Installation 8 | Just go-get it: 9 | ```bash 10 | $ go get github.com/tommsawyer/collect/cmd/collect 11 | ``` 12 | 13 | ## Motivation 14 | 15 | Sometimes I need to quickly collect all pprof profiles for future optimization. It's very frustrating to do it with long curl commands like: 16 | ```bash 17 | $ curl -sK -v http://localhost:8080/debug/pprof/heap > heap.out && curl -sK -v http://localhost:8080/debug/pprof/allocs > allocs.out && curl -sK -v http://localhost:8080/debug/pprof/goroutine > goroutine.out && curl -sK -v http://localhost:8080/debug/pprof/profile > profile.out && curl -o ./trace "http://localhost:8080/debug/pprof/trace?debug=1&seconds=20" 18 | 19 | ``` 20 | 21 | Also: 22 | - it doesn't run concurrently, resulting in slow execution 23 | - you have to manually move profiles to some directories if you want to store them for future comparison 24 | - you need to wait for the command to complete and run it again if you want to collect profiles several times 25 | 26 | ## Usage 27 | Provide url from which profiles will be scraped: 28 | ```bash 29 | $ collect -u=http://localhost:8080 30 | ``` 31 | This will download allocs, heap, goroutine and cpu profiles and save them into a directory structure like this: 32 | 33 | ``` 34 | - localhost 8080 35 | - YYYY MM DD 36 | - HH MM SS 37 | - allocs 38 | - heap 39 | - profile 40 | - goroutine 41 | ``` 42 | 43 | You can provide as many urls as you want: 44 | ```bash 45 | $ collect -u=http://localhost:8080 -u=http://localhost:7070 46 | ``` 47 | 48 | You can choose which profiles will be scraped: 49 | ```bash 50 | $ collect -p=allocs -p=heap -u=http://localhost:8080 51 | ``` 52 | 53 | Query parameters for profiles are also supported: 54 | ```bash 55 | $ collect -p=trace\?seconds\=20 -u=http://localhost:8080 56 | ``` 57 | 58 | Use `-l` flag to collect profiles in an endless loop(until Ctrl-C). This will collect profiles every 60 seconds (you can redefine interval with `-i`). 59 | ```bash 60 | $ collect -l -u=http://localhost:8080 61 | ``` 62 | 63 | ## Command-Line flags 64 | | Flag | Default | Usage | 65 | | ----------- | -------------------------------| -------------------------------------------| 66 | | -u | | url from which profiles will be collected. | 67 | | -p | allocs,heap,goroutine,profile | profiles to collect. | 68 | | -l | false | collect profiles in endless loop | 69 | | -i | 60s | interval between collecting. use with -l | 70 | | -d | ./ | directory to put the pprof files in. | 71 | | -k | false | keep going collect if some requests failed.| 72 | -------------------------------------------------------------------------------- /cmd/collect/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "time" 10 | 11 | "github.com/jessevdk/go-flags" 12 | "github.com/tommsawyer/collect/profiles" 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | var cfg struct { 17 | Hosts []string `short:"u" description:"hosts from which profiles will be collected" required:"true"` 18 | Profiles []string `short:"p" description:"profiles to collect. Possible options: allocs/heap/goroutine/profile/trace." default:"allocs" default:"heap" default:"goroutine" default:"profile"` 19 | Loop bool `short:"l" description:"collect many times (until Ctrl-C)"` 20 | Interval time.Duration `short:"i" description:"interval between collecting (use with -l)" default:"60s"` 21 | Directory string `short:"d" description:"directory to put the pprof files in" default:"."` 22 | KeepGoing bool `short:"k" description:"keep going collect profiles if some requests failed"` 23 | } 24 | 25 | func main() { 26 | if _, err := flags.Parse(&cfg); err != nil { 27 | return 28 | } 29 | 30 | log.Println("collecting profiles. hit Ctrl-C any time to stop.") 31 | 32 | ctx, cancel := context.WithCancel(context.Background()) 33 | defer cancel() 34 | interrupt := make(chan os.Signal, 1) 35 | signal.Notify(interrupt, os.Interrupt) 36 | go func() { 37 | <-interrupt 38 | cancel() 39 | }() 40 | 41 | if !cfg.Loop { 42 | if err := collectAndDump(ctx, cfg.Directory, cfg.Hosts, cfg.Profiles, cfg.KeepGoing); err != nil { 43 | log.Fatalln(err) 44 | } 45 | 46 | return 47 | } 48 | 49 | for { 50 | if err := collectAndDump(ctx, cfg.Directory, cfg.Hosts, cfg.Profiles, cfg.KeepGoing); err != nil { 51 | if errors.Is(err, context.Canceled) { 52 | return 53 | } 54 | 55 | log.Fatalln(err) 56 | } 57 | 58 | log.Printf("sleeping for %v before next collect...", cfg.Interval) 59 | select { 60 | case <-time.After(cfg.Interval): 61 | case <-ctx.Done(): 62 | return 63 | } 64 | } 65 | } 66 | 67 | func collectAndDump(ctx context.Context, baseDir string, hosts []string, profilesToCollect []string, ignoreNetworkErrors bool) error { 68 | g, ctx := errgroup.WithContext(ctx) 69 | 70 | for _, host := range hosts { 71 | h := host 72 | g.Go(func() error { 73 | collected, err := profiles.Collect(ctx, h, profilesToCollect, ignoreNetworkErrors) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | return profiles.Dump(ctx, baseDir, h, collected) 79 | }) 80 | } 81 | 82 | return g.Wait() 83 | } 84 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tommsawyer/collect 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/jessevdk/go-flags v1.5.0 7 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= 2 | github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= 3 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 4 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 5 | golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= 6 | golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 7 | -------------------------------------------------------------------------------- /profiles/profiles.go: -------------------------------------------------------------------------------- 1 | package profiles 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "net/url" 12 | "os" 13 | "path" 14 | "strings" 15 | "sync" 16 | "time" 17 | 18 | "golang.org/x/sync/errgroup" 19 | ) 20 | 21 | // Dump will dump every profile into given folder following this structure: 22 | // - provided directory 23 | // - host port 24 | // - YYYY MM DD 25 | // - HH MM SS 26 | // - profile 27 | func Dump(ctx context.Context, dir, base string, profiles map[string][]byte) error { 28 | if len(profiles) == 0 { 29 | return nil 30 | } 31 | 32 | u, err := url.Parse(base) 33 | if err != nil { 34 | return err 35 | } 36 | 37 | folder := path.Join(dir, u.Hostname()+" "+u.Port()) + "/" + time.Now().Format("2006 01 02/15 04 05") 38 | if err := os.MkdirAll(folder, os.ModePerm); err != nil { 39 | return fmt.Errorf("cannot create directory %q: %w", folder, err) 40 | } 41 | 42 | for profile, content := range profiles { 43 | filePath := path.Join(folder, profile) 44 | if err := ioutil.WriteFile(filePath, content, os.ModePerm); err != nil { 45 | return fmt.Errorf("cannot write %q profile: %w", profile, err) 46 | } 47 | log.Printf("[%s] wrote %s to %s\n", base, profile, filePath) 48 | } 49 | 50 | return nil 51 | } 52 | 53 | // Collect will collect all provided profiles. 54 | // 55 | // You can add query parameters to profile like so: 56 | // 57 | // Collect(ctx, "http://localhost:8080", []string{"trace?seconds=5"}) 58 | func Collect(ctx context.Context, baseURL string, profiles []string, ignoreNetworkErrors bool) (map[string][]byte, error) { 59 | client := &http.Client{ 60 | Timeout: time.Minute, 61 | } 62 | 63 | var mx sync.Mutex 64 | collectedProfiles := make(map[string][]byte, len(profiles)) 65 | 66 | g, ctx := errgroup.WithContext(ctx) 67 | for _, profile := range profiles { 68 | url := baseURL + "/debug/pprof/" + profile 69 | profileName := strings.Split(profile, "?")[0] 70 | g.Go(func() error { 71 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 72 | if err != nil { 73 | return fmt.Errorf("cannot build url: %w", err) 74 | } 75 | 76 | log.Printf("[%s] collecting %s\n", baseURL, profileName) 77 | resp, err := client.Do(req) 78 | if err != nil { 79 | if ignoreNetworkErrors && !errors.Is(err, context.Canceled) { 80 | log.Printf("cannot collect %s: %s", profileName, err.Error()) 81 | return nil 82 | } 83 | return fmt.Errorf("cannot collect %s: %w", profileName, err) 84 | } 85 | defer resp.Body.Close() 86 | 87 | bytes, err := io.ReadAll(resp.Body) 88 | if err != nil { 89 | return fmt.Errorf("cannot collect %s: %w", profileName, err) 90 | } 91 | 92 | mx.Lock() 93 | collectedProfiles[profileName] = bytes 94 | mx.Unlock() 95 | 96 | log.Printf("[%s] successfully collected %s\n", baseURL, profileName) 97 | return nil 98 | }) 99 | } 100 | if err := g.Wait(); err != nil { 101 | return nil, err 102 | } 103 | 104 | return collectedProfiles, nil 105 | } 106 | -------------------------------------------------------------------------------- /profiles/profiles_test.go: -------------------------------------------------------------------------------- 1 | package profiles 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/fs" 7 | "log" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "path" 12 | "path/filepath" 13 | "strings" 14 | "testing" 15 | ) 16 | 17 | type testLogger struct { 18 | t *testing.T 19 | } 20 | 21 | func (w *testLogger) LogTo(t *testing.T) { 22 | w.t = t 23 | } 24 | 25 | func (w *testLogger) Write(msg []byte) (int, error) { 26 | w.t.Log(string(msg)) 27 | return len(msg), nil 28 | } 29 | 30 | func TestCollectProfiles(t *testing.T) { 31 | testLog := &testLogger{t} 32 | log.SetOutput(testLog) 33 | defer log.SetOutput(os.Stdout) 34 | 35 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 36 | fmt.Fprint(w, strings.TrimPrefix(r.URL.String(), "/debug/pprof/")) 37 | })) 38 | defer srv.Close() 39 | 40 | t.Run("collect one profile", func(t *testing.T) { 41 | testLog.LogTo(t) 42 | profiles, err := Collect(context.Background(), srv.URL, []string{"allocs"}, false) 43 | if err != nil { 44 | t.Errorf("error should be nil, but got %v", err) 45 | } 46 | 47 | if string(profiles["allocs"]) != "allocs" { 48 | t.Errorf("should download allocs profile but got %s", string(profiles["allocs"])) 49 | } 50 | }) 51 | 52 | t.Run("collect many profiles", func(t *testing.T) { 53 | testLog.LogTo(t) 54 | profiles, err := Collect(context.Background(), srv.URL, []string{"allocs", "profile"}, false) 55 | if err != nil { 56 | t.Errorf("error should be nil, but got %v", err) 57 | } 58 | 59 | if string(profiles["allocs"]) != "allocs" { 60 | t.Errorf("should download allocs profile but got %s", string(profiles["allocs"])) 61 | } 62 | if string(profiles["profile"]) != "profile" { 63 | t.Errorf("should dowlnoad cpu profile but got %s", string(profiles["profile"])) 64 | } 65 | }) 66 | 67 | t.Run("collect profile with query parameters", func(t *testing.T) { 68 | testLog.LogTo(t) 69 | profiles, err := Collect(context.Background(), srv.URL, []string{"trace?seconds=5"}, false) 70 | if err != nil { 71 | t.Errorf("error should be nil, but got %v", err) 72 | } 73 | 74 | if string(profiles["trace"]) != "trace?seconds=5" { 75 | t.Errorf("should download trace profile") 76 | } 77 | }) 78 | 79 | t.Run("returns error when cannot build url", func(t *testing.T) { 80 | _, err := Collect(context.Background(), "http://wrong.wrong\n", []string{"allocs"}, false) 81 | if err == nil || !strings.Contains(err.Error(), "cannot build url:") { 82 | t.Error("should return error when wrong url provided") 83 | } 84 | }) 85 | 86 | t.Run("returns error when server is unreachable", func(t *testing.T) { 87 | _, err := Collect(context.Background(), "http://wrong.wrong", []string{"allocs"}, false) 88 | if err == nil || !strings.Contains(err.Error(), "cannot collect allocs:") { 89 | t.Error("should return error when server is unreachable") 90 | } 91 | }) 92 | 93 | t.Run("returns no error when server is unreachable and ignore network errors is true", func(t *testing.T) { 94 | _, err := Collect(context.Background(), "http://wrong.wrong", []string{"allocs"}, true) 95 | if err != nil { 96 | t.Errorf("error should be nil, but got %v", err) 97 | } 98 | }) 99 | } 100 | 101 | func TestDump(t *testing.T) { 102 | testLog := &testLogger{t} 103 | log.SetOutput(testLog) 104 | defer log.SetOutput(os.Stdout) 105 | 106 | testDir := path.Join(os.TempDir(), "test") 107 | err := os.Mkdir(path.Join(os.TempDir(), "test"), os.ModePerm) 108 | if err != nil { 109 | t.Fatalf("cannot create test directory: %v", err) 110 | } 111 | defer os.RemoveAll(testDir) 112 | 113 | profiles := map[string][]byte{ 114 | "allocs": []byte("allocs"), 115 | "heap": []byte("heap"), 116 | } 117 | err = Dump(context.Background(), testDir, "http://localhost:8080", profiles) 118 | if err != nil { 119 | t.Fatalf("error should be nil, but got %v", err) 120 | } 121 | 122 | fileContents := map[string][]byte{} 123 | err = filepath.Walk(testDir, func(path string, info fs.FileInfo, err error) error { 124 | if !info.IsDir() { 125 | content, err := os.ReadFile(path) 126 | if err != nil { 127 | return err 128 | } 129 | fileContents[info.Name()] = content 130 | } 131 | return nil 132 | }) 133 | if err != nil { 134 | t.Fatalf("cannot walk through files: %v", err) 135 | } 136 | 137 | for profile, content := range profiles { 138 | if string(fileContents[profile]) != string(content) { 139 | t.Errorf("file %s contains wrong data: %q", profile, string(fileContents[profile])) 140 | } 141 | } 142 | } 143 | 144 | func TestDumpParseURL(t *testing.T) { 145 | testLog := &testLogger{t} 146 | log.SetOutput(testLog) 147 | defer log.SetOutput(os.Stdout) 148 | 149 | err := Dump(context.Background(), "./", "http://localhost:8080\n", map[string][]byte{ 150 | "test": []byte("test"), 151 | }) 152 | if err == nil { 153 | t.Fatalf("should return an error when cannot parse url") 154 | } 155 | } 156 | 157 | func TestDumpCannotCreateDirectory(t *testing.T) { 158 | testLog := &testLogger{t} 159 | log.SetOutput(testLog) 160 | defer log.SetOutput(os.Stdout) 161 | 162 | err := Dump(context.Background(), "/dev/null/:", "http://localhost:8080", map[string][]byte{ 163 | "test": []byte("test"), 164 | }) 165 | if err == nil { 166 | t.Fatalf("should return an error when cannot create directory") 167 | } 168 | } 169 | --------------------------------------------------------------------------------