├── .gitignore ├── LICENSE ├── README.md ├── example └── main.go ├── go.mod └── monitor.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # antifreeze 2 | 3 | antifreeze is a package that detects goroutines that have been waiting for too long. 4 | 5 | **Warning: this package is alpha and may not work as intended.** 6 | 7 | You can exclude functions that may block forever with `antifreeze.Exclude` and `antifreeze.ExcludeNamed`. 8 | 9 | Example program that will panic ~1min after you have made a request to it. 10 | 11 | ``` go 12 | package main 13 | 14 | import ( 15 | "fmt" 16 | "net/http" 17 | "time" 18 | 19 | "github.com/egonelbre/antifreeze" 20 | ) 21 | 22 | func init() { 23 | antifreeze.SetFrozenLimit(1 * time.Minute) 24 | } 25 | 26 | var ch = make(chan int) 27 | 28 | func runner() { 29 | antifreeze.Exclude() 30 | <-ch 31 | } 32 | 33 | func main() { 34 | go runner() 35 | http.Handle("/", http.HandlerFunc(index)) 36 | http.ListenAndServe(":8000", nil) 37 | } 38 | 39 | func index(w http.ResponseWriter, r *http.Request) { 40 | fmt.Println("R:", r) 41 | <-ch 42 | } 43 | ``` 44 | 45 | See [godoc](https://godoc.org/github.com/egonelbre/antifreeze) for more information. -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "sync" 7 | "time" 8 | 9 | "github.com/egonelbre/antifreeze" 10 | ) 11 | 12 | func init() { 13 | antifreeze.SetFrozenLimit(1 * time.Minute) 14 | } 15 | 16 | var ch = make(chan int) 17 | var x sync.Mutex 18 | 19 | func runner() { 20 | antifreeze.Exclude() 21 | <-ch 22 | } 23 | 24 | func main() { 25 | go runner() 26 | http.Handle("/", http.HandlerFunc(index)) 27 | http.Handle("/wait", http.HandlerFunc(wait)) 28 | http.Handle("/mutex", http.HandlerFunc(mutex)) 29 | log.Println("Make a request to either:") 30 | log.Println(" 127.0.0.1:8000/wait") 31 | log.Println(" 127.0.0.1:8000/mutex") 32 | http.ListenAndServe("127.0.0.1:8000", nil) 33 | } 34 | 35 | func index(w http.ResponseWriter, r *http.Request) {} 36 | 37 | func wait(w http.ResponseWriter, r *http.Request) { 38 | log.Println("Receive:", r.UserAgent()) 39 | <-ch 40 | } 41 | 42 | func mutex(w http.ResponseWriter, r *http.Request) { 43 | log.Println("Mutex:", r.UserAgent()) 44 | x.Lock() 45 | } 46 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/egonelbre/antifreeze 2 | 3 | go 1.17 4 | -------------------------------------------------------------------------------- /monitor.go: -------------------------------------------------------------------------------- 1 | package antifreeze 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "os" 8 | "runtime" 9 | "strconv" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const ( 16 | // DefaultBufferSize specifies how much buffer to use for reading stack 17 | // profiles by default. 18 | DefaultBufferSize = 8 << 20 19 | // DefaultFrozenLimit specifies when a goroutine should be considered stuck. 20 | DefaultFrozenLimit = 10 * time.Minute 21 | ) 22 | 23 | var ( 24 | mu sync.Mutex 25 | _limit = int(DefaultFrozenLimit / time.Minute) 26 | _buffer = make([]byte, DefaultBufferSize) 27 | _whitelist = make(map[string]struct{}) 28 | ) 29 | 30 | var faulting = map[string]struct{}{ 31 | "chan send": {}, 32 | "chan receive": {}, 33 | } 34 | 35 | // SetBufferSize sets the maximum buffer size for reading stack profiles. 36 | // Set this lower, e.g. 1<<10 (10MB) to reduce memory footprint at the cost of 37 | // possibly not tracking some goroutines. 38 | // Set it higher, e.g. 64<<10 (64MB) if you expect a lot of goroutines. 39 | func SetBufferSize(size int) { 40 | mu.Lock() 41 | _buffer = make([]byte, size) 42 | mu.Unlock() 43 | } 44 | 45 | // SetFrozenLimit sets the limit after which the goroutine should be considered 46 | // as frozen. 47 | func SetFrozenLimit(limit time.Duration) { 48 | mu.Lock() 49 | if limit < time.Minute { 50 | panic("limit must be larger than 1 minute") 51 | } 52 | _limit = int(limit / time.Minute) 53 | mu.Unlock() 54 | } 55 | 56 | // Exclude excludes all goroutines that contain current function in the callstack. 57 | func Exclude() { 58 | if pc, _, _, ok := runtime.Caller(1); ok { 59 | fn := runtime.FuncForPC(pc) 60 | if fn != nil { 61 | ExcludeNamed(fn.Name()) 62 | } 63 | } 64 | } 65 | 66 | // ExcludeNamed excludes all goroutines that contain the named func 67 | // The func name must be fully qualified. 68 | // 69 | // antifreeze.ExcludeNamed("main.runner") 70 | // antifreeze.ExcludeNamed("net/http.ListenAndServe") 71 | // antifreeze.ExcludeNamed("net.(*pollDesc).Wait") 72 | func ExcludeNamed(name string) { 73 | mu.Lock() 74 | next := make(map[string]struct{}, len(_whitelist)+1) 75 | for fnname := range _whitelist { 76 | next[fnname] = struct{}{} 77 | } 78 | next[name] = struct{}{} 79 | _whitelist = next 80 | mu.Unlock() 81 | } 82 | 83 | func params() (buffer []byte, limit int, whitelist map[string]struct{}) { 84 | mu.Lock() 85 | buffer = _buffer 86 | limit = _limit 87 | whitelist = _whitelist 88 | mu.Unlock() 89 | return 90 | } 91 | 92 | func init() { 93 | go monitor() 94 | } 95 | 96 | func monitor() { 97 | check() 98 | for range time.Tick(30 * time.Second) { 99 | check() 100 | } 101 | } 102 | 103 | func skiptrace(scanner *bufio.Scanner) { 104 | for scanner.Scan() && scanner.Text() == "" { 105 | return 106 | } 107 | } 108 | 109 | func isfaulting(kind string) bool { 110 | return kind == "chan send" || kind == "chan receive" || 111 | kind == "semacquire" 112 | } 113 | 114 | func check() { 115 | buf, limit, whitelist := params() 116 | n := runtime.Stack(buf, true) 117 | scanner := bufio.NewScanner(bytes.NewReader(buf[:n])) 118 | 119 | for scanner.Scan() { 120 | // goroutine 60 [chan receive, 1 minutes]: 121 | line := scanner.Text() 122 | if line == "" { 123 | continue 124 | } 125 | 126 | // has it been stuck at least 1 minute 127 | if !strings.Contains(line, "minutes") { 128 | skiptrace(scanner) 129 | continue 130 | } 131 | 132 | rest := line 133 | 134 | // strip "goroutine " 135 | rest = rest[10:] 136 | p := strings.IndexByte(rest, ' ') 137 | 138 | // extract id 139 | id := rest[:p] 140 | rest = rest[p+1:] 141 | 142 | p = strings.IndexByte(rest, ',') 143 | kind := rest[1:p] 144 | rest = rest[p+2:] 145 | 146 | if !isfaulting(kind) { 147 | skiptrace(scanner) 148 | continue 149 | } 150 | 151 | p = strings.IndexByte(rest, ' ') 152 | minutes_str := rest[:p] 153 | minutes, err := strconv.Atoi(minutes_str) 154 | if err != nil { 155 | skiptrace(scanner) 156 | continue 157 | } 158 | 159 | if minutes >= limit { 160 | whitelisted := false 161 | var outbuf bytes.Buffer 162 | fmt.Fprintln(&outbuf, line) 163 | for scanner.Scan() { 164 | line := scanner.Text() 165 | fmt.Fprintln(&outbuf, line) 166 | if line == "" { 167 | break 168 | } 169 | p := strings.LastIndexByte(line, '(') 170 | if p >= 0 { 171 | if _, exists := whitelist[line[:p]]; exists { 172 | whitelisted = true 173 | skiptrace(scanner) 174 | break 175 | } 176 | } 177 | } 178 | 179 | if !whitelisted { 180 | os.Stdout.Write(outbuf.Bytes()) 181 | panic("goroutine " + id + " was frozen") 182 | } 183 | } 184 | 185 | skiptrace(scanner) 186 | } 187 | } 188 | --------------------------------------------------------------------------------