├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── circuitbreaker.go ├── circuitbreaker_test.go ├── client.go ├── example_test.go ├── go.mod ├── go.sum ├── panel.go ├── panel_test.go ├── window.go └── window_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.test 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | ## 2.2.0 - 2016-08-09 5 | 6 | ### Added 7 | - Externally provided event listener channel (@spencerkimball) 8 | 9 | ### Deprecated 10 | - Nothing 11 | 12 | ### Removed 13 | - Nothing 14 | 15 | ### Fixed 16 | - Reduce allocations around last failure time storage 17 | - Use the Clock for window code as well 18 | - Remove test data race 19 | - Fix race condition in `state()` (@tamird) 20 | 21 | ## 2.1.7 - 2016-07-27 22 | 23 | ### Added 24 | - Nothing 25 | 26 | ### Deprecated 27 | - Nothing 28 | 29 | ### Removed 30 | - Nothing 31 | 32 | ### Fixed 33 | - Set `Backoff.MaxElapsedTime` to 0 as default [@matope] 34 | - Use a lock when modifying `nextBackoff` 35 | - Fix goroutine leak when using timeouts [@isaldana] 36 | - Fix window buckets that should be empty [@isaldana] 37 | - Update backoff package, which has been renamed 38 | 39 | ## 2.1.6 - 2016-02-02 40 | 41 | ### Added 42 | - Nothing 43 | 44 | ### Deprecated 45 | - Nothing 46 | 47 | ### Removed 48 | - Nothing 49 | 50 | ### Fixed 51 | - client.Do() was not returning the error when it timed out [@ryanmurf] 52 | 53 | ## 2.1.5 - 2015-11-19 54 | 55 | ### Added 56 | - Nothing 57 | 58 | ### Deprecated 59 | - Nothing 60 | 61 | ### Removed 62 | - Nothing 63 | 64 | ### Fixed 65 | - Respect backoff.Stop [@bc-vincent-zhao] 66 | 67 | ## 2.1.4 - 2015-09-01 68 | 69 | ### Added 70 | - Nothing 71 | 72 | ### Deprecated 73 | - Nothing 74 | 75 | ### Removed 76 | - Nothing 77 | 78 | ### Fixed 79 | - HTTP client was using a new panel object instead of the one it added the breaker to [@ryanmurf] 80 | 81 | ## 2.1.3 - 2015-08-05 82 | 83 | ### Added 84 | - Configurable bucket time and number [@thraxil] 85 | - Use mock clock for test [@andreas] 86 | 87 | ### Deprecated 88 | - Nothing 89 | 90 | ### Removed 91 | - Nothing 92 | 93 | ### Fixed 94 | - Bug in statsd bucket name documentation / example [@thraxil] 95 | 96 | ## 2.1.2 - 2015-04-03 97 | 98 | ### Added 99 | - Nothing 100 | 101 | ### Deprecated 102 | - Nothing 103 | 104 | ### Removed 105 | - Nothing 106 | 107 | ### Fixed 108 | - Simplify Call() for rate breaker, fixing a reset bug 109 | 110 | ## 2.1.1 - 2014-10-29 111 | 112 | ### Added 113 | - Nothing 114 | 115 | ### Deprecated 116 | - Nothing 117 | 118 | ### Removed 119 | - Nothing 120 | 121 | ### Fixed 122 | - Ensure the half opens counter resets when the breaker resets, or auto-resetting may not occur 123 | 124 | ## 2.1.0 - 2014-10-16 125 | 126 | ### Added 127 | - Failure, Sucess counts and Error Rate is now calculated over a sliding window 128 | - Number of buckets in the window and the time the window spans are tuneable 129 | 130 | ### Deprecated 131 | - Nothing 132 | 133 | ### Removed 134 | - Nothing 135 | 136 | ### Fixed 137 | - A race condition in Call() 138 | 139 | ## 2.0.2 - 2014-10-13 140 | 141 | ### Added 142 | - ResetCounters 143 | 144 | ### Deprecated 145 | - Nothing 146 | 147 | ### Removed 148 | - Nothing 149 | 150 | ### Fixed 151 | - Nothing 152 | 153 | ## 2.0.1 - 2014-10-13 154 | 155 | ### Added 156 | - Nothing 157 | 158 | ### Deprecated 159 | - Nothing 160 | 161 | ### Removed 162 | - Nothing 163 | 164 | ### Fixed 165 | - Error rate should return 0.0 if there have been no samples 166 | 167 | ## 2.0.0 - 2014-10-13 168 | 169 | ### Added 170 | - All circuit breakers are now a Breaker with trip semantics handled by a TripFunc 171 | - NewConsecutiveBreaker 172 | - NewRateBreaker 173 | - ConsecFailures 174 | - ErrorRate 175 | - Success 176 | - Successes 177 | - Retry logic now uses cenkalti/backoff, exponential backoff by default 178 | 179 | ### Deprecated 180 | - Nothing 181 | 182 | ### Removed 183 | - TrippableBreaker, ThresholdBreaker, FrequencyBreaker, TimeoutBreaker; all handled by Breaker now 184 | - NewFrequencyBreaker, replaced by NewConsecutiveBreaker 185 | - NewTimeoutBreaker, time out semantics are now handled by Call() 186 | - NoOp(), use a Breaker with no TripFunc instead 187 | 188 | ### Fixed 189 | - Nothing 190 | 191 | ## 1.1.2 - 2014-08-20 192 | 193 | ### Added 194 | - Nothing 195 | 196 | ### Deprecated 197 | - Nothing 198 | 199 | ### Fixed 200 | - For a FrequencyBreaker, Failures() should return the count since the duration start, even after resetting. 201 | 202 | ## 1.1.1 - 2014-08-20 203 | 204 | ### Added 205 | - Nothing 206 | 207 | ### Deprecated 208 | - Nothing 209 | 210 | ### Fixed 211 | - Only send the reset event if the breaker was in a tripped state 212 | 213 | ## 1.1.0 - 2014-08-16 214 | 215 | ### Added 216 | - Re-export a Panels Circuits map. It's handy and if you mess it up, it's on you. 217 | 218 | ### Deprecated 219 | - Nothing 220 | 221 | ### Removed 222 | - Nothing 223 | 224 | ### Fixed 225 | - Nothing 226 | 227 | ## 1.0.0 - 2014-08-16 228 | 229 | ### Added 230 | - This will be the public API for version 1.0.0. This project will follow semver rules. 231 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2014 Scott Barron 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # circuitbreaker 2 | 3 | Circuitbreaker provides an easy way to use the Circuit Breaker pattern in a 4 | Go program. 5 | 6 | Circuit breakers are typically used when your program makes remote calls. 7 | Remote calls can often hang for a while before they time out. If your 8 | application makes a lot of these requests, many resources can be tied 9 | up waiting for these time outs to occur. A circuit breaker wraps these 10 | remote calls and will trip after a defined amount of failures or time outs 11 | occur. When a circuit breaker is tripped any future calls will avoid making 12 | the remote call and return an error to the caller. In the meantime, the 13 | circuit breaker will periodically allow some calls to be tried again and 14 | will close the circuit if those are successful. 15 | 16 | You can read more about this pattern and how it's used at: 17 | - [Martin Fowler's bliki](http://martinfowler.com/bliki/CircuitBreaker.html) 18 | - [The Netflix Tech Blog](http://techblog.netflix.com/2012/02/fault-tolerance-in-high-volume.html) 19 | - [Release It!](http://pragprog.com/book/mnee/release-it) 20 | 21 | [![GoDoc](https://godoc.org/github.com/rubyist/circuitbreaker?status.svg)](https://godoc.org/github.com/rubyist/circuitbreaker) 22 | 23 | ## Installation 24 | 25 | ``` 26 | go get github.com/rubyist/circuitbreaker 27 | ``` 28 | 29 | ## Examples 30 | 31 | Here is a quick example of what circuitbreaker provides 32 | 33 | ```go 34 | // Creates a circuit breaker that will trip if the function fails 10 times 35 | cb := circuit.NewThresholdBreaker(10) 36 | 37 | events := cb.Subscribe() 38 | go func() { 39 | for { 40 | e := <-events 41 | // Monitor breaker events like BreakerTripped, BreakerReset, BreakerFail, BreakerReady 42 | } 43 | }() 44 | 45 | cb.Call(func() error { 46 | // This is where you'll do some remote call 47 | // If it fails, return an error 48 | }, 0) 49 | ``` 50 | 51 | Circuitbreaker can also wrap a time out around the remote call. 52 | 53 | ```go 54 | // Creates a circuit breaker that will trip after 10 failures 55 | // using a time out of 5 seconds 56 | cb := circuit.NewThresholdBreaker(10) 57 | 58 | cb.Call(func() error { 59 | // This is where you'll do some remote call 60 | // If it fails, return an error 61 | }, time.Second * 5) // This will time out after 5 seconds, which counts as a failure 62 | 63 | // Proceed as above 64 | 65 | ``` 66 | 67 | Circuitbreaker can also trip based on the number of consecutive failures. 68 | 69 | ```go 70 | // Creates a circuit breaker that will trip if 10 consecutive failures occur 71 | cb := circuit.NewConsecutiveBreaker(10) 72 | 73 | // Proceed as above 74 | ``` 75 | 76 | Circuitbreaker can trip based on the error rate. 77 | 78 | ```go 79 | // Creates a circuit breaker based on the error rate 80 | cb := circuit.NewRateBreaker(0.95, 100) // trip when error rate hits 95%, with at least 100 samples 81 | 82 | // Proceed as above 83 | ``` 84 | 85 | If it doesn't make sense to wrap logic in Call(), breakers can be handled manually. 86 | 87 | ```go 88 | cb := circuit.NewThresholdBreaker(10) 89 | 90 | for { 91 | if cb.Ready() { 92 | // Breaker is not tripped, proceed 93 | err := doSomething() 94 | if err != nil { 95 | cb.Fail() // This will trip the breaker once it's failed 10 times 96 | continue 97 | } 98 | cb.Success() 99 | } else { 100 | // Breaker is in a tripped state. 101 | } 102 | } 103 | ``` 104 | 105 | Circuitbreaker also provides a wrapper around `http.Client` that will wrap a 106 | time out around any request. 107 | 108 | ```go 109 | // Passing in nil will create a regular http.Client. 110 | // You can also build your own http.Client and pass it in 111 | client := circuit.NewHTTPClient(time.Second * 5, 10, nil) 112 | 113 | resp, err := client.Get("http://example.com/resource.json") 114 | ``` 115 | 116 | See the godoc for more examples. 117 | 118 | ## Bugs, Issues, Feedback 119 | 120 | Right here on GitHub: [https://github.com/rubyist/circuitbreaker](https://github.com/rubyist/circuitbreaker) 121 | -------------------------------------------------------------------------------- /circuitbreaker.go: -------------------------------------------------------------------------------- 1 | // Package circuit implements the Circuit Breaker pattern. It will wrap 2 | // a function call (typically one which uses remote services) and monitors for 3 | // failures and/or time outs. When a threshold of failures or time outs has been 4 | // reached, future calls to the function will not run. During this state, the 5 | // breaker will periodically allow the function to run and, if it is successful, 6 | // will start running the function again. 7 | // 8 | // Circuit includes three types of circuit breakers: 9 | // 10 | // A Threshold Breaker will trip when the failure count reaches a given threshold. 11 | // It does not matter how long it takes to reach the threshold and the failures do 12 | // not need to be consecutive. 13 | // 14 | // A Consecutive Breaker will trip when the consecutive failure count reaches a given 15 | // threshold. It does not matter how long it takes to reach the threshold, but the 16 | // failures do need to be consecutive. 17 | // 18 | // 19 | // When wrapping blocks of code with a Breaker's Call() function, a time out can be 20 | // specified. If the time out is reached, the breaker's Fail() function will be called. 21 | // 22 | // 23 | // Other types of circuit breakers can be easily built by creating a Breaker and 24 | // adding a custom TripFunc. A TripFunc is called when a Breaker Fail()s and receives 25 | // the breaker as an argument. It then returns true or false to indicate whether the 26 | // breaker should trip. 27 | // 28 | // The package also provides a wrapper around an http.Client that wraps all of 29 | // the http.Client functions with a Breaker. 30 | // 31 | package circuit 32 | 33 | import ( 34 | "context" 35 | "errors" 36 | "sync" 37 | "sync/atomic" 38 | "time" 39 | 40 | "github.com/cenkalti/backoff" 41 | "github.com/facebookgo/clock" 42 | ) 43 | 44 | // BreakerEvent indicates the type of event received over an event channel 45 | type BreakerEvent int 46 | 47 | const ( 48 | // BreakerTripped is sent when a breaker trips 49 | BreakerTripped BreakerEvent = iota 50 | 51 | // BreakerReset is sent when a breaker resets 52 | BreakerReset BreakerEvent = iota 53 | 54 | // BreakerFail is sent when Fail() is called 55 | BreakerFail BreakerEvent = iota 56 | 57 | // BreakerReady is sent when the breaker enters the half open state and is ready to retry 58 | BreakerReady BreakerEvent = iota 59 | ) 60 | 61 | // ListenerEvent includes a reference to the circuit breaker and the event. 62 | type ListenerEvent struct { 63 | CB *Breaker 64 | Event BreakerEvent 65 | } 66 | 67 | type state int 68 | 69 | const ( 70 | open state = iota 71 | halfopen state = iota 72 | closed state = iota 73 | ) 74 | 75 | var ( 76 | defaultInitialBackOffInterval = 500 * time.Millisecond 77 | defaultBackoffMaxElapsedTime = 0 * time.Second 78 | ) 79 | 80 | // Error codes returned by Call 81 | var ( 82 | ErrBreakerOpen = errors.New("breaker open") 83 | ErrBreakerTimeout = errors.New("breaker time out") 84 | ) 85 | 86 | // TripFunc is a function called by a Breaker's Fail() function and determines whether 87 | // the breaker should trip. It will receive the Breaker as an argument and returns a 88 | // boolean. By default, a Breaker has no TripFunc. 89 | type TripFunc func(*Breaker) bool 90 | 91 | // Breaker is the base of a circuit breaker. It maintains failure and success counters 92 | // as well as the event subscribers. 93 | type Breaker struct { 94 | // BackOff is the backoff policy that is used when determining if the breaker should 95 | // attempt to retry. A breaker created with NewBreaker will use an exponential backoff 96 | // policy by default. 97 | BackOff backoff.BackOff 98 | 99 | // ShouldTrip is a TripFunc that determines whether a Fail() call should trip the breaker. 100 | // A breaker created with NewBreaker will not have a ShouldTrip by default, and thus will 101 | // never automatically trip. 102 | ShouldTrip TripFunc 103 | 104 | // Clock is used for controlling time in tests. 105 | Clock clock.Clock 106 | 107 | _ [4]byte // pad to fix golang issue #599 108 | consecFailures int64 109 | lastFailure int64 // stored as nanoseconds since the Unix epoch 110 | halfOpens int64 111 | counts *window 112 | nextBackOff time.Duration 113 | tripped int32 114 | broken int32 115 | eventReceivers []chan BreakerEvent 116 | listeners []chan ListenerEvent 117 | backoffLock sync.Mutex 118 | } 119 | 120 | // Options holds breaker configuration options. 121 | type Options struct { 122 | BackOff backoff.BackOff 123 | Clock clock.Clock 124 | ShouldTrip TripFunc 125 | WindowTime time.Duration 126 | WindowBuckets int 127 | } 128 | 129 | // NewBreakerWithOptions creates a base breaker with a specified backoff, clock and TripFunc 130 | func NewBreakerWithOptions(options *Options) *Breaker { 131 | if options == nil { 132 | options = &Options{} 133 | } 134 | 135 | if options.Clock == nil { 136 | options.Clock = clock.New() 137 | } 138 | 139 | if options.BackOff == nil { 140 | b := backoff.NewExponentialBackOff() 141 | b.InitialInterval = defaultInitialBackOffInterval 142 | b.MaxElapsedTime = defaultBackoffMaxElapsedTime 143 | b.Clock = options.Clock 144 | b.Reset() 145 | options.BackOff = b 146 | } 147 | 148 | if options.WindowTime == 0 { 149 | options.WindowTime = DefaultWindowTime 150 | } 151 | 152 | if options.WindowBuckets == 0 { 153 | options.WindowBuckets = DefaultWindowBuckets 154 | } 155 | 156 | return &Breaker{ 157 | BackOff: options.BackOff, 158 | Clock: options.Clock, 159 | ShouldTrip: options.ShouldTrip, 160 | nextBackOff: options.BackOff.NextBackOff(), 161 | counts: newWindow(options.WindowTime, options.WindowBuckets), 162 | } 163 | } 164 | 165 | // NewBreaker creates a base breaker with an exponential backoff and no TripFunc 166 | func NewBreaker() *Breaker { 167 | return NewBreakerWithOptions(nil) 168 | } 169 | 170 | // NewThresholdBreaker creates a Breaker with a ThresholdTripFunc. 171 | func NewThresholdBreaker(threshold int64) *Breaker { 172 | return NewBreakerWithOptions(&Options{ 173 | ShouldTrip: ThresholdTripFunc(threshold), 174 | }) 175 | } 176 | 177 | // NewConsecutiveBreaker creates a Breaker with a ConsecutiveTripFunc. 178 | func NewConsecutiveBreaker(threshold int64) *Breaker { 179 | return NewBreakerWithOptions(&Options{ 180 | ShouldTrip: ConsecutiveTripFunc(threshold), 181 | }) 182 | } 183 | 184 | // NewRateBreaker creates a Breaker with a RateTripFunc. 185 | func NewRateBreaker(rate float64, minSamples int64) *Breaker { 186 | return NewBreakerWithOptions(&Options{ 187 | ShouldTrip: RateTripFunc(rate, minSamples), 188 | }) 189 | } 190 | 191 | // Subscribe returns a channel of BreakerEvents. Whenever the breaker changes state, 192 | // the state will be sent over the channel. See BreakerEvent for the types of events. 193 | func (cb *Breaker) Subscribe() <-chan BreakerEvent { 194 | eventReader := make(chan BreakerEvent) 195 | output := make(chan BreakerEvent, 100) 196 | 197 | go func() { 198 | for v := range eventReader { 199 | select { 200 | case output <- v: 201 | default: 202 | <-output 203 | output <- v 204 | } 205 | } 206 | }() 207 | cb.eventReceivers = append(cb.eventReceivers, eventReader) 208 | return output 209 | } 210 | 211 | // AddListener adds a channel of ListenerEvents on behalf of a listener. 212 | // The listener channel must be buffered. 213 | func (cb *Breaker) AddListener(listener chan ListenerEvent) { 214 | cb.listeners = append(cb.listeners, listener) 215 | } 216 | 217 | // RemoveListener removes a channel previously added via AddListener. 218 | // Once removed, the channel will no longer receive ListenerEvents. 219 | // Returns true if the listener was found and removed. 220 | func (cb *Breaker) RemoveListener(listener chan ListenerEvent) bool { 221 | for i, receiver := range cb.listeners { 222 | if listener == receiver { 223 | cb.listeners = append(cb.listeners[:i], cb.listeners[i+1:]...) 224 | return true 225 | } 226 | } 227 | return false 228 | } 229 | 230 | // Trip will trip the circuit breaker. After Trip() is called, Tripped() will 231 | // return true. 232 | func (cb *Breaker) Trip() { 233 | atomic.StoreInt32(&cb.tripped, 1) 234 | now := cb.Clock.Now() 235 | atomic.StoreInt64(&cb.lastFailure, now.UnixNano()) 236 | cb.sendEvent(BreakerTripped) 237 | } 238 | 239 | // Reset will reset the circuit breaker. After Reset() is called, Tripped() will 240 | // return false. 241 | func (cb *Breaker) Reset() { 242 | atomic.StoreInt32(&cb.broken, 0) 243 | atomic.StoreInt32(&cb.tripped, 0) 244 | atomic.StoreInt64(&cb.halfOpens, 0) 245 | cb.ResetCounters() 246 | cb.sendEvent(BreakerReset) 247 | } 248 | 249 | // ResetCounters will reset only the failures, consecFailures, and success counters 250 | func (cb *Breaker) ResetCounters() { 251 | atomic.StoreInt64(&cb.consecFailures, 0) 252 | cb.counts.Reset() 253 | } 254 | 255 | // Tripped returns true if the circuit breaker is tripped, false if it is reset. 256 | func (cb *Breaker) Tripped() bool { 257 | return atomic.LoadInt32(&cb.tripped) == 1 258 | } 259 | 260 | // Break trips the circuit breaker and prevents it from auto resetting. Use this when 261 | // manual control over the circuit breaker state is needed. 262 | func (cb *Breaker) Break() { 263 | atomic.StoreInt32(&cb.broken, 1) 264 | cb.Trip() 265 | } 266 | 267 | // Failures returns the number of failures for this circuit breaker. 268 | func (cb *Breaker) Failures() int64 { 269 | return cb.counts.Failures() 270 | } 271 | 272 | // ConsecFailures returns the number of consecutive failures that have occured. 273 | func (cb *Breaker) ConsecFailures() int64 { 274 | return atomic.LoadInt64(&cb.consecFailures) 275 | } 276 | 277 | // Successes returns the number of successes for this circuit breaker. 278 | func (cb *Breaker) Successes() int64 { 279 | return cb.counts.Successes() 280 | } 281 | 282 | // Fail is used to indicate a failure condition the Breaker should record. It will 283 | // increment the failure counters and store the time of the last failure. If the 284 | // breaker has a TripFunc it will be called, tripping the breaker if necessary. 285 | func (cb *Breaker) Fail() { 286 | cb.counts.Fail() 287 | atomic.AddInt64(&cb.consecFailures, 1) 288 | now := cb.Clock.Now() 289 | atomic.StoreInt64(&cb.lastFailure, now.UnixNano()) 290 | cb.sendEvent(BreakerFail) 291 | if cb.ShouldTrip != nil && cb.ShouldTrip(cb) { 292 | cb.Trip() 293 | } 294 | } 295 | 296 | // Success is used to indicate a success condition the Breaker should record. If 297 | // the success was triggered by a retry attempt, the breaker will be Reset(). 298 | func (cb *Breaker) Success() { 299 | cb.backoffLock.Lock() 300 | cb.BackOff.Reset() 301 | cb.nextBackOff = cb.BackOff.NextBackOff() 302 | cb.backoffLock.Unlock() 303 | 304 | state := cb.state() 305 | if state == halfopen { 306 | cb.Reset() 307 | } 308 | atomic.StoreInt64(&cb.consecFailures, 0) 309 | cb.counts.Success() 310 | } 311 | 312 | // ErrorRate returns the current error rate of the Breaker, expressed as a floating 313 | // point number (e.g. 0.9 for 90%), since the last time the breaker was Reset. 314 | func (cb *Breaker) ErrorRate() float64 { 315 | return cb.counts.ErrorRate() 316 | } 317 | 318 | // Ready will return true if the circuit breaker is ready to call the function. 319 | // It will be ready if the breaker is in a reset state, or if it is time to retry 320 | // the call for auto resetting. 321 | func (cb *Breaker) Ready() bool { 322 | state := cb.state() 323 | if state == halfopen { 324 | atomic.StoreInt64(&cb.halfOpens, 0) 325 | cb.sendEvent(BreakerReady) 326 | } 327 | return state == closed || state == halfopen 328 | } 329 | 330 | // Call wraps a function the Breaker will protect. A failure is recorded 331 | // whenever the function returns an error. If the called function takes longer 332 | // than timeout to run, a failure will be recorded. 333 | func (cb *Breaker) Call(circuit func() error, timeout time.Duration) error { 334 | return cb.CallContext(context.Background(), circuit, timeout) 335 | } 336 | 337 | // CallContext is same as Call but if the ctx is canceled after the circuit returned an error, 338 | // the error will not be marked as a failure because the call was canceled intentionally. 339 | func (cb *Breaker) CallContext(ctx context.Context, circuit func() error, timeout time.Duration) error { 340 | var err error 341 | 342 | if !cb.Ready() { 343 | return ErrBreakerOpen 344 | } 345 | 346 | if timeout == 0 { 347 | err = circuit() 348 | } else { 349 | c := make(chan error, 1) 350 | go func() { 351 | c <- circuit() 352 | close(c) 353 | }() 354 | 355 | select { 356 | case e := <-c: 357 | err = e 358 | case <-cb.Clock.After(timeout): 359 | err = ErrBreakerTimeout 360 | } 361 | } 362 | 363 | if err != nil { 364 | if ctx.Err() != context.Canceled { 365 | cb.Fail() 366 | } 367 | return err 368 | } 369 | 370 | cb.Success() 371 | return nil 372 | } 373 | 374 | // state returns the state of the TrippableBreaker. The states available are: 375 | // closed - the circuit is in a reset state and is operational 376 | // open - the circuit is in a tripped state 377 | // halfopen - the circuit is in a tripped state but the reset timeout has passed 378 | func (cb *Breaker) state() state { 379 | tripped := cb.Tripped() 380 | if tripped { 381 | if atomic.LoadInt32(&cb.broken) == 1 { 382 | return open 383 | } 384 | 385 | last := atomic.LoadInt64(&cb.lastFailure) 386 | since := cb.Clock.Now().Sub(time.Unix(0, last)) 387 | 388 | cb.backoffLock.Lock() 389 | defer cb.backoffLock.Unlock() 390 | 391 | if cb.nextBackOff != backoff.Stop && since > cb.nextBackOff { 392 | if atomic.CompareAndSwapInt64(&cb.halfOpens, 0, 1) { 393 | cb.nextBackOff = cb.BackOff.NextBackOff() 394 | return halfopen 395 | } 396 | return open 397 | } 398 | return open 399 | } 400 | return closed 401 | } 402 | 403 | func (cb *Breaker) sendEvent(event BreakerEvent) { 404 | for _, receiver := range cb.eventReceivers { 405 | receiver <- event 406 | } 407 | for _, listener := range cb.listeners { 408 | le := ListenerEvent{CB: cb, Event: event} 409 | select { 410 | case listener <- le: 411 | default: 412 | <-listener 413 | listener <- le 414 | } 415 | } 416 | } 417 | 418 | // ThresholdTripFunc returns a TripFunc with that trips whenever 419 | // the failure count meets the threshold. 420 | func ThresholdTripFunc(threshold int64) TripFunc { 421 | return func(cb *Breaker) bool { 422 | return cb.Failures() == threshold 423 | } 424 | } 425 | 426 | // ConsecutiveTripFunc returns a TripFunc that trips whenever 427 | // the consecutive failure count meets the threshold. 428 | func ConsecutiveTripFunc(threshold int64) TripFunc { 429 | return func(cb *Breaker) bool { 430 | return cb.ConsecFailures() == threshold 431 | } 432 | } 433 | 434 | // RateTripFunc returns a TripFunc that trips whenever the 435 | // error rate hits the threshold. The error rate is calculated as such: 436 | // f = number of failures 437 | // s = number of successes 438 | // e = f / (f + s) 439 | // The error rate is calculated over a sliding window of 10 seconds (by default) 440 | // This TripFunc will not trip until there have been at least minSamples events. 441 | func RateTripFunc(rate float64, minSamples int64) TripFunc { 442 | return func(cb *Breaker) bool { 443 | samples := cb.Failures() + cb.Successes() 444 | return samples >= minSamples && cb.ErrorRate() >= rate 445 | } 446 | } 447 | -------------------------------------------------------------------------------- /circuitbreaker_test.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync/atomic" 7 | "testing" 8 | "time" 9 | 10 | "github.com/cenkalti/backoff" 11 | "github.com/facebookgo/clock" 12 | ) 13 | 14 | func init() { 15 | defaultInitialBackOffInterval = time.Millisecond 16 | } 17 | 18 | func TestBreakerTripping(t *testing.T) { 19 | cb := NewBreaker() 20 | 21 | if cb.Tripped() { 22 | t.Fatal("expected breaker to not be tripped") 23 | } 24 | 25 | cb.Trip() 26 | if !cb.Tripped() { 27 | t.Fatal("expected breaker to be tripped") 28 | } 29 | 30 | cb.Reset() 31 | if cb.Tripped() { 32 | t.Fatal("expected breaker to have been reset") 33 | } 34 | } 35 | 36 | func TestBreakerCounts(t *testing.T) { 37 | cb := NewBreaker() 38 | 39 | cb.Fail() 40 | if failures := cb.Failures(); failures != 1 { 41 | t.Fatalf("expected failure count to be 1, got %d", failures) 42 | } 43 | 44 | cb.Fail() 45 | if consecFailures := cb.ConsecFailures(); consecFailures != 2 { 46 | t.Fatalf("expected 2 consecutive failures, got %d", consecFailures) 47 | } 48 | 49 | cb.Success() 50 | if successes := cb.Successes(); successes != 1 { 51 | t.Fatalf("expected success count to be 1, got %d", successes) 52 | } 53 | if consecFailures := cb.ConsecFailures(); consecFailures != 0 { 54 | t.Fatalf("expected 0 consecutive failures, got %d", consecFailures) 55 | } 56 | 57 | cb.Reset() 58 | if failures := cb.Failures(); failures != 0 { 59 | t.Fatalf("expected failure count to be 0, got %d", failures) 60 | } 61 | if successes := cb.Successes(); successes != 0 { 62 | t.Fatalf("expected success count to be 0, got %d", successes) 63 | } 64 | if consecFailures := cb.ConsecFailures(); consecFailures != 0 { 65 | t.Fatalf("expected 0 consecutive failures, got %d", consecFailures) 66 | } 67 | } 68 | 69 | func TestErrorRate(t *testing.T) { 70 | cb := NewBreaker() 71 | if er := cb.ErrorRate(); er != 0.0 { 72 | t.Fatalf("expected breaker with no samples to have 0 error rate, got %f", er) 73 | } 74 | } 75 | 76 | func TestBreakerEvents(t *testing.T) { 77 | c := clock.NewMock() 78 | cb := NewBreaker() 79 | cb.Clock = c 80 | events := cb.Subscribe() 81 | 82 | cb.Trip() 83 | if e := <-events; e != BreakerTripped { 84 | t.Fatalf("expected to receive a trip event, got %d", e) 85 | } 86 | 87 | c.Add(cb.nextBackOff + 1) 88 | cb.Ready() 89 | if e := <-events; e != BreakerReady { 90 | t.Fatalf("expected to receive a breaker ready event, got %d", e) 91 | } 92 | 93 | cb.Reset() 94 | if e := <-events; e != BreakerReset { 95 | t.Fatalf("expected to receive a reset event, got %d", e) 96 | } 97 | 98 | cb.Fail() 99 | if e := <-events; e != BreakerFail { 100 | t.Fatalf("expected to receive a fail event, got %d", e) 101 | } 102 | } 103 | 104 | func TestAddRemoveListener(t *testing.T) { 105 | c := clock.NewMock() 106 | cb := NewBreaker() 107 | cb.Clock = c 108 | events := make(chan ListenerEvent, 100) 109 | cb.AddListener(events) 110 | 111 | cb.Trip() 112 | if e := <-events; e.Event != BreakerTripped { 113 | t.Fatalf("expected to receive a trip event, got %v", e) 114 | } 115 | 116 | c.Add(cb.nextBackOff + 1) 117 | cb.Ready() 118 | if e := <-events; e.Event != BreakerReady { 119 | t.Fatalf("expected to receive a breaker ready event, got %v", e) 120 | } 121 | 122 | cb.Reset() 123 | if e := <-events; e.Event != BreakerReset { 124 | t.Fatalf("expected to receive a reset event, got %v", e) 125 | } 126 | 127 | cb.Fail() 128 | if e := <-events; e.Event != BreakerFail { 129 | t.Fatalf("expected to receive a fail event, got %v", e) 130 | } 131 | 132 | cb.RemoveListener(events) 133 | cb.Reset() 134 | select { 135 | case e := <-events: 136 | t.Fatalf("after removing listener, should not receive reset event; got %v", e) 137 | default: 138 | // Expected. 139 | } 140 | } 141 | 142 | func TestTrippableBreakerState(t *testing.T) { 143 | c := clock.NewMock() 144 | cb := NewBreaker() 145 | cb.Clock = c 146 | 147 | if !cb.Ready() { 148 | t.Fatal("expected breaker to be ready") 149 | } 150 | 151 | cb.Trip() 152 | if cb.Ready() { 153 | t.Fatal("expected breaker to not be ready") 154 | } 155 | c.Add(cb.nextBackOff + 1) 156 | if !cb.Ready() { 157 | t.Fatal("expected breaker to be ready after reset timeout") 158 | } 159 | 160 | cb.Fail() 161 | c.Add(cb.nextBackOff + 1) 162 | if !cb.Ready() { 163 | t.Fatal("expected breaker to be ready after reset timeout, post failure") 164 | } 165 | } 166 | 167 | func TestTrippableBreakerManualBreak(t *testing.T) { 168 | c := clock.NewMock() 169 | cb := NewBreaker() 170 | cb.Clock = c 171 | cb.Break() 172 | c.Add(cb.nextBackOff + 1) 173 | 174 | if cb.Ready() { 175 | t.Fatal("expected breaker to still be tripped") 176 | } 177 | 178 | cb.Reset() 179 | cb.Trip() 180 | c.Add(cb.nextBackOff + 1) 181 | if !cb.Ready() { 182 | t.Fatal("expected breaker to be ready") 183 | } 184 | } 185 | 186 | func TestThresholdBreaker(t *testing.T) { 187 | cb := NewThresholdBreaker(2) 188 | 189 | if cb.Tripped() { 190 | t.Fatal("expected threshold breaker to be open") 191 | } 192 | 193 | cb.Fail() 194 | if cb.Tripped() { 195 | t.Fatal("expected threshold breaker to still be open") 196 | } 197 | 198 | cb.Fail() 199 | if !cb.Tripped() { 200 | t.Fatal("expected threshold breaker to be tripped") 201 | } 202 | 203 | cb.Reset() 204 | if failures := cb.Failures(); failures != 0 { 205 | t.Fatalf("expected reset to set failures to 0, got %d", failures) 206 | } 207 | if cb.Tripped() { 208 | t.Fatal("expected threshold breaker to be open") 209 | } 210 | } 211 | 212 | func TestConsecutiveBreaker(t *testing.T) { 213 | cb := NewConsecutiveBreaker(3) 214 | 215 | if cb.Tripped() { 216 | t.Fatal("expected consecutive breaker to be open") 217 | } 218 | 219 | cb.Fail() 220 | cb.Success() 221 | cb.Fail() 222 | cb.Fail() 223 | if cb.Tripped() { 224 | t.Fatal("expected consecutive breaker to be open") 225 | } 226 | cb.Fail() 227 | if !cb.Tripped() { 228 | t.Fatal("expected consecutive breaker to be tripped") 229 | } 230 | } 231 | 232 | func TestThresholdBreakerCalling(t *testing.T) { 233 | circuit := func() error { 234 | return fmt.Errorf("error") 235 | } 236 | 237 | cb := NewThresholdBreaker(2) 238 | 239 | err := cb.Call(circuit, 0) // First failure 240 | if err == nil { 241 | t.Fatal("expected threshold breaker to error") 242 | } 243 | if cb.Tripped() { 244 | t.Fatal("expected threshold breaker to be open") 245 | } 246 | 247 | err = cb.Call(circuit, 0) // Second failure trips 248 | if err == nil { 249 | t.Fatal("expected threshold breaker to error") 250 | } 251 | if !cb.Tripped() { 252 | t.Fatal("expected threshold breaker to be tripped") 253 | } 254 | } 255 | 256 | func TestThresholdBreakerCallingContext(t *testing.T) { 257 | circuit := func() error { 258 | return fmt.Errorf("error") 259 | } 260 | 261 | cb := NewThresholdBreaker(2) 262 | ctx, cancel := context.WithCancel(context.Background()) 263 | 264 | err := cb.CallContext(ctx, circuit, 0) // First failure 265 | if err == nil { 266 | t.Fatal("expected threshold breaker to error") 267 | } 268 | if cb.Tripped() { 269 | t.Fatal("expected threshold breaker to be open") 270 | } 271 | 272 | // Cancel the next Call. 273 | cancel() 274 | 275 | err = cb.CallContext(ctx, circuit, 0) // Second failure but it's canceled 276 | if err == nil { 277 | t.Fatal("expected threshold breaker to error") 278 | } 279 | if cb.Tripped() { 280 | t.Fatal("expected threshold breaker to be open") 281 | } 282 | 283 | err = cb.CallContext(context.Background(), circuit, 0) // Thirt failure trips 284 | if err == nil { 285 | t.Fatal("expected threshold breaker to error") 286 | } 287 | if !cb.Tripped() { 288 | t.Fatal("expected threshold breaker to be tripped") 289 | } 290 | } 291 | 292 | func TestThresholdBreakerResets(t *testing.T) { 293 | called := 0 294 | success := false 295 | circuit := func() error { 296 | if called == 0 { 297 | called++ 298 | return fmt.Errorf("error") 299 | } 300 | success = true 301 | return nil 302 | } 303 | 304 | c := clock.NewMock() 305 | cb := NewThresholdBreaker(1) 306 | cb.Clock = c 307 | err := cb.Call(circuit, 0) 308 | if err == nil { 309 | t.Fatal("Expected cb to return an error") 310 | } 311 | 312 | c.Add(cb.nextBackOff + 1) 313 | for i := 0; i < 4; i++ { 314 | err = cb.Call(circuit, 0) 315 | if err != nil { 316 | t.Fatal("Expected cb to be successful") 317 | } 318 | 319 | if !success { 320 | t.Fatal("Expected cb to have been reset") 321 | } 322 | } 323 | } 324 | 325 | func TestTimeoutBreaker(t *testing.T) { 326 | wait := make(chan struct{}) 327 | 328 | c := clock.NewMock() 329 | called := int32(0) 330 | 331 | circuit := func() error { 332 | wait <- struct{}{} 333 | atomic.AddInt32(&called, 1) 334 | <-wait 335 | return nil 336 | } 337 | 338 | cb := NewThresholdBreaker(1) 339 | cb.Clock = c 340 | 341 | errc := make(chan error) 342 | go func() { errc <- cb.Call(circuit, time.Millisecond) }() 343 | 344 | <-wait 345 | c.Add(time.Millisecond * 3) 346 | wait <- struct{}{} 347 | 348 | err := <-errc 349 | if err == nil { 350 | t.Fatal("expected timeout breaker to return an error") 351 | } 352 | 353 | go cb.Call(circuit, time.Millisecond) 354 | <-wait 355 | c.Add(time.Millisecond * 3) 356 | wait <- struct{}{} 357 | 358 | if !cb.Tripped() { 359 | t.Fatal("expected timeout breaker to be open") 360 | } 361 | } 362 | 363 | func TestRateBreakerTripping(t *testing.T) { 364 | cb := NewRateBreaker(0.5, 4) 365 | cb.Success() 366 | cb.Success() 367 | cb.Fail() 368 | cb.Fail() 369 | 370 | if !cb.Tripped() { 371 | t.Fatal("expected rate breaker to be tripped") 372 | } 373 | 374 | if er := cb.ErrorRate(); er != 0.5 { 375 | t.Fatalf("expected error rate to be 0.5, got %f", er) 376 | } 377 | } 378 | 379 | func TestRateBreakerSampleSize(t *testing.T) { 380 | cb := NewRateBreaker(0.5, 100) 381 | cb.Fail() 382 | 383 | if cb.Tripped() { 384 | t.Fatal("expected rate breaker to not be tripped yet") 385 | } 386 | } 387 | 388 | func TestRateBreakerResets(t *testing.T) { 389 | serviceError := fmt.Errorf("service error") 390 | 391 | called := 0 392 | success := false 393 | circuit := func() error { 394 | if called < 4 { 395 | called++ 396 | return serviceError 397 | } 398 | success = true 399 | return nil 400 | } 401 | 402 | c := clock.NewMock() 403 | cb := NewRateBreaker(0.5, 4) 404 | cb.Clock = c 405 | var err error 406 | for i := 0; i < 4; i++ { 407 | err = cb.Call(circuit, 0) 408 | if err == nil { 409 | t.Fatal("Expected cb to return an error (closed breaker, service failure)") 410 | } else if err != serviceError { 411 | t.Fatal("Expected cb to return error from service (closed breaker, service failure)") 412 | } 413 | } 414 | 415 | err = cb.Call(circuit, 0) 416 | if err == nil { 417 | t.Fatal("Expected cb to return an error (open breaker)") 418 | } else if err != ErrBreakerOpen { 419 | t.Fatal("Expected cb to return open open breaker error (open breaker)") 420 | } 421 | 422 | c.Add(cb.nextBackOff + 1) 423 | err = cb.Call(circuit, 0) 424 | if err != nil { 425 | t.Fatal("Expected cb to be successful") 426 | } 427 | 428 | if !success { 429 | t.Fatal("Expected cb to have been reset") 430 | } 431 | } 432 | 433 | func TestNeverRetryAfterBackoffStops(t *testing.T) { 434 | cb := NewBreakerWithOptions(&Options{ 435 | BackOff: &backoff.StopBackOff{}, 436 | }) 437 | 438 | cb.Trip() 439 | 440 | // circuit should be open and never retry again 441 | // when nextBackoff is backoff.Stop 442 | called := 0 443 | cb.Call(func() error { 444 | called = 1 445 | return nil 446 | }, 0) 447 | 448 | if called == 1 { 449 | t.Fatal("Expected cb to never retry") 450 | } 451 | } 452 | 453 | // TestPartialSecondBackoff ensures that the breaker event less than nextBackoff value 454 | // time after tripping the breaker isn't allowed. 455 | func TestPartialSecondBackoff(t *testing.T) { 456 | c := clock.NewMock() 457 | cb := NewBreaker() 458 | cb.Clock = c 459 | 460 | // Set the time to 0.5 seconds after the epoch, then trip the breaker. 461 | c.Add(500 * time.Millisecond) 462 | cb.Trip() 463 | 464 | // Move forward 100 milliseconds in time and ensure that the backoff time 465 | // is set to a larger number than the clock advanced. 466 | c.Add(100 * time.Millisecond) 467 | cb.nextBackOff = 500 * time.Millisecond 468 | if cb.Ready() { 469 | t.Fatalf("expected breaker not to be ready after less time than nextBackoff had passed") 470 | } 471 | 472 | c.Add(401 * time.Millisecond) 473 | if !cb.Ready() { 474 | t.Fatalf("expected breaker to be ready after more than nextBackoff time had passed") 475 | } 476 | } 477 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | "net/url" 7 | "time" 8 | ) 9 | 10 | // HTTPClient is a wrapper around http.Client that provides circuit breaker capabilities. 11 | // 12 | // By default, the client will use its defaultBreaker. A BreakerLookup function may be 13 | // provided to allow different breakers to be used based on the circumstance. See the 14 | // implementation of NewHostBasedHTTPClient for an example of this. 15 | type HTTPClient struct { 16 | Client *http.Client 17 | BreakerTripped func() 18 | BreakerReset func() 19 | BreakerLookup func(*HTTPClient, interface{}) *Breaker 20 | Panel *Panel 21 | timeout time.Duration 22 | } 23 | 24 | var defaultBreakerName = "_default" 25 | 26 | // NewHTTPClient provides a circuit breaker wrapper around http.Client. 27 | // It wraps all of the regular http.Client functions. Specifying 0 for timeout will 28 | // give a breaker that does not check for time outs. 29 | func NewHTTPClient(timeout time.Duration, threshold int64, client *http.Client) *HTTPClient { 30 | breaker := NewThresholdBreaker(threshold) 31 | return NewHTTPClientWithBreaker(breaker, timeout, client) 32 | } 33 | 34 | // NewHostBasedHTTPClient provides a circuit breaker wrapper around http.Client. This 35 | // client will use one circuit breaker per host parsed from the request URL. This allows 36 | // you to use a single HTTPClient for multiple hosts with one host's breaker not affecting 37 | // the other hosts. 38 | func NewHostBasedHTTPClient(timeout time.Duration, threshold int64, client *http.Client) *HTTPClient { 39 | brclient := NewHTTPClient(timeout, threshold, client) 40 | 41 | brclient.BreakerLookup = func(c *HTTPClient, val interface{}) *Breaker { 42 | rawURL := val.(string) 43 | parsedURL, err := url.Parse(rawURL) 44 | if err != nil { 45 | breaker, _ := c.Panel.Get(defaultBreakerName) 46 | return breaker 47 | } 48 | host := parsedURL.Host 49 | 50 | cb, ok := c.Panel.Get(host) 51 | if !ok { 52 | cb = NewThresholdBreaker(threshold) 53 | c.Panel.Add(host, cb) 54 | } 55 | 56 | return cb 57 | } 58 | 59 | return brclient 60 | } 61 | 62 | // NewHTTPClientWithBreaker provides a circuit breaker wrapper around http.Client. 63 | // It wraps all of the regular http.Client functions using the provided Breaker. 64 | func NewHTTPClientWithBreaker(breaker *Breaker, timeout time.Duration, client *http.Client) *HTTPClient { 65 | if client == nil { 66 | client = &http.Client{} 67 | } 68 | 69 | panel := NewPanel() 70 | panel.Add(defaultBreakerName, breaker) 71 | 72 | brclient := &HTTPClient{Client: client, Panel: panel, timeout: timeout} 73 | brclient.BreakerLookup = func(c *HTTPClient, val interface{}) *Breaker { 74 | cb, _ := c.Panel.Get(defaultBreakerName) 75 | return cb 76 | } 77 | 78 | events := breaker.Subscribe() 79 | go func() { 80 | event := <-events 81 | switch event { 82 | case BreakerTripped: 83 | brclient.runBreakerTripped() 84 | case BreakerReset: 85 | brclient.runBreakerReset() 86 | } 87 | }() 88 | 89 | return brclient 90 | } 91 | 92 | // Do wraps http.Client Do() 93 | func (c *HTTPClient) Do(req *http.Request) (*http.Response, error) { 94 | var resp *http.Response 95 | var err error 96 | breaker := c.breakerLookup(req.URL.String()) 97 | err = breaker.Call(func() error { 98 | resp, err = c.Client.Do(req) 99 | return err 100 | }, c.timeout) 101 | return resp, err 102 | } 103 | 104 | // Get wraps http.Client Get() 105 | func (c *HTTPClient) Get(url string) (*http.Response, error) { 106 | var resp *http.Response 107 | breaker := c.breakerLookup(url) 108 | err := breaker.Call(func() error { 109 | aresp, err := c.Client.Get(url) 110 | resp = aresp 111 | return err 112 | }, c.timeout) 113 | return resp, err 114 | } 115 | 116 | // Head wraps http.Client Head() 117 | func (c *HTTPClient) Head(url string) (*http.Response, error) { 118 | var resp *http.Response 119 | breaker := c.breakerLookup(url) 120 | err := breaker.Call(func() error { 121 | aresp, err := c.Client.Head(url) 122 | resp = aresp 123 | return err 124 | }, c.timeout) 125 | return resp, err 126 | } 127 | 128 | // Post wraps http.Client Post() 129 | func (c *HTTPClient) Post(url string, bodyType string, body io.Reader) (*http.Response, error) { 130 | var resp *http.Response 131 | breaker := c.breakerLookup(url) 132 | err := breaker.Call(func() error { 133 | aresp, err := c.Client.Post(url, bodyType, body) 134 | resp = aresp 135 | return err 136 | }, c.timeout) 137 | return resp, err 138 | } 139 | 140 | // PostForm wraps http.Client PostForm() 141 | func (c *HTTPClient) PostForm(url string, data url.Values) (*http.Response, error) { 142 | var resp *http.Response 143 | breaker := c.breakerLookup(url) 144 | err := breaker.Call(func() error { 145 | aresp, err := c.Client.PostForm(url, data) 146 | resp = aresp 147 | return err 148 | }, c.timeout) 149 | return resp, err 150 | } 151 | 152 | func (c *HTTPClient) breakerLookup(val interface{}) *Breaker { 153 | if c.BreakerLookup != nil { 154 | return c.BreakerLookup(c, val) 155 | } 156 | cb, _ := c.Panel.Get(defaultBreakerName) 157 | return cb 158 | } 159 | 160 | func (c *HTTPClient) runBreakerTripped() { 161 | if c.BreakerTripped != nil { 162 | c.BreakerTripped() 163 | } 164 | } 165 | 166 | func (c *HTTPClient) runBreakerReset() { 167 | if c.BreakerReset != nil { 168 | c.BreakerReset() 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/peterbourgon/g2s" 7 | 8 | "io/ioutil" 9 | "log" 10 | "time" 11 | ) 12 | 13 | func ExampleNewThresholdBreaker() { 14 | // This example sets up a ThresholdBreaker that will trip if remoteCall returns 15 | // an error 10 times in a row. The error returned by Call() will be the error 16 | // returned by remoteCall, unless the breaker has been tripped, in which case 17 | // it will return ErrBreakerOpen. 18 | breaker := NewThresholdBreaker(10) 19 | err := breaker.Call(remoteCall, 0) 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | } 24 | 25 | func ExampleNewThresholdBreaker_manual() { 26 | // This example demonstrates the manual use of a ThresholdBreaker. The breaker 27 | // will trip when Fail is called 10 times in a row. 28 | breaker := NewThresholdBreaker(10) 29 | if breaker.Ready() { 30 | err := remoteCall() 31 | if err != nil { 32 | breaker.Fail() 33 | log.Fatal(err) 34 | } else { 35 | breaker.Success() 36 | } 37 | } 38 | } 39 | 40 | func ExampleNewThresholdBreaker_timeout() { 41 | // This example sets up a ThresholdBreaker that will trip if remoteCall 42 | // returns an error OR takes longer than one second 10 times in a row. The 43 | // error returned by Call() will be the error returned by remoteCall with 44 | // two exceptions: if remoteCall takes longer than one second the return 45 | // value will be ErrBreakerTimeout, if the breaker has been tripped the 46 | // return value will be ErrBreakerOpen. 47 | breaker := NewThresholdBreaker(10) 48 | err := breaker.Call(remoteCall, time.Second) 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | } 53 | 54 | func ExampleNewConsecutiveBreaker() { 55 | // This example sets up a FrequencyBreaker that will trip if remoteCall returns 56 | // an error 10 times in a row within a period of 2 minutes. 57 | breaker := NewConsecutiveBreaker(10) 58 | err := breaker.Call(remoteCall, 0) 59 | if err != nil { 60 | log.Fatal(err) 61 | } 62 | } 63 | 64 | func ExampleHTTPClient() { 65 | // This example sets up an HTTP client wrapped in a ThresholdBreaker. The 66 | // breaker will trip with the same behavior as ThresholdBreaker. 67 | client := NewHTTPClient(time.Second*5, 10, nil) 68 | 69 | resp, err := client.Get("http://example.com/resource.json") 70 | if err != nil { 71 | log.Fatal(err) 72 | } 73 | resource, err := ioutil.ReadAll(resp.Body) 74 | resp.Body.Close() 75 | if err != nil { 76 | log.Fatal(err) 77 | } 78 | fmt.Printf("%s", resource) 79 | } 80 | 81 | func ExampleBreaker_events() { 82 | // This example demonstrates the BreakerTripped and BreakerReset callbacks. These are 83 | // available on all breaker types. 84 | breaker := NewThresholdBreaker(1) 85 | events := breaker.Subscribe() 86 | 87 | go func() { 88 | for { 89 | e := <-events 90 | switch e { 91 | case BreakerTripped: 92 | log.Println("breaker tripped") 93 | case BreakerReset: 94 | log.Println("breaker reset") 95 | case BreakerFail: 96 | log.Println("breaker fail") 97 | case BreakerReady: 98 | log.Println("breaker ready") 99 | } 100 | } 101 | }() 102 | 103 | breaker.Fail() 104 | breaker.Reset() 105 | } 106 | 107 | func ExamplePanel() { 108 | // This example demonstrates using a Panel to aggregate and manage circuit breakers. 109 | breaker1 := NewThresholdBreaker(10) 110 | breaker2 := NewRateBreaker(0.95, 100) 111 | 112 | panel := NewPanel() 113 | panel.Add("breaker1", breaker1) 114 | panel.Add("breaker2", breaker2) 115 | 116 | // Elsewhere in the code ... 117 | b1, _ := panel.Get("breaker1") 118 | b1.Call(func() error { 119 | // Do some work 120 | return nil 121 | }, 0) 122 | 123 | b2, _ := panel.Get("breaker2") 124 | b2.Call(func() error { 125 | // Do some work 126 | return nil 127 | }, 0) 128 | } 129 | 130 | func ExamplePanel_stats() { 131 | // This example demonstrates how to push circuit breaker stats to statsd via a Panel. 132 | // This example uses g2s. Anything conforming to the Statter interface can be used. 133 | s, err := g2s.Dial("udp", "statsd-server:8125") 134 | if err != nil { 135 | log.Fatal(err) 136 | } 137 | 138 | breaker := NewThresholdBreaker(10) 139 | panel := NewPanel() 140 | panel.Statter = s 141 | panel.StatsPrefixf = "sys.production.%s" 142 | panel.Add("x", breaker) 143 | 144 | breaker.Trip() // sys.production.circuit.x.tripped 145 | breaker.Reset() // sys.production.circuit.x.reset, sys.production.circuit.x.trip-time 146 | breaker.Fail() // sys.production.circuit.x.fail 147 | breaker.Ready() // sys.production.circuit.x.ready (if it's tripped and ready to retry) 148 | } 149 | 150 | func remoteCall() error { 151 | // Expensive remote call 152 | return nil 153 | } 154 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rubyist/circuitbreaker 2 | 3 | go 1.21.6 4 | 5 | require ( 6 | github.com/cenkalti/backoff v2.2.1+incompatible // indirect 7 | github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect 8 | github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= 2 | github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 3 | github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= 4 | github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= 5 | github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk= 6 | github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= 7 | -------------------------------------------------------------------------------- /panel.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | var defaultStatsPrefixf = "circuit.%s" 10 | 11 | // Statter interface provides a way to gather statistics from breakers 12 | type Statter interface { 13 | Counter(sampleRate float32, bucket string, n ...int) 14 | Timing(sampleRate float32, bucket string, d ...time.Duration) 15 | Gauge(sampleRate float32, bucket string, value ...string) 16 | } 17 | 18 | // PanelEvent wraps a BreakerEvent and provides the string name of the breaker 19 | type PanelEvent struct { 20 | Name string 21 | Event BreakerEvent 22 | } 23 | 24 | // Panel tracks a group of circuit breakers by name. 25 | type Panel struct { 26 | Statter Statter 27 | StatsPrefixf string 28 | 29 | Circuits map[string]*Breaker 30 | 31 | lastTripTimes map[string]time.Time 32 | tripTimesLock sync.RWMutex 33 | panelLock sync.RWMutex 34 | eventReceivers []chan PanelEvent 35 | } 36 | 37 | // NewPanel creates a new Panel 38 | func NewPanel() *Panel { 39 | return &Panel{ 40 | Circuits: make(map[string]*Breaker), 41 | Statter: &noopStatter{}, 42 | StatsPrefixf: defaultStatsPrefixf, 43 | lastTripTimes: make(map[string]time.Time)} 44 | } 45 | 46 | // Add sets the name as a reference to the given circuit breaker. 47 | func (p *Panel) Add(name string, cb *Breaker) { 48 | p.panelLock.Lock() 49 | p.Circuits[name] = cb 50 | p.panelLock.Unlock() 51 | 52 | events := cb.Subscribe() 53 | 54 | go func() { 55 | for event := range events { 56 | for _, receiver := range p.eventReceivers { 57 | receiver <- PanelEvent{name, event} 58 | } 59 | switch event { 60 | case BreakerTripped: 61 | p.breakerTripped(name) 62 | case BreakerReset: 63 | p.breakerReset(name) 64 | case BreakerFail: 65 | p.breakerFail(name) 66 | case BreakerReady: 67 | p.breakerReady(name) 68 | } 69 | } 70 | }() 71 | } 72 | 73 | // Get retrieves a circuit breaker by name. If no circuit breaker exists, it 74 | // returns the NoOp one and sets ok to false. 75 | func (p *Panel) Get(name string) (*Breaker, bool) { 76 | p.panelLock.RLock() 77 | cb, ok := p.Circuits[name] 78 | p.panelLock.RUnlock() 79 | 80 | if ok { 81 | return cb, ok 82 | } 83 | 84 | return NewBreaker(), ok 85 | } 86 | 87 | // Subscribe returns a channel of PanelEvents. Whenever a breaker changes state, 88 | // the PanelEvent will be sent over the channel. See BreakerEvent for the types of events. 89 | func (p *Panel) Subscribe() <-chan PanelEvent { 90 | eventReader := make(chan PanelEvent) 91 | output := make(chan PanelEvent, 100) 92 | 93 | go func() { 94 | for v := range eventReader { 95 | select { 96 | case output <- v: 97 | default: 98 | <-output 99 | output <- v 100 | } 101 | } 102 | }() 103 | p.eventReceivers = append(p.eventReceivers, eventReader) 104 | return output 105 | } 106 | 107 | func (p *Panel) breakerTripped(name string) { 108 | p.Statter.Counter(1.0, fmt.Sprintf(p.StatsPrefixf, name)+".tripped", 1) 109 | p.tripTimesLock.Lock() 110 | p.lastTripTimes[name] = time.Now() 111 | p.tripTimesLock.Unlock() 112 | } 113 | 114 | func (p *Panel) breakerReset(name string) { 115 | bucket := fmt.Sprintf(p.StatsPrefixf, name) 116 | 117 | p.Statter.Counter(1.0, bucket+".reset", 1) 118 | 119 | p.tripTimesLock.RLock() 120 | lastTrip := p.lastTripTimes[name] 121 | p.tripTimesLock.RUnlock() 122 | 123 | if !lastTrip.IsZero() { 124 | p.Statter.Timing(1.0, bucket+".trip-time", time.Since(lastTrip)) 125 | p.tripTimesLock.Lock() 126 | p.lastTripTimes[name] = time.Time{} 127 | p.tripTimesLock.Unlock() 128 | } 129 | } 130 | 131 | func (p *Panel) breakerFail(name string) { 132 | p.Statter.Counter(1.0, fmt.Sprintf(p.StatsPrefixf, name)+".fail", 1) 133 | } 134 | 135 | func (p *Panel) breakerReady(name string) { 136 | p.Statter.Counter(1.0, fmt.Sprintf(p.StatsPrefixf, name)+".ready", 1) 137 | } 138 | 139 | type noopStatter struct { 140 | } 141 | 142 | func (*noopStatter) Counter(sampleRate float32, bucket string, n ...int) {} 143 | func (*noopStatter) Timing(sampleRate float32, bucket string, d ...time.Duration) {} 144 | func (*noopStatter) Gauge(sampleRate float32, bucket string, value ...string) {} 145 | -------------------------------------------------------------------------------- /panel_test.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "reflect" 5 | "sync" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestPanelGet(t *testing.T) { 11 | rb := NewBreaker() 12 | p := NewPanel() 13 | p.Add("a", rb) 14 | 15 | a, ok := p.Get("a") 16 | if a != rb { 17 | t.Errorf("Expected 'a' to have a %s, got %s", 18 | reflect.TypeOf(rb), reflect.TypeOf(a)) 19 | } 20 | if !ok { 21 | t.Errorf("Expected ok to be true") 22 | } 23 | 24 | a, ok = p.Get("missing") 25 | } 26 | 27 | func TestPanelAdd(t *testing.T) { 28 | p := NewPanel() 29 | rb := NewBreaker() 30 | 31 | p.Add("a", rb) 32 | 33 | if a, _ := p.Get("a"); a != rb { 34 | t.Errorf("Expected 'a' to have a %s, got %s", 35 | reflect.TypeOf(rb), reflect.TypeOf(a)) 36 | } 37 | } 38 | 39 | func TestPanelStats(t *testing.T) { 40 | statter := newTestStatter() 41 | p := NewPanel() 42 | p.Statter = statter 43 | rb := NewBreaker() 44 | p.Add("breaker", rb) 45 | 46 | rb.Fail() 47 | rb.Trip() 48 | time.Sleep(rb.nextBackOff) 49 | rb.Ready() 50 | rb.Reset() 51 | 52 | time.Sleep(rb.nextBackOff) 53 | 54 | if c := statter.Count("circuit.breaker.tripped"); c != 1 { 55 | t.Fatalf("expected trip count to be 1, got %d", c) 56 | } 57 | 58 | if c := statter.Count("circuit.breaker.reset"); c != 1 { 59 | t.Fatalf("expected reset count to be 1, got %d", c) 60 | } 61 | 62 | if c := statter.Time("circuit.breaker.trip-time"); c == 0 { 63 | t.Fatalf("expected trip time to have been counted, got %v", c) 64 | } 65 | 66 | if c := statter.Count("circuit.breaker.fail"); c != 1 { 67 | t.Fatalf("expected fail count to be 1, got %d", c) 68 | } 69 | 70 | if c := statter.Count("circuit.breaker.ready"); c != 1 { 71 | t.Fatalf("expected ready count to be 1, got %d", c) 72 | } 73 | } 74 | 75 | type testStatter struct { 76 | Counts map[string]int 77 | Timings map[string]time.Duration 78 | l sync.Mutex 79 | } 80 | 81 | func newTestStatter() *testStatter { 82 | return &testStatter{Counts: make(map[string]int), Timings: make(map[string]time.Duration)} 83 | } 84 | 85 | func (s *testStatter) Count(name string) int { 86 | s.l.Lock() 87 | defer s.l.Unlock() 88 | return s.Counts[name] 89 | } 90 | 91 | func (s *testStatter) Time(name string) time.Duration { 92 | s.l.Lock() 93 | defer s.l.Unlock() 94 | return s.Timings[name] 95 | } 96 | 97 | func (s *testStatter) Counter(sampleRate float32, bucket string, n ...int) { 98 | for _, x := range n { 99 | s.l.Lock() 100 | s.Counts[bucket] += x 101 | s.l.Unlock() 102 | } 103 | } 104 | 105 | func (s *testStatter) Timing(sampleRate float32, bucket string, d ...time.Duration) { 106 | for _, x := range d { 107 | s.l.Lock() 108 | s.Timings[bucket] += x 109 | s.l.Unlock() 110 | } 111 | } 112 | 113 | func (*testStatter) Gauge(sampleRate float32, bucket string, value ...string) {} 114 | -------------------------------------------------------------------------------- /window.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "container/ring" 5 | "sync" 6 | "time" 7 | 8 | "github.com/facebookgo/clock" 9 | ) 10 | 11 | var ( 12 | // DefaultWindowTime is the default time the window covers, 10 seconds. 13 | DefaultWindowTime = time.Millisecond * 10000 14 | 15 | // DefaultWindowBuckets is the default number of buckets the window holds, 10. 16 | DefaultWindowBuckets = 10 17 | ) 18 | 19 | // bucket holds counts of failures and successes 20 | type bucket struct { 21 | failure int64 22 | success int64 23 | } 24 | 25 | // Reset resets the counts to 0 26 | func (b *bucket) Reset() { 27 | b.failure = 0 28 | b.success = 0 29 | } 30 | 31 | // Fail increments the failure count 32 | func (b *bucket) Fail() { 33 | b.failure++ 34 | } 35 | 36 | // Sucecss increments the success count 37 | func (b *bucket) Success() { 38 | b.success++ 39 | } 40 | 41 | // window maintains a ring of buckets and increments the failure and success 42 | // counts of the current bucket. Once a specified time has elapsed, it will 43 | // advance to the next bucket, reseting its counts. This allows the keeping of 44 | // rolling statistics on the counts. 45 | type window struct { 46 | buckets *ring.Ring 47 | bucketTime time.Duration 48 | bucketLock sync.RWMutex 49 | lastAccess time.Time 50 | clock clock.Clock 51 | } 52 | 53 | // newWindow creates a new window. windowTime is the time covering the entire 54 | // window. windowBuckets is the number of buckets the window is divided into. 55 | // An example: a 10 second window with 10 buckets will have 10 buckets covering 56 | // 1 second each. 57 | func newWindow(windowTime time.Duration, windowBuckets int) *window { 58 | buckets := ring.New(windowBuckets) 59 | for i := 0; i < buckets.Len(); i++ { 60 | buckets.Value = &bucket{} 61 | buckets = buckets.Next() 62 | } 63 | 64 | clock := clock.New() 65 | 66 | bucketTime := time.Duration(windowTime.Nanoseconds() / int64(windowBuckets)) 67 | return &window{ 68 | buckets: buckets, 69 | bucketTime: bucketTime, 70 | clock: clock, 71 | lastAccess: clock.Now(), 72 | } 73 | } 74 | 75 | // Fail records a failure in the current bucket. 76 | func (w *window) Fail() { 77 | w.bucketLock.Lock() 78 | b := w.getLatestBucket() 79 | b.Fail() 80 | w.bucketLock.Unlock() 81 | } 82 | 83 | // Success records a success in the current bucket. 84 | func (w *window) Success() { 85 | w.bucketLock.Lock() 86 | b := w.getLatestBucket() 87 | b.Success() 88 | w.bucketLock.Unlock() 89 | } 90 | 91 | // Failures returns the total number of failures recorded in all buckets. 92 | func (w *window) Failures() int64 { 93 | w.bucketLock.RLock() 94 | 95 | var failures int64 96 | w.buckets.Do(func(x interface{}) { 97 | b := x.(*bucket) 98 | failures += b.failure 99 | }) 100 | 101 | w.bucketLock.RUnlock() 102 | return failures 103 | } 104 | 105 | // Successes returns the total number of successes recorded in all buckets. 106 | func (w *window) Successes() int64 { 107 | w.bucketLock.RLock() 108 | 109 | var successes int64 110 | w.buckets.Do(func(x interface{}) { 111 | b := x.(*bucket) 112 | successes += b.success 113 | }) 114 | w.bucketLock.RUnlock() 115 | return successes 116 | } 117 | 118 | // ErrorRate returns the error rate calculated over all buckets, expressed as 119 | // a floating point number (e.g. 0.9 for 90%) 120 | func (w *window) ErrorRate() float64 { 121 | var total int64 122 | var failures int64 123 | 124 | w.bucketLock.RLock() 125 | w.buckets.Do(func(x interface{}) { 126 | b := x.(*bucket) 127 | total += b.failure + b.success 128 | failures += b.failure 129 | }) 130 | w.bucketLock.RUnlock() 131 | 132 | if total == 0 { 133 | return 0.0 134 | } 135 | 136 | return float64(failures) / float64(total) 137 | } 138 | 139 | // Reset resets the count of all buckets. 140 | func (w *window) Reset() { 141 | w.bucketLock.Lock() 142 | 143 | w.buckets.Do(func(x interface{}) { 144 | x.(*bucket).Reset() 145 | }) 146 | w.bucketLock.Unlock() 147 | } 148 | 149 | // getLatestBucket returns the current bucket. If the bucket time has elapsed 150 | // it will move to the next bucket, resetting its counts and updating the last 151 | // access time before returning it. getLatestBucket assumes that the caller has 152 | // locked the bucketLock 153 | func (w *window) getLatestBucket() *bucket { 154 | var b *bucket 155 | b = w.buckets.Value.(*bucket) 156 | elapsed := w.clock.Now().Sub(w.lastAccess) 157 | 158 | if elapsed > w.bucketTime { 159 | // Reset the buckets between now and number of buckets ago. If 160 | // that is more that the existing buckets, reset all. 161 | for i := 0; i < w.buckets.Len(); i++ { 162 | w.buckets = w.buckets.Next() 163 | b = w.buckets.Value.(*bucket) 164 | b.Reset() 165 | elapsed = time.Duration(int64(elapsed) - int64(w.bucketTime)) 166 | if elapsed < w.bucketTime { 167 | // Done resetting buckets. 168 | break 169 | } 170 | } 171 | w.lastAccess = w.clock.Now() 172 | } 173 | return b 174 | } 175 | -------------------------------------------------------------------------------- /window_test.go: -------------------------------------------------------------------------------- 1 | package circuit 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/facebookgo/clock" 8 | ) 9 | 10 | func TestWindowCounts(t *testing.T) { 11 | w := newWindow(time.Millisecond*10, 2) 12 | w.Fail() 13 | w.Fail() 14 | w.Success() 15 | w.Success() 16 | 17 | if f := w.Failures(); f != 2 { 18 | t.Fatalf("expected window to have 2 failures, got %d", f) 19 | } 20 | 21 | if s := w.Successes(); s != 2 { 22 | t.Fatalf("expected window to have 2 successes, got %d", s) 23 | } 24 | 25 | if r := w.ErrorRate(); r != 0.5 { 26 | t.Fatalf("expected window to have 0.5 error rate, got %f", r) 27 | } 28 | 29 | w.Reset() 30 | if f := w.Failures(); f != 0 { 31 | t.Fatalf("expected reset window to have 0 failures, got %d", f) 32 | } 33 | if s := w.Successes(); s != 0 { 34 | t.Fatalf("expected window to have 0 successes, got %d", s) 35 | } 36 | } 37 | 38 | func TestWindowSlides(t *testing.T) { 39 | c := clock.NewMock() 40 | 41 | w := newWindow(time.Millisecond*10, 2) 42 | w.clock = c 43 | w.lastAccess = c.Now() 44 | 45 | w.Fail() 46 | c.Add(time.Millisecond * 6) 47 | w.Fail() 48 | 49 | counts := 0 50 | w.buckets.Do(func(x interface{}) { 51 | b := x.(*bucket) 52 | if b.failure > 0 { 53 | counts++ 54 | } 55 | }) 56 | 57 | if counts != 2 { 58 | t.Fatalf("expected 2 buckets to have failures, got %d", counts) 59 | } 60 | 61 | c.Add(time.Millisecond * 15) 62 | w.Success() 63 | counts = 0 64 | w.buckets.Do(func(x interface{}) { 65 | b := x.(*bucket) 66 | if b.failure > 0 { 67 | counts++ 68 | } 69 | }) 70 | 71 | if counts != 0 { 72 | t.Fatalf("expected 0 buckets to have failures, got %d", counts) 73 | } 74 | } 75 | --------------------------------------------------------------------------------