├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── lint.yml │ └── unit_tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── breaker.go ├── breaker_test.go ├── go.mod ├── go.sum ├── state.go └── state_test.go /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Proposed Changes 2 | 3 | 4 | 5 | ## Further Comments 6 | 7 | 8 | 9 | ## Checklist 10 | 11 | Please read the [CLA (https://www.mercari.com/cla/)](https://www.mercari.com/cla/) carefully before submitting your contribution to Mercari. 12 | 13 | - [ ] Under any circumstances, by submitting your contribution, you are deemed to accept and agree to be bound by the terms and conditions of the CLA. 14 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | on: [push, pull_request] 3 | jobs: 4 | golangci-lint: 5 | name: lint library 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@v2 9 | - name: golangci-lint 10 | uses: golangci/golangci-lint-action@v2 11 | with: 12 | version: v1.29 -------------------------------------------------------------------------------- /.github/workflows/unit_tests.yml: -------------------------------------------------------------------------------- 1 | name: acceptance 2 | on: [push, pull_request] 3 | jobs: 4 | acceptance: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | go-version: 9 | - '~1.15' 10 | - '~1.16' 11 | - '~1.17' 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: actions/setup-go@v2 15 | with: 16 | go-version: ${{ matrix.go-version }} 17 | - run: go test -race -v ./... 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | .idea 8 | 9 | # Test binary, build with `go test -c` 10 | *.test 11 | 12 | # Output of the go coverage tool, specifically when used with LiteIDE 13 | *.out 14 | 15 | .idea 16 | .vscode -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Please read the CLA carefully before submitting your contribution to Mercari. 4 | Under any circumstances, by submitting your contribution, you are deemed to accept and agree to be bound by the terms and conditions of the CLA. 5 | 6 | https://www.mercari.com/cla/ 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Mercari, Inc. 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-circuitbreaker 2 | 3 | [![GoDoc](https://godoc.org/github.com/mercari/go-circuitbreaker?status.svg)](https://godoc.org/github.com/mercari/go-circuitbreaker) 4 | ![lint](https://github.com/mercari/go-circuitbreaker/actions/workflows/lint.yml/badge.svg) 5 | ![unittests](https://github.com/mercari/go-circuitbreaker/actions/workflows/unit_tests.yml/badge.svg) 6 | 7 | go-circuitbreaker is a Circuit Breaker pattern implementation in Go. 8 | 9 | - Provides natural code flow. 10 | - Ignore errors occurred by request cancellation from request callers (in default). 11 | - `Ignore(err)` and `MarkAsSuccess(err)` wrappers enable you to receive non-nil error from wrapped operations without counting it as a failure. 12 | 13 | # What is circuit breaker? 14 | 15 | See: [Circuit Breaker pattern \- Cloud Design Patterns \| Microsoft Docs](https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker) 16 | 17 | ## Initializing the circuit breaker 18 | 19 | This library is using the [functional options pattern](https://github.com/uber-go/guide/blob/master/style.md#functional-options) to instance a new circuit breaker. The functions are the following: 20 | 21 | You can use them like the next example: 22 | 23 | ```go 24 | cb := circuitbreaker.New( 25 | circuitbreaker.WithClock(clock.New()), 26 | circuitbreaker.WithFailOnContextCancel(true), 27 | circuitbreaker.WithFailOnContextDeadline(true), 28 | circuitbreaker.WithHalfOpenMaxSuccesses(10), 29 | circuitbreaker.WithOpenTimeoutBackOff(backoff.NewExponentialBackOff()), 30 | circuitbreaker.WithOpenTimeout(10*time.Second), 31 | circuitbreaker.WithCounterResetInterval(10*time.Second), 32 | // we also have NewTripFuncThreshold and NewTripFuncConsecutiveFailures 33 | circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncFailureRate(10, 0.4)), 34 | circuitbreaker.WithOnStateChangeHookFn(func(from, to circuitbreaker.State) { 35 | log.Printf("state changed from %s to %s\n", from, to) 36 | }), 37 | ) 38 | ``` 39 | 40 | ## Simple Examples with Do. 41 | 42 | Wrapping your code block with `Do()` protects your process with Circuit Breaker. 43 | 44 | ```go 45 | var cb = circuitbreaker.New() 46 | 47 | u, err := cb.Do(ctx, func() (interface{}, error) { 48 | return fetchUserInfo(name) 49 | }) 50 | user, _ := u.(*User) // Casting interface{} into *User safely. 51 | ``` 52 | 53 | ## Example using Done. 54 | 55 | The following example using Ready() and Done() is exactly equals to the above one. Since this style enables you to protect your processes without wrapping it and using type-unsafe interface{}, it would make it easy to implement CB to your existing logic. 56 | 57 | ```go 58 | var cb = circuitbreaker.New() 59 | 60 | func getUserInfo(ctx context.Context, name string) (_ *User,err error) { 61 | if !cb.Ready() { 62 | return nil, circuitbreaker.ErrOpened 63 | } 64 | defer func() { err = cb.Done(ctx, err) } 65 | 66 | return fetchUserInfo(ctx, name) 67 | } 68 | ``` 69 | 70 | 71 | ## Smart handling for context.Canceled and context.DeadlineExceeded 72 | 73 | In microservices architectures, Circuit Breaker is essential to protect services you depend on from cascading failures and keep your services latency low during long-lasting failures. And for microservices written in Go, cancel request by propagating context is also a good convention. But there are some pitfall when you combinate context and circuit breaker. 74 | 75 | Your microservices users are able to cancel your requests. The cancellation will be propagated through the context to your operation which is protected by CB. If the CB mark the canceled execution as a fail, your CB may open even though there is no problem in your infrastructure. It is a false-positive CB-open. It would prevent your other clients from doing tasks successfully because the CB is shared by all clients. Otherwise, if the CB mark the canceled execution as a success, the CB may back to closed unexpectedly. (it's a false-negative). In order to avoid those unexpected issue, `go-circuitbreaker` provides *an option to ignore the errors caused by the context cancellation*. 76 | 77 | The same for timeouts configuration. Especially on gRPC, clients are able to set a timeout to the server's context. It means that clients with too short timeout are able to make other clients operation fail. For this issue, `go-circuitbreaker` provides an option to *an option to ignore deadline exceeded errors* . 78 | 79 | ## Ignore and MarkAsSuccess wrappers 80 | 81 | Sometimes, we would like to receive an error from a protected operation but don't want to mark the request as a failure. For example, a case that protected HTTP request responded 404 NotFound. This application-level error is obviously not caused by a failure, so that we'd like to return nil error, but want to receive non-nil error. Because `go-circuitbreaker` is able to receive errors from protected operations without making them as failures, you don't need to write complicated error handlings in order to achieve your goal. 82 | 83 | ```go 84 | cb := circuitbreaker.New(nil) 85 | 86 | data, err := cb.Do(context.Background(), func() (interface{}, error) { 87 | u, err := fetchUserInfo("john") 88 | if err == errUserNotFound { 89 | return u, circuitbreaker.Ignore(err) // cb does not treat the err as a failure. 90 | } 91 | return u, err 92 | }) 93 | if err != nil { 94 | // Here, you can get errUserNotFound 95 | log.Fatal(err) 96 | } 97 | ``` 98 | 99 | ## Installation 100 | 101 | ```bash 102 | go get github.com/mercari/go-circuitbreaker 103 | ``` 104 | -------------------------------------------------------------------------------- /breaker.go: -------------------------------------------------------------------------------- 1 | package circuitbreaker 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | "time" 9 | 10 | "github.com/benbjohnson/clock" 11 | backoff "github.com/cenkalti/backoff/v3" 12 | ) 13 | 14 | var ( 15 | // ErrOpen is an error to signify that the CB is open and executing 16 | // operations are not allowed. 17 | ErrOpen = errors.New("circuit breaker open") 18 | 19 | // DefaultTripFunc is used when Options.ShouldTrip is nil. 20 | DefaultTripFunc = NewTripFuncThreshold(10) 21 | ) 22 | 23 | // Default setting parameters. 24 | const ( 25 | DefaultInterval = 1 * time.Second 26 | DefaultHalfOpenMaxSuccesses = 4 27 | ) 28 | 29 | // State represents the internal state of CB. 30 | type State string 31 | 32 | // State constants. 33 | const ( 34 | StateClosed State = "closed" 35 | StateOpen State = "open" 36 | StateHalfOpen State = "half-open" 37 | ) 38 | 39 | // DefaultOpenBackOff returns defaultly used BackOff. 40 | func DefaultOpenBackOff() backoff.BackOff { 41 | _backoff := backoff.NewExponentialBackOff() 42 | _backoff.MaxElapsedTime = 0 43 | _backoff.Reset() 44 | return _backoff 45 | } 46 | 47 | // Counters holds internal counter(s) of CircuitBreaker. 48 | type Counters struct { 49 | Successes int64 50 | Failures int64 51 | ConsecutiveSuccesses int64 52 | ConsecutiveFailures int64 53 | } 54 | 55 | func (c *Counters) reset() { *c = Counters{} } 56 | 57 | func (c *Counters) resetSuccesses() { 58 | c.Successes = 0 59 | c.ConsecutiveSuccesses = 0 60 | } 61 | 62 | func (c *Counters) resetFailures() { 63 | c.Failures = 0 64 | c.ConsecutiveFailures = 0 65 | } 66 | 67 | func (c *Counters) incrementSuccesses() { 68 | c.Successes++ 69 | c.ConsecutiveSuccesses++ 70 | c.ConsecutiveFailures = 0 71 | } 72 | 73 | func (c *Counters) incrementFailures() { 74 | c.Failures++ 75 | c.ConsecutiveFailures++ 76 | c.ConsecutiveSuccesses = 0 77 | } 78 | 79 | // StateChangeHook is a function which will be invoked when the state is changed. 80 | type StateChangeHook func(oldState, newState State) 81 | 82 | // TripFunc is a function to determine if CircuitBreaker should open (trip) or 83 | // not. TripFunc is called when cb.Fail was called and the state was 84 | // StateClosed. If TripFunc returns true, the cb's state goes to StateOpen. 85 | type TripFunc func(*Counters) bool 86 | 87 | // NewTripFuncThreshold provides a TripFunc. It returns true if the 88 | // Failures counter is larger than or equals to threshold. 89 | func NewTripFuncThreshold(threshold int64) TripFunc { 90 | return func(cnt *Counters) bool { return cnt.Failures >= threshold } 91 | } 92 | 93 | // NewTripFuncConsecutiveFailures provides a TripFunc that returns true 94 | // if the consecutive failures is larger than or equals to threshold. 95 | func NewTripFuncConsecutiveFailures(threshold int64) TripFunc { 96 | return func(cnt *Counters) bool { return cnt.ConsecutiveFailures >= threshold } 97 | } 98 | 99 | // NewTripFuncFailureRate provides a TripFunc that returns true if the failure 100 | // rate is higher or equals to rate. If the samples are fewer than min, always 101 | // returns false. 102 | func NewTripFuncFailureRate(min int64, rate float64) TripFunc { 103 | return func(cnt *Counters) bool { 104 | if cnt.Successes+cnt.Failures < min { 105 | return false 106 | } 107 | return float64(cnt.Failures)/float64(cnt.Successes+cnt.Failures) >= rate 108 | } 109 | } 110 | 111 | // IgnorableError signals that the operation should not be marked as a failure. 112 | type IgnorableError struct { 113 | err error 114 | } 115 | 116 | func (e *IgnorableError) Error() string { 117 | return fmt.Sprintf("circuitbreaker does not mark this error as a failure: %s", e.err.Error()) 118 | } 119 | 120 | // Unwrap unwraps e. 121 | func (e *IgnorableError) Unwrap() error { return e.err } 122 | 123 | // Ignore wraps the given err in a *IgnorableError. 124 | func Ignore(err error) error { 125 | if err == nil { 126 | return nil 127 | } 128 | return &IgnorableError{err} 129 | } 130 | 131 | // SuccessMarkableError signals that the operation should be mark as success. 132 | type SuccessMarkableError struct { 133 | err error 134 | } 135 | 136 | func (e *SuccessMarkableError) Error() string { 137 | return fmt.Sprintf("circuitbreaker mark this error as a success: %s", e.err.Error()) 138 | } 139 | 140 | // Unwrap unwraps e. 141 | func (e *SuccessMarkableError) Unwrap() error { return e.err } 142 | 143 | // MarkAsSuccess wraps the given err in a *SuccessMarkableError. 144 | func MarkAsSuccess(err error) error { 145 | if err == nil { 146 | return nil 147 | } 148 | return &SuccessMarkableError{err} 149 | } 150 | 151 | // Options holds CircuitBreaker configuration options. 152 | type options struct { 153 | // Clock to be used by CircuitBreaker. If nil, real-time clock is 154 | // used. 155 | clock clock.Clock 156 | 157 | // Interval is the cyclic time period to reset the internal counters 158 | // during state is in StateClosed. 159 | // 160 | // If zero, DefaultInterval is used. If Interval < 0, No interval will 161 | // be triggered. 162 | interval time.Duration 163 | 164 | // OpenTimeout is the period of StateOpened. After OpenTimeout, 165 | // CircuitBreaker's state will be changed to StateHalfOpened. If OpenBackOff 166 | // is not nil, OpenTimeout is ignored. 167 | openTimeout time.Duration 168 | 169 | // OpenBackOff is a Backoff to determine the period of StateOpened. Every 170 | // time the state transitions to StateOpened, OpenBackOff.NextBackOff() 171 | // recalculates the period. When the state transitions to StateClosed, 172 | // OpenBackOff is reset to the initial state. If both OpenTimeout is zero 173 | // value and OpenBackOff is empty, return value of DefaultOpenBackOff() is 174 | // used. 175 | // 176 | // NOTE: Please make sure not to set the ExponentialBackOff.MaxElapsedTime > 177 | // 0 for OpenBackOff. If so, your CB don't close after your period of the 178 | // StateOpened gets longer than the MaxElapsedTime. 179 | openBackOff backoff.BackOff 180 | 181 | // HalfOpenMaxSuccesses is max count of successive successes during the state 182 | // is in StateHalfOpened. If the state is StateHalfOpened and the successive 183 | // successes reaches this threshold, the state of CircuitBreaker changes 184 | // into StateClosed. If zero, DefaultHalfOpenMaxSuccesses is used. 185 | halfOpenMaxSuccesses int64 186 | 187 | // ShouldTrips is a function to determine if the CircuitBreaker should 188 | // trip. If the state is StateClosed and ShouldTrip returns true, 189 | // the state will be changed to StateOpened. 190 | // If nil, DefaultTripFunc is used. 191 | shouldTrip TripFunc 192 | 193 | // OnStateChange is a function which will be invoked when the state is changed. 194 | onStateChange StateChangeHook 195 | 196 | // FailOnContextCancel controls if CircuitBreaker mark an error when the 197 | // passed context.Done() is context.Canceled as a fail. 198 | failOnContextCancel bool 199 | 200 | // FailOnContextDeadline controls if CircuitBreaker mark an error when the 201 | // passed context.Done() is context.DeadlineExceeded as a fail. 202 | failOnContextDeadline bool 203 | } 204 | 205 | // CircuitBreaker provides circuit breaker pattern. 206 | type CircuitBreaker struct { 207 | clock clock.Clock 208 | interval time.Duration 209 | halfOpenMaxSuccesses int64 210 | openBackOff backoff.BackOff 211 | shouldTrip TripFunc 212 | onStateChange StateChangeHook 213 | failOnContextCancel bool 214 | failOnContextDeadline bool 215 | 216 | mu sync.RWMutex 217 | state state 218 | cnt Counters 219 | } 220 | 221 | type fnApplyOptions func(*options) 222 | 223 | // BreakerOption interface for applying configuration in the constructor 224 | type BreakerOption interface { 225 | apply(*options) 226 | } 227 | 228 | func (f fnApplyOptions) apply(options *options) { 229 | f(options) 230 | } 231 | 232 | // WithTripFunc Set the function for counter 233 | func WithTripFunc(tripFunc TripFunc) BreakerOption { 234 | return fnApplyOptions(func(options *options) { 235 | options.shouldTrip = tripFunc 236 | }) 237 | } 238 | 239 | // WithClock Set the clock 240 | func WithClock(clock clock.Clock) BreakerOption { 241 | return fnApplyOptions(func(options *options) { 242 | options.clock = clock 243 | }) 244 | } 245 | 246 | // WithOpenTimeoutBackOff Set the time backoff 247 | func WithOpenTimeoutBackOff(backoff backoff.BackOff) BreakerOption { 248 | return fnApplyOptions(func(options *options) { 249 | options.openBackOff = backoff 250 | }) 251 | } 252 | 253 | // WithOpenTimeout Set the timeout of the circuit breaker 254 | func WithOpenTimeout(timeout time.Duration) BreakerOption { 255 | return fnApplyOptions(func(options *options) { 256 | options.openTimeout = timeout 257 | }) 258 | } 259 | 260 | // WithHalfOpenMaxSuccesses Set the number of half open successes 261 | func WithHalfOpenMaxSuccesses(maxSuccesses int64) BreakerOption { 262 | return fnApplyOptions(func(options *options) { 263 | options.halfOpenMaxSuccesses = maxSuccesses 264 | }) 265 | } 266 | 267 | // WithCounterResetInterval Set the interval of the circuit breaker, which is the cyclic time period to reset the internal counters 268 | func WithCounterResetInterval(interval time.Duration) BreakerOption { 269 | return fnApplyOptions(func(options *options) { 270 | options.interval = interval 271 | }) 272 | } 273 | 274 | // WithFailOnContextCancel Set if the context should fail on cancel 275 | func WithFailOnContextCancel(failOnContextCancel bool) BreakerOption { 276 | return fnApplyOptions(func(options *options) { 277 | options.failOnContextCancel = failOnContextCancel 278 | }) 279 | } 280 | 281 | // WithFailOnContextDeadline Set if the context should fail on deadline 282 | func WithFailOnContextDeadline(failOnContextDeadline bool) BreakerOption { 283 | return fnApplyOptions(func(options *options) { 284 | options.failOnContextDeadline = failOnContextDeadline 285 | }) 286 | } 287 | 288 | // WithOnStateChangeHookFn set a hook function that trigger if the condition of the StateChangeHook is true 289 | func WithOnStateChangeHookFn(hookFn StateChangeHook) BreakerOption { 290 | return fnApplyOptions(func(options *options) { 291 | options.onStateChange = hookFn 292 | }) 293 | } 294 | 295 | func defaultOptions() *options { 296 | return &options{ 297 | shouldTrip: DefaultTripFunc, 298 | clock: clock.New(), 299 | openBackOff: DefaultOpenBackOff(), 300 | openTimeout: 0, 301 | halfOpenMaxSuccesses: DefaultHalfOpenMaxSuccesses, 302 | interval: DefaultInterval, 303 | } 304 | } 305 | 306 | // New returns a new CircuitBreaker 307 | // The constructor will be instanced using the functional options pattern. When creating a new circuit breaker 308 | // we should pass or left it blank if we want to use its default options. 309 | // An example of the constructor would be like this: 310 | // 311 | // cb := circuitbreaker.New( 312 | // circuitbreaker.WithClock(clock.New()), 313 | // circuitbreaker.WithFailOnContextCancel(true), 314 | // circuitbreaker.WithFailOnContextDeadline(true), 315 | // circuitbreaker.WithHalfOpenMaxSuccesses(10), 316 | // circuitbreaker.WithOpenTimeoutBackOff(backoff.NewExponentialBackOff()), 317 | // circuitbreaker.WithOpenTimeout(10*time.Second), 318 | // circuitbreaker.WithCounterResetInterval(10*time.Second), 319 | // // we also have NewTripFuncThreshold and NewTripFuncConsecutiveFailures 320 | // circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncFailureRate(10, 0.4)), 321 | // circuitbreaker.WithOnStateChangeHookFn(func(from, to circuitbreaker.State) { 322 | // log.Printf("state changed from %s to %s\n", from, to) 323 | // }), 324 | // ) 325 | // 326 | // The default options are described in the defaultOptions function 327 | func New(opts ...BreakerOption) *CircuitBreaker { 328 | cbOptions := defaultOptions() 329 | 330 | for _, opt := range opts { 331 | opt.apply(cbOptions) 332 | } 333 | 334 | if cbOptions.openTimeout > 0 { 335 | cbOptions.openBackOff = backoff.NewConstantBackOff(cbOptions.openTimeout) 336 | } 337 | 338 | cb := &CircuitBreaker{ 339 | shouldTrip: cbOptions.shouldTrip, 340 | onStateChange: cbOptions.onStateChange, 341 | clock: cbOptions.clock, 342 | interval: cbOptions.interval, 343 | openBackOff: cbOptions.openBackOff, 344 | halfOpenMaxSuccesses: cbOptions.halfOpenMaxSuccesses, 345 | failOnContextCancel: cbOptions.failOnContextCancel, 346 | failOnContextDeadline: cbOptions.failOnContextDeadline, 347 | } 348 | cb.setState(&stateClosed{}) 349 | return cb 350 | } 351 | 352 | // An Operation is executed by Do(). 353 | type Operation func() (interface{}, error) 354 | 355 | // Do executes the Operation o and returns the return values if 356 | // cb.Ready() is true. If not ready, cb doesn't execute f and returns 357 | // ErrOpen. 358 | // 359 | // If o returns a nil-error, cb counts the execution of Operation as a 360 | // success. Otherwise, cb count it as a failure. 361 | // 362 | // If o returns a *IgnorableError, Do() ignores the result of operation and 363 | // returns the wrapped error. 364 | // 365 | // If o returns a *SuccessMarkableError, Do() count it as a success and returns 366 | // the wrapped error. 367 | // 368 | // If given Options' FailOnContextCancel is false (default), cb.Do 369 | // doesn't mark the Operation's error as a failure if ctx.Err() returns 370 | // context.Canceled. 371 | // 372 | // If given Options' FailOnContextDeadline is false (default), cb.Do 373 | // doesn't mark the Operation's error as a failure if ctx.Err() returns 374 | // context.DeadlineExceeded. 375 | func (cb *CircuitBreaker) Do(ctx context.Context, o Operation) (interface{}, error) { 376 | if !cb.Ready() { 377 | return nil, ErrOpen 378 | } 379 | result, err := o() 380 | return result, cb.Done(ctx, err) 381 | } 382 | 383 | // Ready reports if cb is ready to execute an operation. Ready does not give 384 | // any change to cb. 385 | func (cb *CircuitBreaker) Ready() bool { 386 | cb.mu.RLock() 387 | defer cb.mu.RUnlock() 388 | return cb.state.ready(cb) 389 | } 390 | 391 | // Success signals that an execution of operation has been completed 392 | // successfully to cb. 393 | func (cb *CircuitBreaker) Success() { 394 | cb.mu.Lock() 395 | defer cb.mu.Unlock() 396 | cb.cnt.incrementSuccesses() 397 | cb.state.onSuccess(cb) 398 | } 399 | 400 | // Fail signals that an execution of operation has been failed to cb. 401 | func (cb *CircuitBreaker) Fail() { 402 | cb.mu.Lock() 403 | defer cb.mu.Unlock() 404 | cb.cnt.incrementFailures() 405 | cb.state.onFail(cb) 406 | } 407 | 408 | // FailWithContext calls Fail internally. But if FailOnContextCancel is false 409 | // and ctx is done with context.Canceled error, no Fail() called. Similarly, if 410 | // FailOnContextDeadline is false and ctx is done with context.DeadlineExceeded 411 | // error, no Fail() called. 412 | func (cb *CircuitBreaker) FailWithContext(ctx context.Context) { 413 | if ctxErr := ctx.Err(); ctxErr != nil { 414 | if ctxErr == context.Canceled && !cb.failOnContextCancel { 415 | return 416 | } 417 | if ctxErr == context.DeadlineExceeded && !cb.failOnContextDeadline { 418 | return 419 | } 420 | } 421 | cb.Fail() 422 | } 423 | 424 | // Done is a helper function to finish the protected operation. If err is nil, 425 | // Done calls Success and returns nil. If err is a SuccessMarkableError or 426 | // IgnorableError, Done returns wrapped error. Otherwise, Done calls 427 | // FailWithContext internally. 428 | func (cb *CircuitBreaker) Done(ctx context.Context, err error) error { 429 | if err == nil { 430 | cb.Success() 431 | return nil 432 | } 433 | 434 | if successMarkableErr, ok := err.(*SuccessMarkableError); ok { 435 | cb.Success() 436 | return successMarkableErr.Unwrap() 437 | } 438 | 439 | if ignorableErr, ok := err.(*IgnorableError); ok { 440 | return ignorableErr.Unwrap() 441 | } 442 | 443 | cb.FailWithContext(ctx) 444 | return err 445 | } 446 | 447 | // State reports the curent State of cb. 448 | func (cb *CircuitBreaker) State() State { 449 | cb.mu.Lock() 450 | defer cb.mu.Unlock() 451 | return cb.state.State() 452 | } 453 | 454 | // Counters returns internal counters. If current status is not 455 | // StateClosed, returns zero value. 456 | func (cb *CircuitBreaker) Counters() Counters { 457 | cb.mu.Lock() 458 | defer cb.mu.Unlock() 459 | return cb.cnt 460 | } 461 | 462 | // Reset resets cb's state with StateClosed. 463 | func (cb *CircuitBreaker) Reset() { 464 | cb.mu.Lock() 465 | defer cb.mu.Unlock() 466 | cb.cnt.reset() 467 | cb.setState(&stateClosed{}) 468 | } 469 | 470 | // SetState set state of cb to st. 471 | func (cb *CircuitBreaker) SetState(st State) { 472 | switch st { 473 | case StateClosed: 474 | cb.setStateWithLock(&stateClosed{}) 475 | case StateOpen: 476 | cb.setStateWithLock(&stateOpen{}) 477 | case StateHalfOpen: 478 | cb.setStateWithLock(&stateHalfOpen{}) 479 | default: 480 | panic("undefined state") 481 | } 482 | } 483 | 484 | func (cb *CircuitBreaker) setStateWithLock(s state) { 485 | cb.mu.Lock() 486 | defer cb.mu.Unlock() 487 | cb.setState(s) 488 | } 489 | 490 | func (cb *CircuitBreaker) setState(s state) { 491 | if cb.state != nil { 492 | cb.state.onExit(cb) 493 | } 494 | from := cb.state 495 | cb.state = s 496 | cb.state.onEntry(cb) 497 | cb.handleOnStateChange(from, s) 498 | } 499 | 500 | func (cb *CircuitBreaker) handleOnStateChange(from, to state) { 501 | if from == nil || cb.onStateChange == nil { 502 | return 503 | } 504 | cb.onStateChange(from.State(), to.State()) 505 | } 506 | -------------------------------------------------------------------------------- /breaker_test.go: -------------------------------------------------------------------------------- 1 | package circuitbreaker_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "testing" 9 | "time" 10 | 11 | "github.com/benbjohnson/clock" 12 | "github.com/mercari/go-circuitbreaker" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | type user struct { 17 | name string 18 | age int 19 | } 20 | 21 | func fetchUserInfo(ctx context.Context, name string) (*user, error) { 22 | return &user{name: name, age: 30}, nil 23 | } 24 | 25 | func ExampleCircuitBreaker() { 26 | cb := circuitbreaker.New(nil) 27 | ctx := context.Background() 28 | 29 | data, err := cb.Do(context.Background(), func() (interface{}, error) { 30 | user, err := fetchUserInfo(ctx, "太郎") 31 | if err != nil && err.Error() == "UserNoFound" { 32 | // If you received a application level error, wrap it with Ignore to 33 | // avoid false-positive circuit open. 34 | return nil, circuitbreaker.Ignore(err) 35 | } 36 | return user, err 37 | }) 38 | 39 | if err != nil { 40 | log.Fatalf("failed to fetch user:%s\n", err.Error()) 41 | } 42 | log.Printf("fetched user:%+v\n", data.(*user)) 43 | } 44 | 45 | func TestDo(t *testing.T) { 46 | t.Run("success", func(t *testing.T) { 47 | cb := circuitbreaker.New() 48 | got, err := cb.Do(context.Background(), func() (interface{}, error) { 49 | return "data", nil 50 | }) 51 | assert.NoError(t, err) 52 | assert.Equal(t, "data", got.(string)) 53 | assert.Equal(t, int64(0), cb.Counters().Failures) 54 | }) 55 | 56 | t.Run("error", func(t *testing.T) { 57 | cb := circuitbreaker.New() 58 | wantErr := errors.New("something happens") 59 | got, err := cb.Do(context.Background(), func() (interface{}, error) { 60 | return "data", wantErr 61 | }) 62 | assert.Equal(t, err, wantErr) 63 | assert.Equal(t, "data", got.(string)) 64 | assert.Equal(t, int64(1), cb.Counters().Failures) 65 | }) 66 | 67 | t.Run("ignore", func(t *testing.T) { 68 | cb := circuitbreaker.New() 69 | wantErr := errors.New("something happens") 70 | got, err := cb.Do(context.Background(), func() (interface{}, error) { return "data", circuitbreaker.Ignore(wantErr) }) 71 | assert.Equal(t, err, wantErr) 72 | assert.Equal(t, "data", got.(string)) 73 | assert.Equal(t, int64(0), cb.Counters().Failures) 74 | }) 75 | t.Run("markassuccess", func(t *testing.T) { 76 | cb := circuitbreaker.New() 77 | wantErr := errors.New("something happens") 78 | got, err := cb.Do(context.Background(), func() (interface{}, error) { return "data", circuitbreaker.MarkAsSuccess(wantErr) }) 79 | assert.Equal(t, err, wantErr) 80 | assert.Equal(t, "data", got.(string)) 81 | assert.Equal(t, int64(0), cb.Counters().Failures) 82 | }) 83 | 84 | t.Run("context-canceled", func(t *testing.T) { 85 | tests := []struct { 86 | FailOnContextCancel bool 87 | ExpectedFailures int64 88 | }{ 89 | {FailOnContextCancel: true, ExpectedFailures: 1}, 90 | {FailOnContextCancel: false, ExpectedFailures: 0}, 91 | } 92 | for _, test := range tests { 93 | cancelErr := errors.New("context's Done channel closed.") 94 | t.Run(fmt.Sprintf("FailOnContextCanceled=%t", test.FailOnContextCancel), func(t *testing.T) { 95 | cb := circuitbreaker.New(circuitbreaker.WithFailOnContextCancel(test.FailOnContextCancel)) 96 | ctx, cancel := context.WithCancel(context.Background()) 97 | cancel() 98 | got, err := cb.Do(ctx, func() (interface{}, error) { 99 | <-ctx.Done() 100 | return "", cancelErr 101 | }) 102 | assert.Equal(t, err, cancelErr) 103 | assert.Equal(t, "", got.(string)) 104 | assert.Equal(t, test.ExpectedFailures, cb.Counters().Failures) 105 | }) 106 | } 107 | }) 108 | 109 | t.Run("context-timeout", func(t *testing.T) { 110 | tests := []struct { 111 | FailOnContextDeadline bool 112 | ExpectedFailures int64 113 | }{ 114 | {FailOnContextDeadline: true, ExpectedFailures: 1}, 115 | {FailOnContextDeadline: false, ExpectedFailures: 0}, 116 | } 117 | for _, test := range tests { 118 | timeoutErr := errors.New("context's Done channel closed") 119 | t.Run(fmt.Sprintf("FailOnContextDeadline=%t", test.FailOnContextDeadline), func(t *testing.T) { 120 | cb := circuitbreaker.New(circuitbreaker.WithFailOnContextDeadline(test.FailOnContextDeadline)) 121 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) 122 | defer cancel() 123 | got, err := cb.Do(ctx, func() (interface{}, error) { 124 | <-ctx.Done() 125 | return "", timeoutErr 126 | }) 127 | assert.Equal(t, err, timeoutErr) 128 | assert.Equal(t, "", got.(string)) 129 | assert.Equal(t, test.ExpectedFailures, cb.Counters().Failures) 130 | }) 131 | } 132 | }) 133 | 134 | t.Run("cyclic-state-transition", func(t *testing.T) { 135 | clkMock := clock.NewMock() 136 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(3)), 137 | circuitbreaker.WithClock(clkMock), 138 | circuitbreaker.WithOpenTimeout(1000*time.Millisecond), 139 | circuitbreaker.WithHalfOpenMaxSuccesses(4)) 140 | 141 | wantErr := errors.New("something happens") 142 | 143 | // ( Closed => Open => HalfOpen => Open => HalfOpen => Closed ) x 10 iterations. 144 | for i := 0; i < 10; i++ { 145 | 146 | // State: Closed. 147 | for i := 0; i < 3; i++ { 148 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 149 | got, err := cb.Do(context.Background(), func() (interface{}, error) { return "data", wantErr }) 150 | assert.Equal(t, err, wantErr) 151 | assert.Equal(t, "data", got.(string)) 152 | } 153 | 154 | // State: Closed => Open. Should return nil and ErrOpen error. 155 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 156 | got, err := cb.Do(context.Background(), func() (interface{}, error) { return "data", wantErr }) 157 | assert.Equal(t, err, circuitbreaker.ErrOpen) 158 | assert.Nil(t, got) 159 | 160 | // State: Open => HalfOpen. 161 | clkMock.Add(1000 * time.Millisecond) 162 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 163 | 164 | // State: HalfOpen => Open. 165 | got, err = cb.Do(context.Background(), func() (interface{}, error) { return "data", wantErr }) 166 | assert.Equal(t, err, wantErr) 167 | assert.Equal(t, "data", got.(string)) 168 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 169 | 170 | // State: Open => HalfOpen. 171 | clkMock.Add(1000 * time.Millisecond) 172 | 173 | // State: HalfOpen => Close. 174 | for i := 0; i < 4; i++ { 175 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 176 | got, err = cb.Do(context.Background(), func() (interface{}, error) { return "data", nil }) 177 | assert.NoError(t, err) 178 | assert.Equal(t, "data", got.(string)) 179 | } 180 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 181 | } 182 | }) 183 | } 184 | 185 | func TestCircuitBreakerTripFuncs(t *testing.T) { 186 | t.Run("TripFuncThreshold", func(t *testing.T) { 187 | shouldTrip := circuitbreaker.NewTripFuncThreshold(5) 188 | assert.False(t, shouldTrip(&circuitbreaker.Counters{Failures: 4})) 189 | assert.True(t, shouldTrip(&circuitbreaker.Counters{Failures: 5})) 190 | assert.True(t, shouldTrip(&circuitbreaker.Counters{Failures: 6})) 191 | }) 192 | t.Run("TripFuncConsecutiveFailures", func(t *testing.T) { 193 | shouldTrip := circuitbreaker.NewTripFuncConsecutiveFailures(5) 194 | assert.False(t, shouldTrip(&circuitbreaker.Counters{ConsecutiveFailures: 4})) 195 | assert.True(t, shouldTrip(&circuitbreaker.Counters{ConsecutiveFailures: 5})) 196 | assert.True(t, shouldTrip(&circuitbreaker.Counters{ConsecutiveFailures: 6})) 197 | }) 198 | t.Run("TripFuncFailureRate", func(t *testing.T) { 199 | shouldTrip := circuitbreaker.NewTripFuncFailureRate(10, 0.4) 200 | assert.False(t, shouldTrip(&circuitbreaker.Counters{Successes: 1, Failures: 8})) 201 | assert.True(t, shouldTrip(&circuitbreaker.Counters{Successes: 1, Failures: 9})) 202 | assert.False(t, shouldTrip(&circuitbreaker.Counters{Successes: 60, Failures: 39})) 203 | assert.True(t, shouldTrip(&circuitbreaker.Counters{Successes: 60, Failures: 40})) 204 | assert.True(t, shouldTrip(&circuitbreaker.Counters{Successes: 60, Failures: 41})) 205 | }) 206 | } 207 | 208 | func TestIgnore(t *testing.T) { 209 | t.Run("nil", func(t *testing.T) { 210 | assert.Nil(t, circuitbreaker.Ignore(nil)) 211 | }) 212 | t.Run("ignore", func(t *testing.T) { 213 | originalErr := errors.New("logic error") 214 | if err := circuitbreaker.Ignore(originalErr); err != nil { 215 | assert.Equal(t, err.Error(), "circuitbreaker does not mark this error as a failure: logic error") 216 | nfe, ok := err.(*circuitbreaker.IgnorableError) 217 | assert.True(t, ok) 218 | assert.Equal(t, nfe.Unwrap(), originalErr) 219 | } else { 220 | assert.Fail(t, "there should be an error here") 221 | } 222 | }) 223 | } 224 | 225 | func TestMarkAsSuccess(t *testing.T) { 226 | t.Run("nil", func(t *testing.T) { 227 | assert.Nil(t, circuitbreaker.MarkAsSuccess(nil)) 228 | }) 229 | t.Run("MarkAsSuccess", func(t *testing.T) { 230 | originalErr := errors.New("logic error") 231 | if err := circuitbreaker.MarkAsSuccess(originalErr); err != nil { 232 | assert.Equal(t, err.Error(), "circuitbreaker mark this error as a success: logic error") 233 | nfe, ok := err.(*circuitbreaker.SuccessMarkableError) 234 | assert.True(t, ok) 235 | assert.Equal(t, nfe.Unwrap(), originalErr) 236 | } else { 237 | assert.Fail(t, "there should be an error here") 238 | } 239 | }) 240 | } 241 | 242 | func TestSuccess(t *testing.T) { 243 | cb := circuitbreaker.New() 244 | cb.Success() 245 | assert.Equal(t, circuitbreaker.Counters{Successes: 1, Failures: 0, ConsecutiveSuccesses: 1, ConsecutiveFailures: 0}, cb.Counters()) 246 | 247 | // Test if Success resets ConsecutiveFailures. 248 | cb.Fail() 249 | cb.Success() 250 | assert.Equal(t, circuitbreaker.Counters{Successes: 2, Failures: 1, ConsecutiveSuccesses: 1, ConsecutiveFailures: 0}, cb.Counters()) 251 | 252 | } 253 | 254 | func TestFail(t *testing.T) { 255 | cb := circuitbreaker.New() 256 | cb.Fail() 257 | assert.Equal(t, circuitbreaker.Counters{Successes: 0, Failures: 1, ConsecutiveSuccesses: 0, ConsecutiveFailures: 1}, cb.Counters()) 258 | 259 | // Test if Fail resets ConsecutiveSuccesses. 260 | cb.Success() 261 | cb.Fail() 262 | assert.Equal(t, circuitbreaker.Counters{Successes: 1, Failures: 2, ConsecutiveSuccesses: 0, ConsecutiveFailures: 1}, cb.Counters()) 263 | } 264 | 265 | // TestReset tests if Reset resets all counters. 266 | func TestReset(t *testing.T) { 267 | cb := circuitbreaker.New() 268 | cb.Success() 269 | cb.Reset() 270 | assert.Equal(t, circuitbreaker.Counters{}, cb.Counters()) 271 | 272 | cb.Fail() 273 | cb.Reset() 274 | assert.Equal(t, circuitbreaker.Counters{}, cb.Counters()) 275 | } 276 | 277 | func TestReportFunctions(t *testing.T) { 278 | t.Run("Failed if ctx.Err() == nil", func(t *testing.T) { 279 | cb := circuitbreaker.New() 280 | cb.FailWithContext(context.Background()) 281 | assert.Equal(t, int64(1), cb.Counters().Failures) 282 | }) 283 | t.Run("ctx.Err() == context.Canceled", func(t *testing.T) { 284 | ctx, cancel := context.WithCancel(context.Background()) 285 | cancel() 286 | 287 | cb := circuitbreaker.New() 288 | cb.FailWithContext(ctx) 289 | assert.Equal(t, int64(0), cb.Counters().Failures) 290 | 291 | cb = circuitbreaker.New(circuitbreaker.WithFailOnContextCancel(true)) 292 | cb.FailWithContext(ctx) 293 | assert.Equal(t, int64(1), cb.Counters().Failures) 294 | }) 295 | t.Run("ctx.Err() == context.DeadlineExceeded", func(t *testing.T) { 296 | ctx, cancel := context.WithDeadline(context.Background(), time.Time{}) 297 | defer cancel() 298 | cb := circuitbreaker.New() 299 | cb.FailWithContext(ctx) 300 | assert.Equal(t, int64(0), cb.Counters().Failures) 301 | 302 | cb = circuitbreaker.New(circuitbreaker.WithFailOnContextDeadline(true)) 303 | cb.FailWithContext(ctx) 304 | assert.Equal(t, int64(1), cb.Counters().Failures) 305 | }) 306 | } 307 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mercari/go-circuitbreaker 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/benbjohnson/clock v1.3.0 7 | github.com/cenkalti/backoff/v3 v3.1.1 8 | github.com/stretchr/testify v1.4.0 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= 2 | github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 3 | github.com/cenkalti/backoff/v3 v3.1.1 h1:UBHElAnr3ODEbpqPzX8g5sBcASjoLFtt3L/xwJ01L6E= 4 | github.com/cenkalti/backoff/v3 v3.1.1/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= 5 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 8 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 9 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 10 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 11 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 12 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 14 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 15 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 16 | -------------------------------------------------------------------------------- /state.go: -------------------------------------------------------------------------------- 1 | package circuitbreaker 2 | 3 | import ( 4 | "github.com/benbjohnson/clock" 5 | "github.com/cenkalti/backoff/v3" 6 | ) 7 | 8 | // each implementations of state represents State of circuit breaker. 9 | // 10 | // ref: https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker 11 | type state interface { 12 | State() State 13 | onEntry(cb *CircuitBreaker) 14 | onExit(cb *CircuitBreaker) 15 | ready(cb *CircuitBreaker) bool 16 | onSuccess(cb *CircuitBreaker) 17 | onFail(cb *CircuitBreaker) 18 | } 19 | 20 | // [Closed state] 21 | // /onEntry 22 | // - Reset counters. 23 | // - Start ticker. 24 | // /ready 25 | // - returns true. 26 | // /onFail 27 | // - update counters. 28 | // - If threshold reached, change state to [Open] 29 | // /onTicker 30 | // - reset counters. 31 | // /onExit 32 | // - stop ticker. 33 | type stateClosed struct { 34 | ticker *clock.Ticker 35 | done chan struct{} 36 | } 37 | 38 | func (st *stateClosed) State() State { return StateClosed } 39 | func (st *stateClosed) onEntry(cb *CircuitBreaker) { 40 | cb.cnt.resetFailures() 41 | cb.openBackOff.Reset() 42 | if cb.interval > 0 { 43 | st.ticker = cb.clock.Ticker(cb.interval) 44 | st.done = make(chan struct{}) 45 | go func() { 46 | for { 47 | select { 48 | case <-st.ticker.C: 49 | st.onTicker(cb) 50 | case <-st.done: 51 | st.ticker.Stop() 52 | return 53 | } 54 | } 55 | }() 56 | } 57 | } 58 | 59 | func (st *stateClosed) onExit(cb *CircuitBreaker) { 60 | if st.done != nil { 61 | close(st.done) 62 | } 63 | } 64 | 65 | func (st *stateClosed) onTicker(cb *CircuitBreaker) { 66 | cb.mu.Lock() 67 | defer cb.mu.Unlock() 68 | cb.cnt.reset() 69 | } 70 | 71 | func (st *stateClosed) ready(cb *CircuitBreaker) bool { return true } 72 | func (st *stateClosed) onSuccess(cb *CircuitBreaker) {} 73 | func (st *stateClosed) onFail(cb *CircuitBreaker) { 74 | if cb.shouldTrip(&cb.cnt) { 75 | cb.setState(&stateOpen{}) 76 | } 77 | } 78 | 79 | // [Open state] 80 | // /onEntry 81 | // - Start timer. 82 | // /ready 83 | // - Returns false. 84 | // /onTimer 85 | // - Change state to [HalfOpen]. 86 | // /onExit 87 | // - Stop timer. 88 | type stateOpen struct { 89 | timer *clock.Timer 90 | } 91 | 92 | func (st *stateOpen) State() State { return StateOpen } 93 | func (st *stateOpen) onEntry(cb *CircuitBreaker) { 94 | timeout := cb.openBackOff.NextBackOff() 95 | if timeout != backoff.Stop { 96 | st.timer = cb.clock.AfterFunc(timeout, func() { st.onTimer(cb) }) 97 | } 98 | } 99 | 100 | func (st *stateOpen) onTimer(cb *CircuitBreaker) { cb.setStateWithLock(&stateHalfOpen{}) } 101 | func (st *stateOpen) onExit(cb *CircuitBreaker) { st.timer.Stop() } 102 | func (st *stateOpen) ready(cb *CircuitBreaker) bool { return false } 103 | func (st *stateOpen) onSuccess(cb *CircuitBreaker) {} 104 | func (st *stateOpen) onFail(cb *CircuitBreaker) {} 105 | 106 | // [HalfOpen state] 107 | // /ready 108 | // -> returns true 109 | // /onSuccess 110 | // -> Increment Success counter. 111 | // -> If threshold reached, change state to [Closed]. 112 | // /onFail 113 | // -> change state to [Open]. 114 | type stateHalfOpen struct{} 115 | 116 | func (st *stateHalfOpen) State() State { return StateHalfOpen } 117 | func (st *stateHalfOpen) onEntry(cb *CircuitBreaker) { cb.cnt.resetSuccesses() } 118 | func (st *stateHalfOpen) onExit(cb *CircuitBreaker) {} 119 | func (st *stateHalfOpen) ready(cb *CircuitBreaker) bool { return true } 120 | func (st *stateHalfOpen) onSuccess(cb *CircuitBreaker) { 121 | if cb.cnt.Successes >= cb.halfOpenMaxSuccesses { 122 | cb.setState(&stateClosed{}) 123 | } 124 | } 125 | func (st *stateHalfOpen) onFail(cb *CircuitBreaker) { 126 | cb.setState(&stateOpen{}) 127 | } 128 | -------------------------------------------------------------------------------- /state_test.go: -------------------------------------------------------------------------------- 1 | package circuitbreaker_test 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/benbjohnson/clock" 11 | "github.com/cenkalti/backoff/v3" 12 | "github.com/mercari/go-circuitbreaker" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestCircuitBreakerStateTransitions(t *testing.T) { 17 | clk := clock.NewMock() 18 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(3)), 19 | circuitbreaker.WithClock(clk), 20 | circuitbreaker.WithOpenTimeout(1000*time.Millisecond), 21 | circuitbreaker.WithHalfOpenMaxSuccesses(4)) 22 | 23 | for i := 0; i < 10; i++ { 24 | // Scenario: 3 Fails. State changes to -> StateOpen. 25 | cb.Fail() 26 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 27 | cb.Fail() 28 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 29 | cb.Fail() 30 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 31 | 32 | // Scenario: After OpenTimeout exceeded. -> StateHalfOpen. 33 | assertChangeStateToHalfOpenAfter(t, cb, clk, 1000*time.Millisecond) 34 | 35 | // Scenario: Hit Fail. State back to StateOpen. 36 | cb.Fail() 37 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 38 | 39 | // Scenario: After OpenTimeout exceeded. -> StateHalfOpen. (again) 40 | assertChangeStateToHalfOpenAfter(t, cb, clk, 1000*time.Millisecond) 41 | 42 | // Scenario: Hit Success. State -> StateClosed. 43 | cb.Success() 44 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 45 | cb.Success() 46 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 47 | cb.Success() 48 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 49 | cb.Success() 50 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 51 | } 52 | } 53 | 54 | func TestCircuitBreakerOnStateChange(t *testing.T) { 55 | type stateChange struct { 56 | from circuitbreaker.State 57 | to circuitbreaker.State 58 | } 59 | 60 | expectedStateChanges := []stateChange{ 61 | { 62 | from: circuitbreaker.StateClosed, 63 | to: circuitbreaker.StateOpen, 64 | }, 65 | { 66 | from: circuitbreaker.StateOpen, 67 | to: circuitbreaker.StateHalfOpen, 68 | }, 69 | { 70 | from: circuitbreaker.StateHalfOpen, 71 | to: circuitbreaker.StateOpen, 72 | }, 73 | { 74 | from: circuitbreaker.StateOpen, 75 | to: circuitbreaker.StateHalfOpen, 76 | }, 77 | { 78 | from: circuitbreaker.StateHalfOpen, 79 | to: circuitbreaker.StateClosed, 80 | }, 81 | } 82 | var actualStateChanges []stateChange 83 | 84 | clock := clock.NewMock() 85 | cb := circuitbreaker.New( 86 | circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(3)), 87 | circuitbreaker.WithClock(clock), 88 | circuitbreaker.WithOpenTimeout(1000*time.Millisecond), 89 | circuitbreaker.WithHalfOpenMaxSuccesses(4), 90 | circuitbreaker.WithOnStateChangeHookFn(func(from, to circuitbreaker.State) { 91 | actualStateChanges = append(actualStateChanges, stateChange{ 92 | from: from, 93 | to: to, 94 | }) 95 | }), 96 | ) 97 | 98 | // Scenario: 3 Fails. State changes to -> StateOpen. 99 | cb.Fail() 100 | cb.Fail() 101 | cb.Fail() 102 | 103 | // Scenario: After OpenTimeout exceeded. -> StateHalfOpen. 104 | assertChangeStateToHalfOpenAfter(t, cb, clock, 1000*time.Millisecond) 105 | 106 | // Scenario: Hit Fail. State back to StateOpen. 107 | cb.Fail() 108 | 109 | // Scenario: After OpenTimeout exceeded. -> StateHalfOpen. (again) 110 | assertChangeStateToHalfOpenAfter(t, cb, clock, 1000*time.Millisecond) 111 | 112 | // Scenario: Hit Success. State -> StateClosed. 113 | cb.Success() 114 | cb.Success() 115 | cb.Success() 116 | cb.Success() 117 | 118 | assert.Equal(t, expectedStateChanges, actualStateChanges) 119 | } 120 | 121 | // TestStateClosed tests... 122 | // - Ready() always returns true. 123 | // - Change state if Failures threshold reached. 124 | // - Interval ticker reset the internal counter.. 125 | func TestStateClosed(t *testing.T) { 126 | clk := clock.NewMock() 127 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(3)), 128 | circuitbreaker.WithClock(clk), 129 | circuitbreaker.WithCounterResetInterval(1000*time.Millisecond)) 130 | 131 | t.Run("Ready", func(t *testing.T) { 132 | assert.True(t, cb.Ready()) 133 | }) 134 | 135 | t.Run("open-if-shouldtrip-reached", func(t *testing.T) { 136 | cb.Reset() 137 | cb.Fail() 138 | cb.Fail() 139 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 140 | cb.Fail() 141 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 142 | }) 143 | 144 | t.Run("ticker-reset-the-counter", func(t *testing.T) { 145 | cb.Reset() 146 | cb.Success() 147 | cb.Fail() 148 | clk.Add(999 * time.Millisecond) 149 | assert.Equal(t, circuitbreaker.Counters{Successes: 1, Failures: 1, ConsecutiveFailures: 1}, cb.Counters()) 150 | clk.Add(1 * time.Millisecond) 151 | assert.Equal(t, circuitbreaker.Counters{}, cb.Counters()) 152 | }) 153 | } 154 | 155 | // TestStateOpen tests... 156 | // - Ready() always returns false. 157 | // - Change state to StateHalfOpen after timer. 158 | func TestStateOpen(t *testing.T) { 159 | clk := clock.NewMock() 160 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(3)), 161 | circuitbreaker.WithClock(clk), 162 | circuitbreaker.WithOpenTimeout(500*time.Millisecond)) 163 | t.Run("Ready", func(t *testing.T) { 164 | cb.SetState(circuitbreaker.StateOpen) 165 | assert.False(t, cb.Ready()) 166 | }) 167 | t.Run("HalfOpen-when-timer-triggered", func(t *testing.T) { 168 | cb.SetState(circuitbreaker.StateOpen) 169 | cb.Fail() 170 | cb.Success() 171 | 172 | clk.Add(499 * time.Millisecond) 173 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 174 | 175 | clk.Add(1 * time.Millisecond) 176 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 177 | assert.Equal(t, circuitbreaker.Counters{Failures: 1}, cb.Counters()) // successes reset. 178 | }) 179 | t.Run("HalfOpen-with-ExponentialOpenBackOff", func(t *testing.T) { 180 | clkMock := clock.NewMock() 181 | backoffTest := &backoff.ExponentialBackOff{ 182 | InitialInterval: 1000 * time.Millisecond, 183 | RandomizationFactor: 0, 184 | Multiplier: 2, 185 | MaxInterval: 5 * time.Second, 186 | MaxElapsedTime: 0, 187 | Clock: clkMock, 188 | } 189 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(1)), 190 | circuitbreaker.WithHalfOpenMaxSuccesses(1), 191 | circuitbreaker.WithClock(clkMock), 192 | circuitbreaker.WithOpenTimeoutBackOff(backoffTest)) 193 | backoffTest.Reset() 194 | 195 | tests := []struct { 196 | f func() 197 | after time.Duration 198 | }{ 199 | {f: cb.Fail, after: 1000 * time.Millisecond}, 200 | {f: cb.Fail, after: 2000 * time.Millisecond}, 201 | {f: cb.Fail, after: 4000 * time.Millisecond}, 202 | {f: cb.Fail, after: 5000 * time.Millisecond}, 203 | {f: func() { cb.Success(); cb.Fail() }, after: 1000 * time.Millisecond}, 204 | } 205 | for _, test := range tests { 206 | test.f() 207 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 208 | 209 | clkMock.Add(test.after - 1) 210 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 211 | 212 | clkMock.Add(1) 213 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 214 | } 215 | }) 216 | t.Run("OpenBackOff", func(t *testing.T) { 217 | clkMock := clock.NewMock() 218 | backoffTest := &backoff.ExponentialBackOff{ 219 | InitialInterval: 1000 * time.Millisecond, 220 | RandomizationFactor: 0, 221 | Multiplier: 2, 222 | MaxInterval: 5 * time.Second, 223 | MaxElapsedTime: 0, 224 | Clock: clkMock, 225 | } 226 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(1)), 227 | circuitbreaker.WithHalfOpenMaxSuccesses(1), 228 | circuitbreaker.WithClock(clkMock), 229 | circuitbreaker.WithOpenTimeoutBackOff(backoffTest)) 230 | backoffTest.Reset() 231 | 232 | tests := []struct { 233 | f func() 234 | after time.Duration 235 | }{ 236 | {f: cb.Fail, after: 1000 * time.Millisecond}, 237 | {f: cb.Fail, after: 2000 * time.Millisecond}, 238 | {f: cb.Fail, after: 4000 * time.Millisecond}, 239 | {f: cb.Fail, after: 5000 * time.Millisecond}, 240 | {f: func() { cb.Success(); cb.Fail() }, after: 1000 * time.Millisecond}, 241 | } 242 | for _, test := range tests { 243 | test.f() 244 | assertChangeStateToHalfOpenAfter(t, cb, clkMock, test.after) 245 | } 246 | }) 247 | } 248 | 249 | func assertChangeStateToHalfOpenAfter(t *testing.T, cb *circuitbreaker.CircuitBreaker, clock *clock.Mock, after time.Duration) { 250 | clock.Add(after - 1) 251 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 252 | 253 | clock.Add(1) 254 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 255 | } 256 | 257 | // StateOpen Test 258 | // - Ready() always returns true. 259 | // - If get a fail, the state changes to Open. 260 | // - If get a success, the state changes to Closed. 261 | func TestHalfOpen(t *testing.T) { 262 | clkMock := clock.NewMock() 263 | cb := circuitbreaker.New(circuitbreaker.WithTripFunc(circuitbreaker.NewTripFuncThreshold(3)), 264 | circuitbreaker.WithClock(clkMock), 265 | circuitbreaker.WithHalfOpenMaxSuccesses(4)) 266 | t.Run("Ready", func(t *testing.T) { 267 | cb.Reset() 268 | cb.SetState(circuitbreaker.StateHalfOpen) 269 | assert.True(t, cb.Ready()) 270 | }) 271 | t.Run("Open-if-got-a-fail", func(t *testing.T) { 272 | cb.Reset() 273 | cb.SetState(circuitbreaker.StateHalfOpen) 274 | 275 | cb.Fail() 276 | assert.Equal(t, circuitbreaker.StateOpen, cb.State()) 277 | assert.Equal(t, circuitbreaker.Counters{Failures: 1, ConsecutiveFailures: 1}, cb.Counters()) // no reset 278 | }) 279 | t.Run("Close-if-success-reaches-HalfOpenMaxSuccesses", func(t *testing.T) { 280 | cb.Reset() 281 | cb.Fail() 282 | cb.SetState(circuitbreaker.StateHalfOpen) 283 | 284 | cb.Success() 285 | cb.Success() 286 | cb.Success() 287 | assert.Equal(t, circuitbreaker.StateHalfOpen, cb.State()) 288 | 289 | cb.Success() 290 | assert.Equal(t, circuitbreaker.StateClosed, cb.State()) 291 | assert.Equal(t, circuitbreaker.Counters{Successes: 4, Failures: 0, ConsecutiveSuccesses: 4}, cb.Counters()) // Failures reset. 292 | }) 293 | } 294 | 295 | func run(wg *sync.WaitGroup, f func()) { 296 | wg.Add(1) 297 | go func() { 298 | defer wg.Done() 299 | f() 300 | }() 301 | } 302 | 303 | func TestRace(t *testing.T) { 304 | clock := clock.NewMock() 305 | cb := circuitbreaker.New( 306 | circuitbreaker.WithTripFunc(func(_ *circuitbreaker.Counters) bool { return true }), 307 | circuitbreaker.WithClock(clock), 308 | circuitbreaker.WithCounterResetInterval(1000*time.Millisecond), 309 | ) 310 | wg := &sync.WaitGroup{} 311 | run(wg, func() { 312 | cb.SetState(circuitbreaker.StateClosed) 313 | }) 314 | run(wg, func() { 315 | cb.Reset() 316 | }) 317 | run(wg, func() { 318 | _ = cb.Done(context.Background(), errors.New("")) 319 | }) 320 | run(wg, func() { 321 | _, _ = cb.Do(context.Background(), func() (interface{}, error) { 322 | return nil, nil 323 | }) 324 | }) 325 | run(wg, func() { 326 | cb.State() 327 | }) 328 | run(wg, func() { 329 | cb.Fail() 330 | }) 331 | run(wg, func() { 332 | cb.Counters() 333 | }) 334 | run(wg, func() { 335 | cb.Ready() 336 | }) 337 | run(wg, func() { 338 | cb.Success() 339 | }) 340 | run(wg, func() { 341 | cb.FailWithContext(context.Background()) 342 | }) 343 | wg.Wait() 344 | } 345 | --------------------------------------------------------------------------------