├── .travis.yml ├── LICENSE ├── README.md ├── clock.go ├── example_test.go ├── exp.go ├── go.mod ├── go.sum ├── regular.go ├── retry.go ├── retry_test.go └── strategy.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go_import_path: gopkg.in/retry.v1 3 | go: 4 | - 1.8.x 5 | - 1.x 6 | - master 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2011-2015 Canonical Ltd. 2 | This software is licensed under the LGPLv3, included below. 3 | 4 | As a special exception to the GNU Lesser General Public License version 3 5 | ("LGPL3"), the copyright holders of this Library give you permission to 6 | convey to a third party a Combined Work that links statically or dynamically 7 | to this Library without providing any Minimal Corresponding Source or 8 | Minimal Application Code as set out in 4d or providing the installation 9 | information set out in section 4e, provided that you comply with the other 10 | provisions of LGPL3 and provided that you meet, for the Application the 11 | terms and conditions of the license(s) which apply to the Application. 12 | 13 | Except as stated in this special exception, the provisions of LGPL3 will 14 | continue to comply in full to this Library. If you modify this Library, you 15 | may apply this exception to your version of this Library, but you are not 16 | obliged to do so. If you do not wish to do so, delete this exception 17 | statement from your version. This exception does not (and cannot) modify any 18 | license terms which apply to the Application, with which you must still 19 | comply. 20 | 21 | 22 | GNU LESSER GENERAL PUBLIC LICENSE 23 | Version 3, 29 June 2007 24 | 25 | Copyright (C) 2007 Free Software Foundation, Inc. 26 | Everyone is permitted to copy and distribute verbatim copies 27 | of this license document, but changing it is not allowed. 28 | 29 | 30 | This version of the GNU Lesser General Public License incorporates 31 | the terms and conditions of version 3 of the GNU General Public 32 | License, supplemented by the additional permissions listed below. 33 | 34 | 0. Additional Definitions. 35 | 36 | As used herein, "this License" refers to version 3 of the GNU Lesser 37 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 38 | General Public License. 39 | 40 | "The Library" refers to a covered work governed by this License, 41 | other than an Application or a Combined Work as defined below. 42 | 43 | An "Application" is any work that makes use of an interface provided 44 | by the Library, but which is not otherwise based on the Library. 45 | Defining a subclass of a class defined by the Library is deemed a mode 46 | of using an interface provided by the Library. 47 | 48 | A "Combined Work" is a work produced by combining or linking an 49 | Application with the Library. The particular version of the Library 50 | with which the Combined Work was made is also called the "Linked 51 | Version". 52 | 53 | The "Minimal Corresponding Source" for a Combined Work means the 54 | Corresponding Source for the Combined Work, excluding any source code 55 | for portions of the Combined Work that, considered in isolation, are 56 | based on the Application, and not on the Linked Version. 57 | 58 | The "Corresponding Application Code" for a Combined Work means the 59 | object code and/or source code for the Application, including any data 60 | and utility programs needed for reproducing the Combined Work from the 61 | Application, but excluding the System Libraries of the Combined Work. 62 | 63 | 1. Exception to Section 3 of the GNU GPL. 64 | 65 | You may convey a covered work under sections 3 and 4 of this License 66 | without being bound by section 3 of the GNU GPL. 67 | 68 | 2. Conveying Modified Versions. 69 | 70 | If you modify a copy of the Library, and, in your modifications, a 71 | facility refers to a function or data to be supplied by an Application 72 | that uses the facility (other than as an argument passed when the 73 | facility is invoked), then you may convey a copy of the modified 74 | version: 75 | 76 | a) under this License, provided that you make a good faith effort to 77 | ensure that, in the event an Application does not supply the 78 | function or data, the facility still operates, and performs 79 | whatever part of its purpose remains meaningful, or 80 | 81 | b) under the GNU GPL, with none of the additional permissions of 82 | this License applicable to that copy. 83 | 84 | 3. Object Code Incorporating Material from Library Header Files. 85 | 86 | The object code form of an Application may incorporate material from 87 | a header file that is part of the Library. You may convey such object 88 | code under terms of your choice, provided that, if the incorporated 89 | material is not limited to numerical parameters, data structure 90 | layouts and accessors, or small macros, inline functions and templates 91 | (ten or fewer lines in length), you do both of the following: 92 | 93 | a) Give prominent notice with each copy of the object code that the 94 | Library is used in it and that the Library and its use are 95 | covered by this License. 96 | 97 | b) Accompany the object code with a copy of the GNU GPL and this license 98 | document. 99 | 100 | 4. Combined Works. 101 | 102 | You may convey a Combined Work under terms of your choice that, 103 | taken together, effectively do not restrict modification of the 104 | portions of the Library contained in the Combined Work and reverse 105 | engineering for debugging such modifications, if you also do each of 106 | the following: 107 | 108 | a) Give prominent notice with each copy of the Combined Work that 109 | the Library is used in it and that the Library and its use are 110 | covered by this License. 111 | 112 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 113 | document. 114 | 115 | c) For a Combined Work that displays copyright notices during 116 | execution, include the copyright notice for the Library among 117 | these notices, as well as a reference directing the user to the 118 | copies of the GNU GPL and this license document. 119 | 120 | d) Do one of the following: 121 | 122 | 0) Convey the Minimal Corresponding Source under the terms of this 123 | License, and the Corresponding Application Code in a form 124 | suitable for, and under terms that permit, the user to 125 | recombine or relink the Application with a modified version of 126 | the Linked Version to produce a modified Combined Work, in the 127 | manner specified by section 6 of the GNU GPL for conveying 128 | Corresponding Source. 129 | 130 | 1) Use a suitable shared library mechanism for linking with the 131 | Library. A suitable mechanism is one that (a) uses at run time 132 | a copy of the Library already present on the user's computer 133 | system, and (b) will operate properly with a modified version 134 | of the Library that is interface-compatible with the Linked 135 | Version. 136 | 137 | e) Provide Installation Information, but only if you would otherwise 138 | be required to provide such information under section 6 of the 139 | GNU GPL, and only to the extent that such information is 140 | necessary to install and execute a modified version of the 141 | Combined Work produced by recombining or relinking the 142 | Application with a modified version of the Linked Version. (If 143 | you use option 4d0, the Installation Information must accompany 144 | the Minimal Corresponding Source and Corresponding Application 145 | Code. If you use option 4d1, you must provide the Installation 146 | Information in the manner specified by section 6 of the GNU GPL 147 | for conveying Corresponding Source.) 148 | 149 | 5. Combined Libraries. 150 | 151 | You may place library facilities that are a work based on the 152 | Library side by side in a single library together with other library 153 | facilities that are not Applications and are not covered by this 154 | License, and convey such a combined library under terms of your 155 | choice, if you do both of the following: 156 | 157 | a) Accompany the combined library with a copy of the same work based 158 | on the Library, uncombined with any other library facilities, 159 | conveyed under the terms of this License. 160 | 161 | b) Give prominent notice with the combined library that part of it 162 | is a work based on the Library, and explaining where to find the 163 | accompanying uncombined form of the same work. 164 | 165 | 6. Revised Versions of the GNU Lesser General Public License. 166 | 167 | The Free Software Foundation may publish revised and/or new versions 168 | of the GNU Lesser General Public License from time to time. Such new 169 | versions will be similar in spirit to the present version, but may 170 | differ in detail to address new problems or concerns. 171 | 172 | Each version is given a distinguishing version number. If the 173 | Library as you received it specifies that a certain numbered version 174 | of the GNU Lesser General Public License "or any later version" 175 | applies to it, you have the option of following the terms and 176 | conditions either of that published version or of any later version 177 | published by the Free Software Foundation. If the Library as you 178 | received it does not specify a version number of the GNU Lesser 179 | General Public License, you may choose any version of the GNU Lesser 180 | General Public License ever published by the Free Software Foundation. 181 | 182 | If the Library as you received it specifies that a proxy can decide 183 | whether future versions of the GNU Lesser General Public License shall 184 | apply, that proxy's public statement of acceptance of any version is 185 | permanent authorization for you to choose that version for the 186 | Library. 187 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # retry 2 | -- 3 | import "gopkg.in/retry.v1" 4 | 5 | Package retry provides a framework for retrying actions. It does not itself 6 | invoke the action to be retried, but is intended to be used in a retry loop. 7 | 8 | The basic usage is as follows: 9 | 10 | for a := someStrategy.Start(); a.Next(); { 11 | try() 12 | } 13 | 14 | See examples for details of suggested usage. 15 | 16 | ## Usage 17 | 18 | #### type Attempt 19 | 20 | ```go 21 | type Attempt struct { 22 | } 23 | ``` 24 | 25 | Attempt represents a running retry attempt. 26 | 27 | #### func Start 28 | 29 | ```go 30 | func Start(strategy Strategy, clk Clock) *Attempt 31 | ``` 32 | Start begins a new sequence of attempts for the given strategy using the given 33 | Clock implementation for time keeping. If clk is nil, the time package will be 34 | used to keep time. 35 | 36 | #### func StartWithCancel 37 | 38 | ```go 39 | func StartWithCancel(strategy Strategy, clk Clock, stop <-chan struct{}) *Attempt 40 | ``` 41 | StartWithCancel is like Start except that if a value is received on stop while 42 | waiting, the attempt will be aborted. 43 | 44 | #### func (*Attempt) Count 45 | 46 | ```go 47 | func (a *Attempt) Count() int 48 | ``` 49 | Count returns the current attempt count number, starting at 1. It returns 0 if 50 | called before Next is called. When the loop has terminated, it holds the total 51 | number of retries made. 52 | 53 | #### func (*Attempt) More 54 | 55 | ```go 56 | func (a *Attempt) More() bool 57 | ``` 58 | More reports whether there are more retry attempts to be made. It does not 59 | sleep. 60 | 61 | If More returns false, Next will return false. If More returns true, Next will 62 | return true except when the attempt has been explicitly stopped via the stop 63 | channel. 64 | 65 | #### func (*Attempt) Next 66 | 67 | ```go 68 | func (a *Attempt) Next() bool 69 | ``` 70 | Next reports whether another attempt should be made, waiting as necessary until 71 | it's time for the attempt. It always returns true the first time it is called 72 | unless a value is received on the stop channel - we are guaranteed to make at 73 | least one attempt unless stopped. 74 | 75 | #### func (*Attempt) Stopped 76 | 77 | ```go 78 | func (a *Attempt) Stopped() bool 79 | ``` 80 | Stopped reports whether the attempt has terminated because a value was received 81 | on the stop channel. 82 | 83 | #### type Clock 84 | 85 | ```go 86 | type Clock interface { 87 | Now() time.Time 88 | After(time.Duration) <-chan time.Time 89 | } 90 | ``` 91 | 92 | Clock represents a virtual clock interface that can be replaced for testing. 93 | 94 | #### type Exponential 95 | 96 | ```go 97 | type Exponential struct { 98 | // Initial holds the initial delay. 99 | Initial time.Duration 100 | // Factor holds the factor that the delay time will be multiplied 101 | // by on each iteration. 102 | Factor float64 103 | // MaxDelay holds the maximum delay between the start 104 | // of attempts. If this is zero, there is no maximum delay. 105 | MaxDelay time.Duration 106 | } 107 | ``` 108 | 109 | Exponential represents an exponential backoff retry strategy. To limit the 110 | number of attempts or their overall duration, wrap this in LimitCount or 111 | LimitDuration. 112 | 113 | #### func (Exponential) NewTimer 114 | 115 | ```go 116 | func (r Exponential) NewTimer(now time.Time) Timer 117 | ``` 118 | NewTimer implements Strategy.NewTimer. 119 | 120 | #### type Regular 121 | 122 | ```go 123 | type Regular struct { 124 | // Total specifies the total duration of the attempt. 125 | Total time.Duration 126 | 127 | // Delay specifies the interval between the start of each try 128 | // in the burst. If an try takes longer than Delay, the 129 | // next try will happen immediately. 130 | Delay time.Duration 131 | 132 | // Min holds the minimum number of retries. It overrides Total. 133 | // To limit the maximum number of retries, use LimitCount. 134 | Min int 135 | } 136 | ``` 137 | 138 | Regular represents a strategy that repeats at regular intervals. 139 | 140 | #### func (Regular) NewTimer 141 | 142 | ```go 143 | func (r Regular) NewTimer(now time.Time) Timer 144 | ``` 145 | NewTimer implements Strategy.NewTimer. 146 | 147 | #### func (Regular) Start 148 | 149 | ```go 150 | func (r Regular) Start(clk Clock) *Attempt 151 | ``` 152 | Start is short for Start(r, clk, nil) 153 | 154 | #### type Strategy 155 | 156 | ```go 157 | type Strategy interface { 158 | // NewTimer is called when the strategy is started - it is 159 | // called with the time that the strategy is started and returns 160 | // an object that is used to find out how long to sleep before 161 | // each retry attempt. 162 | NewTimer(now time.Time) Timer 163 | } 164 | ``` 165 | 166 | Strategy is implemented by types that represent a retry strategy. 167 | 168 | Note: You probably won't need to implement a new strategy - the existing types 169 | and functions are intended to be sufficient for most purposes. 170 | 171 | #### func LimitCount 172 | 173 | ```go 174 | func LimitCount(n int, strategy Strategy) Strategy 175 | ``` 176 | LimitCount limits the number of attempts that the given strategy will perform to 177 | n. Note that all strategies will allow at least one attempt. 178 | 179 | #### func LimitTime 180 | 181 | ```go 182 | func LimitTime(limit time.Duration, strategy Strategy) Strategy 183 | ``` 184 | LimitTime limits the given strategy such that no attempt will made after the 185 | given duration has elapsed. 186 | 187 | #### type Timer 188 | 189 | ```go 190 | type Timer interface { 191 | // NextSleep is called with the time that Next or More has been 192 | // called and returns the length of time to sleep before the 193 | // next retry. If no more attempts should be made it should 194 | // return false, and the returned duration will be ignored. 195 | // 196 | // Note that NextSleep is called once after each iteration has 197 | // completed, assuming the retry loop is continuing. 198 | NextSleep(now time.Time) (time.Duration, bool) 199 | } 200 | ``` 201 | 202 | Timer represents a source of timing events for a retry strategy. 203 | -------------------------------------------------------------------------------- /clock.go: -------------------------------------------------------------------------------- 1 | package retry // import "gopkg.in/retry.v1" 2 | 3 | import "time" 4 | 5 | // Clock represents a virtual clock interface that 6 | // can be replaced for testing. 7 | type Clock interface { 8 | Now() time.Time 9 | After(time.Duration) <-chan time.Time 10 | } 11 | 12 | // WallClock exposes wall-clock time as returned by time.Now. 13 | type wallClock struct{} 14 | 15 | // Now implements Clock.Now. 16 | func (wallClock) Now() time.Time { 17 | return time.Now() 18 | } 19 | 20 | // After implements Clock.After. 21 | func (wallClock) After(d time.Duration) <-chan time.Time { 22 | return time.After(d) 23 | } 24 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package retry_test // import "gopkg.in/retry.v1" 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "gopkg.in/retry.v1" 8 | ) 9 | 10 | func doSomething() (int, error) { return 0, nil } 11 | 12 | func shouldRetry(error) bool { return false } 13 | 14 | func doSomethingWith(int) {} 15 | 16 | func ExampleAttempt_More() { 17 | // This example shows how Attempt.More can be used to help 18 | // structure an attempt loop. If the godoc example code allowed 19 | // us to make the example return an error, we would uncomment 20 | // the commented return statements. 21 | attempts := retry.Regular{ 22 | Total: 1 * time.Second, 23 | Delay: 250 * time.Millisecond, 24 | } 25 | for attempt := attempts.Start(nil); attempt.Next(); { 26 | x, err := doSomething() 27 | if shouldRetry(err) && attempt.More() { 28 | continue 29 | } 30 | if err != nil { 31 | // return err 32 | return 33 | } 34 | doSomethingWith(x) 35 | } 36 | // return ErrTimedOut 37 | return 38 | } 39 | 40 | func ExampleExponential() { 41 | // This example shows a retry loop that will retry an 42 | // HTTP POST request with an exponential backoff 43 | // for up to 30s. 44 | strategy := retry.LimitTime(30*time.Second, 45 | retry.Exponential{ 46 | Initial: 10 * time.Millisecond, 47 | Factor: 1.5, 48 | }, 49 | ) 50 | for a := retry.Start(strategy, nil); a.Next(); { 51 | if reply, err := http.Post("http://example.com/form", "", nil); err == nil { 52 | reply.Body.Close() 53 | break 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /exp.go: -------------------------------------------------------------------------------- 1 | package retry // import "gopkg.in/retry.v1" 2 | 3 | import ( 4 | "math/rand" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | var ( 10 | // randomMu guards random. 11 | randomMu sync.Mutex 12 | // random is used as a random number source for jitter. 13 | // We avoid using the global math/rand source 14 | // as we don't want to be responsible for seeding it, 15 | // and its lock may be more contended. 16 | random = rand.New(rand.NewSource(time.Now().UnixNano())) 17 | ) 18 | 19 | // Exponential represents an exponential backoff retry strategy. 20 | // To limit the number of attempts or their overall duration, wrap 21 | // this in LimitCount or LimitDuration. 22 | type Exponential struct { 23 | // Initial holds the initial delay. 24 | Initial time.Duration 25 | // Factor holds the factor that the delay time will be multiplied 26 | // by on each iteration. If this is zero, a factor of two will be used. 27 | Factor float64 28 | // MaxDelay holds the maximum delay between the start 29 | // of attempts. If this is zero, there is no maximum delay. 30 | MaxDelay time.Duration 31 | // Jitter specifies whether jitter should be added to the 32 | // retry interval. The algorithm used is described as "Full Jitter" 33 | // in https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/ 34 | Jitter bool 35 | } 36 | 37 | type exponentialTimer struct { 38 | strategy Exponential 39 | start time.Time 40 | end time.Time 41 | delay time.Duration 42 | } 43 | 44 | // NewTimer implements Strategy.NewTimer. 45 | func (r Exponential) NewTimer(now time.Time) Timer { 46 | if r.Factor <= 0 { 47 | r.Factor = 2 48 | } 49 | return &exponentialTimer{ 50 | strategy: r, 51 | start: now, 52 | delay: r.Initial, 53 | } 54 | } 55 | 56 | // NextSleep implements Timer.NextSleep. 57 | func (a *exponentialTimer) NextSleep(now time.Time) (time.Duration, bool) { 58 | sleep := a.delay - now.Sub(a.start) 59 | if sleep <= 0 { 60 | sleep = 0 61 | } 62 | if a.strategy.Jitter { 63 | sleep = randDuration(sleep) 64 | } 65 | // Set the start of the next try. 66 | a.start = now.Add(sleep) 67 | a.delay = time.Duration(float64(a.delay) * a.strategy.Factor) 68 | if a.strategy.MaxDelay > 0 && a.delay > a.strategy.MaxDelay { 69 | a.delay = a.strategy.MaxDelay 70 | } 71 | return sleep, true 72 | } 73 | 74 | func randDuration(max time.Duration) time.Duration { 75 | if max <= 0 { 76 | return 0 77 | } 78 | randomMu.Lock() 79 | defer randomMu.Unlock() 80 | return time.Duration(random.Int63n(int64(max))) 81 | } 82 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module gopkg.in/retry.v1 2 | 3 | require ( 4 | github.com/frankban/quicktest v1.2.2 5 | github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a 6 | ) 7 | 8 | go 1.12 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/frankban/quicktest v1.2.2 h1:xfmOhhoH5fGPgbEAlhLpJH9p0z/0Qizio9osmvn9IUY= 2 | github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= 3 | github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42 h1:q3pnF5JFBNRz8sRD+IRj7Y6DMyYGTNqnZ9axTbSfoNI= 4 | github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 5 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 6 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 7 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 8 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 9 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 10 | github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM= 11 | github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= 12 | -------------------------------------------------------------------------------- /regular.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Canonical Ltd. 2 | // Licensed under the LGPLv3, see LICENCE file for details. 3 | 4 | package retry // import "gopkg.in/retry.v1" 5 | 6 | import ( 7 | "time" 8 | ) 9 | 10 | // Regular represents a strategy that repeats at regular intervals. 11 | type Regular struct { 12 | // Total specifies the total duration of the attempt. 13 | Total time.Duration 14 | 15 | // Delay specifies the interval between the start of each try 16 | // in the burst. If an try takes longer than Delay, the 17 | // next try will happen immediately. 18 | Delay time.Duration 19 | 20 | // Min holds the minimum number of retries. It overrides Total. 21 | // To limit the maximum number of retries, use LimitCount. 22 | Min int 23 | } 24 | 25 | // regularTimer holds a running instantiation of the Regular timer. 26 | type regularTimer struct { 27 | strategy Regular 28 | count int 29 | // start holds when the current try started. 30 | start time.Time 31 | end time.Time 32 | } 33 | 34 | // Start is short for Start(r, clk, nil) 35 | func (r Regular) Start(clk Clock) *Attempt { 36 | return Start(r, clk) 37 | } 38 | 39 | // NewTimer implements Strategy.NewTimer. 40 | func (r Regular) NewTimer(now time.Time) Timer { 41 | return ®ularTimer{ 42 | strategy: r, 43 | start: now, 44 | end: now.Add(r.Total), 45 | } 46 | } 47 | 48 | // NextSleep implements Timer.NextSleep. 49 | func (a *regularTimer) NextSleep(now time.Time) (time.Duration, bool) { 50 | sleep := a.strategy.Delay - now.Sub(a.start) 51 | if sleep <= 0 { 52 | sleep = 0 53 | } 54 | a.count++ 55 | // Set the start of the next try. 56 | a.start = now.Add(sleep) 57 | if a.count < a.strategy.Min { 58 | return sleep, true 59 | } 60 | // The next try is not before the end - no more attempts. 61 | if !a.start.Before(a.end) { 62 | return 0, false 63 | } 64 | return sleep, true 65 | } 66 | -------------------------------------------------------------------------------- /retry.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Canonical Ltd. 2 | // Licensed under the LGPLv3, see LICENCE file for details. 3 | 4 | // Package retry implements flexible retry loops, including support for 5 | // channel cancellation, mocked time, and composable retry strategies 6 | // including exponential backoff with jitter. 7 | // 8 | // The basic usage is as follows: 9 | // 10 | // for a := retry.Start(someStrategy, nil); a.Next(); { 11 | // try() 12 | // } 13 | // 14 | // See examples for details of suggested usage. 15 | package retry // import "gopkg.in/retry.v1" 16 | 17 | import ( 18 | "time" 19 | ) 20 | 21 | // Strategy is implemented by types that represent a retry strategy. 22 | // 23 | // Note: You probably won't need to implement a new strategy - the existing types 24 | // and functions are intended to be sufficient for most purposes. 25 | type Strategy interface { 26 | // NewTimer is called when the strategy is started - it is 27 | // called with the time that the strategy is started and returns 28 | // an object that is used to find out how long to sleep before 29 | // each retry attempt. 30 | NewTimer(now time.Time) Timer 31 | } 32 | 33 | // Timer represents a source of timing events for a retry strategy. 34 | type Timer interface { 35 | // NextSleep is called with the time that Next or More has been 36 | // called and returns the length of time to sleep before the 37 | // next retry. If no more attempts should be made it should 38 | // return false, and the returned duration will be ignored. 39 | // 40 | // Note that NextSleep is called once after each iteration has 41 | // completed, assuming the retry loop is continuing. 42 | NextSleep(now time.Time) (time.Duration, bool) 43 | } 44 | 45 | // Attempt represents a running retry attempt. 46 | type Attempt struct { 47 | clock Clock 48 | stop <-chan struct{} 49 | timer Timer 50 | 51 | // next holds when the next attempt should start. 52 | // It is valid only when known is true. 53 | next time.Time 54 | 55 | // count holds the iteration count. 56 | count int 57 | 58 | // known holds whether next and running are known. 59 | known bool 60 | 61 | // running holds whether the attempt is still going. 62 | running bool 63 | 64 | // stopped holds whether the attempt has been stopped. 65 | stopped bool 66 | } 67 | 68 | // Start begins a new sequence of attempts for the given strategy using 69 | // the given Clock implementation for time keeping. If clk is 70 | // nil, the time package will be used to keep time. 71 | func Start(strategy Strategy, clk Clock) *Attempt { 72 | return StartWithCancel(strategy, clk, nil) 73 | } 74 | 75 | // StartWithCancel is like Start except that if a value 76 | // is received on stop while waiting, the attempt will be aborted. 77 | func StartWithCancel(strategy Strategy, clk Clock, stop <-chan struct{}) *Attempt { 78 | if clk == nil { 79 | clk = wallClock{} 80 | } 81 | now := clk.Now() 82 | return &Attempt{ 83 | clock: clk, 84 | stop: stop, 85 | timer: strategy.NewTimer(now), 86 | known: true, 87 | running: true, 88 | next: now, 89 | } 90 | } 91 | 92 | // Next reports whether another attempt should be made, waiting as 93 | // necessary until it's time for the attempt. It always returns true the 94 | // first time it is called unless a value is received on the stop 95 | // channel - we are guaranteed to make at least one attempt unless 96 | // stopped. 97 | func (a *Attempt) Next() bool { 98 | if !a.More() { 99 | return false 100 | } 101 | sleep := a.next.Sub(a.clock.Now()) 102 | if sleep <= 0 { 103 | // We're not going to sleep for any length of time, 104 | // so guarantee that we respect the stop channel. This 105 | // ensures that we make no attempts if Next is called 106 | // with a value available on the stop channel. 107 | select { 108 | case <-a.stop: 109 | a.stopped = true 110 | a.running = false 111 | return false 112 | default: 113 | a.known = false 114 | a.count++ 115 | return true 116 | } 117 | } 118 | select { 119 | case <-a.clock.After(sleep): 120 | a.known = false 121 | a.count++ 122 | case <-a.stop: 123 | a.running = false 124 | a.stopped = true 125 | } 126 | return a.running 127 | } 128 | 129 | // More reports whether there are more retry attempts to be made. It 130 | // does not sleep. 131 | // 132 | // If More returns false, Next will return false. If More returns true, 133 | // Next will return true except when the attempt has been explicitly 134 | // stopped via the stop channel. 135 | func (a *Attempt) More() bool { 136 | if !a.known { 137 | now := a.clock.Now() 138 | sleepDuration, running := a.timer.NextSleep(now) 139 | a.next, a.running, a.known = now.Add(sleepDuration), running, true 140 | } 141 | return a.running 142 | } 143 | 144 | // Stopped reports whether the attempt has terminated because 145 | // a value was received on the stop channel. 146 | func (a *Attempt) Stopped() bool { 147 | return a.stopped 148 | } 149 | 150 | // Count returns the current attempt count number, starting at 1. 151 | // It returns 0 if called before Next is called. 152 | // When the loop has terminated, it holds the total number 153 | // of retries made. 154 | func (a *Attempt) Count() int { 155 | return a.count 156 | } 157 | -------------------------------------------------------------------------------- /retry_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011, 2012, 2013 Canonical Ltd. 2 | // Licensed under the LGPLv3, see LICENCE file for details. 3 | 4 | package retry_test // import "gopkg.in/retry.v1" 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/rogpeppe/clock/testclock" 11 | qt "github.com/frankban/quicktest" 12 | 13 | "gopkg.in/retry.v1" 14 | ) 15 | 16 | func TestAttemptTiming(t *testing.T) { 17 | c := qt.New(t) 18 | testAttempt := retry.Regular{ 19 | Total: 0.25e9, 20 | Delay: 0.1e9, 21 | } 22 | want := []time.Duration{0, 0.1e9, 0.2e9, 0.2e9} 23 | got := make([]time.Duration, 0, len(want)) // avoid allocation when testing timing 24 | t0 := time.Now() 25 | a := testAttempt.Start(nil) 26 | for a.Next() { 27 | got = append(got, time.Now().Sub(t0)) 28 | } 29 | got = append(got, time.Now().Sub(t0)) 30 | c.Assert(a.Stopped(), qt.Equals, false) 31 | c.Assert(got, qt.HasLen, len(want)) 32 | const margin = 0.01e9 33 | for i, got := range want { 34 | lo := want[i] - margin 35 | hi := want[i] + margin 36 | if got < lo || got > hi { 37 | c.Errorf("attempt %d want %g got %g", i, want[i].Seconds(), got.Seconds()) 38 | } 39 | } 40 | } 41 | 42 | func TestAttemptNextMore(t *testing.T) { 43 | c := qt.New(t) 44 | a := retry.Regular{}.Start(nil) 45 | c.Assert(a.Next(), qt.Equals, true) 46 | c.Assert(a.Next(), qt.Equals, false) 47 | 48 | a = retry.Regular{}.Start(nil) 49 | c.Assert(a.Next(), qt.Equals, true) 50 | c.Assert(a.More(), qt.Equals, false) 51 | c.Assert(a.Next(), qt.Equals, false) 52 | 53 | a = retry.Regular{Total: 2e8}.Start(nil) 54 | c.Assert(a.Next(), qt.Equals, true) 55 | c.Assert(a.More(), qt.Equals, true) 56 | time.Sleep(2e8) 57 | c.Assert(a.More(), qt.Equals, true) 58 | c.Assert(a.Next(), qt.Equals, true) 59 | c.Assert(a.Next(), qt.Equals, false) 60 | 61 | a = retry.Regular{Total: 1e8, Min: 2}.Start(nil) 62 | time.Sleep(1e8) 63 | c.Assert(a.Next(), qt.Equals, true) 64 | c.Assert(a.More(), qt.Equals, true) 65 | c.Assert(a.Next(), qt.Equals, true) 66 | c.Assert(a.More(), qt.Equals, false) 67 | c.Assert(a.Next(), qt.Equals, false) 68 | } 69 | 70 | func TestAttemptWithStop(t *testing.T) { 71 | c := qt.New(t) 72 | stop := make(chan struct{}) 73 | close(stop) 74 | done := make(chan struct{}) 75 | go func() { 76 | strategy := retry.Regular{ 77 | Delay: 5 * time.Second, 78 | Total: 30 * time.Second, 79 | } 80 | a := retry.StartWithCancel(strategy, nil, stop) 81 | for a.Next() { 82 | c.Errorf("unexpected attempt") 83 | } 84 | c.Check(a.Stopped(), qt.Equals, true) 85 | close(done) 86 | }() 87 | assertReceive(c, done, "attempt loop abort") 88 | } 89 | 90 | func TestAttemptWithLaterStop(t *testing.T) { 91 | c := qt.New(t) 92 | clock := testclock.NewClock(time.Now()) 93 | stop := make(chan struct{}) 94 | done := make(chan struct{}) 95 | progress := make(chan struct{}, 10) 96 | go func() { 97 | strategy := retry.Regular{ 98 | Delay: 5 * time.Second, 99 | Total: 30 * time.Second, 100 | } 101 | a := retry.StartWithCancel(strategy, clock, stop) 102 | for a.Next() { 103 | progress <- struct{}{} 104 | } 105 | c.Check(a.Stopped(), qt.Equals, true) 106 | close(done) 107 | }() 108 | assertReceive(c, progress, "progress") 109 | clock.Advance(5 * time.Second) 110 | assertReceive(c, progress, "progress") 111 | clock.Advance(2 * time.Second) 112 | close(stop) 113 | assertReceive(c, done, "attempt loop abort") 114 | select { 115 | case <-progress: 116 | c.Fatalf("unxpected loop iteration after stop") 117 | default: 118 | } 119 | } 120 | 121 | func TestAttemptWithMockClock(t *testing.T) { 122 | c := qt.New(t) 123 | clock := testclock.NewClock(time.Now()) 124 | strategy := retry.Regular{ 125 | Delay: 5 * time.Second, 126 | Total: 30 * time.Second, 127 | } 128 | progress := make(chan struct{}) 129 | done := make(chan struct{}) 130 | go func() { 131 | for a := strategy.Start(clock); a.Next(); { 132 | progress <- struct{}{} 133 | } 134 | close(done) 135 | }() 136 | assertReceive(c, progress, "progress first time") 137 | clock.Advance(5 * time.Second) 138 | assertReceive(c, progress, "progress second time") 139 | clock.Advance(5 * time.Second) 140 | assertReceive(c, progress, "progress third time") 141 | clock.Advance(30 * time.Second) 142 | assertReceive(c, progress, "progress fourth time") 143 | assertReceive(c, done, "loop finish") 144 | } 145 | 146 | type strategyTest struct { 147 | about string 148 | strategy retry.Strategy 149 | calls []nextCall 150 | terminates bool 151 | } 152 | 153 | type nextCall struct { 154 | // t holds the time since the timer was started that 155 | // the Next call will be made. 156 | t time.Duration 157 | // delay holds the length of time that a call made at 158 | // time t is expected to sleep for. 159 | sleep time.Duration 160 | } 161 | 162 | var strategyTests = []strategyTest{{ 163 | about: "regular retry (same params as TestAttemptTiming)", 164 | strategy: retry.Regular{ 165 | Total: 0.25e9, 166 | Delay: 0.1e9, 167 | }, 168 | calls: []nextCall{ 169 | {0, 0}, 170 | {0, 0.1e9}, 171 | {0.1e9, 0.1e9}, 172 | {0.2e9, 0}, 173 | }, 174 | terminates: true, 175 | }, { 176 | about: "regular retry with calls at different times", 177 | strategy: retry.Regular{ 178 | Total: 2.5e9, 179 | Delay: 1e9, 180 | }, 181 | calls: []nextCall{ 182 | {0.5e9, 0}, 183 | {0.5e9, 0.5e9}, 184 | {1.1e9, 0.9e9}, 185 | {2.2e9, 0}, 186 | }, 187 | terminates: true, 188 | }, { 189 | about: "regular retry with call after next deadline", 190 | strategy: retry.Regular{ 191 | Total: 3.5e9, 192 | Delay: 1e9, 193 | }, 194 | calls: []nextCall{ 195 | {0.5e9, 0}, 196 | // We call Next at well beyond the deadline, 197 | // so we get a zero delay, but subsequent events 198 | // resume pace. 199 | {2e9, 0}, 200 | {2.1e9, 0.9e9}, 201 | {3e9, 0}, 202 | }, 203 | terminates: true, 204 | }, { 205 | about: "exponential retry", 206 | strategy: retry.Exponential{ 207 | Initial: 1e9, 208 | Factor: 2, 209 | }, 210 | calls: []nextCall{ 211 | {0, 0}, 212 | {0.1e9, 0.9e9}, 213 | {1e9, 2e9}, 214 | {3e9, 4e9}, 215 | {7e9, 8e9}, 216 | }, 217 | }, { 218 | about: "exponential retry with zero factor defaults to 2", 219 | strategy: retry.Exponential{ 220 | Initial: 1e9, 221 | }, 222 | calls: []nextCall{ 223 | {0, 0}, 224 | {0.1e9, 0.9e9}, 225 | {1e9, 2e9}, 226 | {3e9, 4e9}, 227 | {7e9, 8e9}, 228 | }, 229 | }, { 230 | about: "time-limited exponential retry", 231 | strategy: retry.LimitTime(5e9, retry.Exponential{ 232 | Initial: 1e9, 233 | Factor: 2, 234 | }), 235 | calls: []nextCall{ 236 | {0, 0}, 237 | {0.1e9, 0.9e9}, 238 | {1e9, 2e9}, 239 | {3e9, 0}, 240 | }, 241 | terminates: true, 242 | }, { 243 | about: "count-limited exponential retry", 244 | strategy: retry.LimitCount(2, retry.Exponential{ 245 | Initial: 1e9, 246 | Factor: 2, 247 | }), 248 | calls: []nextCall{ 249 | {0, 0}, 250 | {0.1e9, 0.9e9}, 251 | {1e9, 0}, 252 | }, 253 | terminates: true, 254 | }} 255 | 256 | func TestStrategies(t *testing.T) { 257 | c := qt.New(t) 258 | for i, test := range strategyTests { 259 | c.Logf("test %d: %s", i, test.about) 260 | testStrategy(c, test) 261 | } 262 | } 263 | 264 | func testStrategy(c *qt.C, test strategyTest) { 265 | t0 := time.Now() 266 | clk := &mockClock{ 267 | now: t0, 268 | } 269 | a := retry.Start(test.strategy, clk) 270 | for i, call := range test.calls { 271 | c.Logf("call %d - %v", i, call.t) 272 | clk.now = t0.Add(call.t) 273 | ok := a.Next() 274 | expectTerminate := test.terminates && i == len(test.calls)-1 275 | c.Assert(ok, qt.Equals, !expectTerminate) 276 | if got, want := clk.now.Sub(t0), call.t+call.sleep; !closeTo(got, want) { 277 | c.Fatalf("incorrect time after Next; got %v want %v", got, want) 278 | } 279 | if ok { 280 | c.Assert(a.Count(), qt.Equals, i+1) 281 | } 282 | } 283 | } 284 | 285 | func TestExponentialWithJitter(t *testing.T) { 286 | c := qt.New(t) 287 | // We use a stochastic test because we don't want 288 | // to mock rand and have detailed dependence on 289 | // the exact way it's used. We run the strategy many 290 | // times and note the delays that we found; if the 291 | // jitter is working, the delays should be roughly equally 292 | // distributed and it shouldn't take long before all the 293 | // buckets are hit. 294 | const numBuckets = 8 295 | tries := []struct { 296 | max time.Duration 297 | buckets [numBuckets]int 298 | }{{ 299 | max: 1e9, 300 | }, { 301 | max: 2e9, 302 | }, { 303 | max: 4e9, 304 | }, { 305 | max: 5e9, 306 | }} 307 | strategy := retry.Exponential{ 308 | Initial: 1e9, 309 | Factor: 2, 310 | MaxDelay: 5e9, 311 | Jitter: true, 312 | } 313 | count := 0 314 | for i := 0; i < 10000 && count < len(tries)*numBuckets; i++ { 315 | clk := &mockClock{ 316 | now: time.Now(), 317 | } 318 | t := clk.Now() 319 | a := retry.Start(strategy, clk) 320 | if !a.Next() { 321 | c.Fatalf("no first try") 322 | } 323 | if clk.Now().Sub(t) != 0 { 324 | c.Fatalf("first try was not immediate") 325 | } 326 | for try := 0; a.Next(); try++ { 327 | if try >= len(tries) { 328 | break 329 | } 330 | d := clk.Now().Sub(t) 331 | t = clk.Now() 332 | max := tries[try].max 333 | if d > max { 334 | c.Fatalf("try %d exceeded max %v; actual duration %v", try, tries[try].max, d) 335 | } 336 | slot := int(float64(d) / float64(max+1) * numBuckets) 337 | if slot >= numBuckets || slot < 0 { 338 | c.Fatalf("try %d slot %d out of range; d %v; max %v", try, slot, d, max) 339 | } 340 | buckets := &tries[try].buckets 341 | if buckets[slot] == 0 { 342 | count++ 343 | } 344 | buckets[slot]++ 345 | } 346 | } 347 | if count < len(tries)*numBuckets { 348 | c.Fatalf("distribution was not evenly spread; tries %#v", tries) 349 | } 350 | } 351 | 352 | func TestGapBetweenMoreAndNext(t *testing.T) { 353 | c := qt.New(t) 354 | t0 := time.Now().UTC() 355 | clk := &mockClock{ 356 | now: t0, 357 | } 358 | a := (&retry.Regular{ 359 | Min: 3, 360 | Delay: time.Second, 361 | }).Start(clk) 362 | c.Assert(a.Next(), qt.Equals, true) 363 | c.Assert(clk.now, qt.Equals, t0) 364 | 365 | clk.now = clk.now.Add(500 * time.Millisecond) 366 | // Sanity check that the first iteration sleeps for half a second. 367 | c.Assert(a.More(), qt.Equals, true) 368 | c.Assert(a.Next(), qt.Equals, true) 369 | c.Assert(clk.now.Sub(t0), qt.Equals, t0.Add(time.Second).Sub(t0)) 370 | 371 | clk.now = clk.now.Add(500 * time.Millisecond) 372 | c.Assert(a.More(), qt.Equals, true) 373 | 374 | // Add a delay between calling More and Next. 375 | // Next should wait until the correct time anyway. 376 | clk.now = clk.now.Add(250 * time.Millisecond) 377 | c.Assert(a.More(), qt.Equals, true) 378 | c.Assert(a.Next(), qt.Equals, true) 379 | c.Assert(clk.now.Sub(t0), qt.Equals, t0.Add(2*time.Second).Sub(t0)) 380 | } 381 | 382 | func TestOnlyOneHitOnZeroTotal(t *testing.T) { 383 | c := qt.New(t) 384 | t0 := time.Now().UTC() 385 | clk := &mockClock{ 386 | now: t0, 387 | } 388 | a := (&retry.Regular{ 389 | Total: 0, 390 | Delay: 0, 391 | Min: 0, 392 | }).Start(clk) 393 | // Even if the clock didn't advanced we want to have only one hit 394 | c.Check(a.Next(), qt.Equals, true) 395 | c.Check(a.More(), qt.Equals, false) 396 | } 397 | 398 | // closeTo reports whether d0 and d1 are close enough 399 | // to one another to cater for inaccuracies of floating point arithmetic. 400 | func closeTo(d0, d1 time.Duration) bool { 401 | const margin = 20 * time.Nanosecond 402 | diff := d1 - d0 403 | if diff < 0 { 404 | diff = -diff 405 | } 406 | return diff < margin 407 | } 408 | 409 | type mockClock struct { 410 | retry.Clock 411 | 412 | now time.Time 413 | sleep func(d time.Duration) 414 | } 415 | 416 | func (c *mockClock) After(d time.Duration) <-chan time.Time { 417 | c.now = c.now.Add(d) 418 | ch := make(chan time.Time) 419 | close(ch) 420 | return ch 421 | } 422 | 423 | func (c *mockClock) Now() time.Time { 424 | return c.now 425 | } 426 | 427 | func assertReceive(c *qt.C, ch <-chan struct{}, what string) { 428 | select { 429 | case <-ch: 430 | case <-time.After(time.Second): 431 | c.Fatalf("timed out waiting for %s", what) 432 | } 433 | } 434 | -------------------------------------------------------------------------------- /strategy.go: -------------------------------------------------------------------------------- 1 | package retry // import "gopkg.in/retry.v1" 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type strategyFunc func(now time.Time) Timer 8 | 9 | // NewTimer implements Strategy.NewTimer. 10 | func (f strategyFunc) NewTimer(now time.Time) Timer { 11 | return f(now) 12 | } 13 | 14 | // LimitCount limits the number of attempts that the given 15 | // strategy will perform to n. Note that all strategies 16 | // will allow at least one attempt. 17 | func LimitCount(n int, strategy Strategy) Strategy { 18 | return strategyFunc(func(now time.Time) Timer { 19 | return &countLimitTimer{ 20 | timer: strategy.NewTimer(now), 21 | remain: n, 22 | } 23 | }) 24 | } 25 | 26 | type countLimitTimer struct { 27 | timer Timer 28 | remain int 29 | } 30 | 31 | func (t *countLimitTimer) NextSleep(now time.Time) (time.Duration, bool) { 32 | if t.remain--; t.remain <= 0 { 33 | return 0, false 34 | } 35 | return t.timer.NextSleep(now) 36 | } 37 | 38 | // LimitTime limits the given strategy such that no attempt will 39 | // made after the given duration has elapsed. 40 | func LimitTime(limit time.Duration, strategy Strategy) Strategy { 41 | return strategyFunc(func(now time.Time) Timer { 42 | return &timeLimitTimer{ 43 | timer: strategy.NewTimer(now), 44 | end: now.Add(limit), 45 | } 46 | }) 47 | } 48 | 49 | type timeLimitTimer struct { 50 | timer Timer 51 | end time.Time 52 | } 53 | 54 | func (t *timeLimitTimer) NextSleep(now time.Time) (time.Duration, bool) { 55 | sleep, ok := t.timer.NextSleep(now) 56 | if ok && now.Add(sleep).After(t.end) { 57 | return 0, false 58 | } 59 | return sleep, ok 60 | } 61 | --------------------------------------------------------------------------------