├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build.boot ├── examples ├── build.boot └── src │ └── example.clj ├── src ├── clj │ └── manifail.clj └── java │ └── manifail │ ├── Aborted.java │ ├── Reset.java │ ├── Retried.java │ └── RetriesExceeded.java └── test └── manifail_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | .nrepl-history 10 | .nrepl-port 11 | .hgignore 12 | .hg/ 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: java 3 | script: boot test 4 | install: 5 | - mkdir -p ~/bin 6 | - export PATH=~/bin:$PATH 7 | - curl -fsSLo ~/bin/boot https://github.com/boot-clj/boot-bin/releases/download/latest/boot.sh 8 | - chmod +x ~/bin/boot 9 | env: 10 | matrix: 11 | - BOOT_CLOJURE_VERSION=1.7.0 12 | - BOOT_CLOJURE_VERSION=1.8.0 13 | jdk: 14 | - oraclejdk8 15 | cache: 16 | directories: 17 | - $HOME/.m2 18 | - $HOME/.boot/cache/bin 19 | - $HOME/.boot/cache/lib 20 | - $HOME/bin 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [Unreleased] 4 | 5 | ## 0.4.0 - 2016-08-31 6 | ### Added 7 | - `*last-result*/*retry-count*/*elapsed-ms*` dynamic variables available inside 8 | the retry block 9 | - `unwrap` to get the value/cause of the marker 10 | ### Changed 11 | - `retry!/abort!` accept an object as an argument in addition to a `Throwable` cause 12 | - `retry!/abort!` markers produce `Throwable` exceptions instead of `ExceptionInfo` 13 | - All special exceptions subclass `Throwable` instead of `RuntimeException` 14 | 15 | ## 0.3.0 - 2016-08-30 16 | ### Added 17 | - `reset!` to reset the execution with a new delays sequence 18 | 19 | ## 0.2.0 - 2016-08-16 20 | ### Changed 21 | - `abort!` and `retry!` can take a `Throwable` cause as their argument 22 | 23 | ## 0.1.0 - 2016-08-15 24 | ### Added 25 | - Initial release: `with-retries`, `forever`, `retries`, `delay`, 26 | `limit-duration`, `limit-retries` 27 | 28 | [Unreleased]: https://github.com/your-name/manifail/compare/0.4.0...HEAD 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # manifail 2 | 3 | [![Build Status](https://travis-ci.org/tulos/manifail.png?branch=master)](https://travis-ci.org/tulos/manifail) 4 | 5 | Handle failures and manage retries without callbacks! 6 | 7 | The semantics are modeled closely to 8 | [Failsafe](https://github.com/jhalterman/failsafe), which has a Clojure wrapper 9 | called [Diehard](https://github.com/sunng87/diehard). 10 | 11 | Manifail differs in that it tries to be composable and not rely on callbacks 12 | for the failure handling logic. The drawback is that it's less precise and 13 | carries more overhead than Failsafe. Also, circuit breakers are currently not 14 | supported. 15 | 16 | ## Disclaimer 17 | 18 | Use Manifail if you're feeling adventurous and/or like to write your code in a 19 | direct callback-less style! Otherwise please consider using Failsafe directly 20 | or through Diehard. 21 | 22 | ## Usage 23 | 24 | `[tulos/manifail "0.4.0"]` 25 | 26 | First, require the namespaces: 27 | 28 | ```clojure 29 | (refer-clojure :exclude '(delay)) 30 | (require '[manifold.deferred :as d]) 31 | (require '[manifold.executor :as ex]) 32 | (use 'manifail) 33 | ``` 34 | 35 | ### Retry building blocks 36 | 37 | To create a retriable execution with Manifail you have to use four parts 38 | working in concert: 39 | 40 | 1. A retry policy - sequence of milliseconds representing the delay of retries 41 | 2. A piece of code potentially having the `retry`/`abort`/`reset` markers 42 | 3. An executor to run the code and retry attempts on 43 | 4. `with-retry` macro or `with-retry*` function wrapping the code to be retried 44 | 45 | It looks like this: 46 | 47 | ```clojure 48 | (def unreliable-service-executor (Executors/newFixedThreadPool 1)) 49 | 50 | (ex/with-executor unreliable-service-executor 51 | (let [retry-delays [10 50 100]] 52 | (with-retries retry-delays 53 | (try (let [result (unreliable-service)] 54 | (when (:error result) 55 | (retry! result))) 56 | result) 57 | (catch UnrecoverableException e 58 | (abort! e)) 59 | (catch SessionExpiredException e 60 | (authenticate!) 61 | (reset! retry-delays))) 62 | ``` 63 | 64 | The `with-retries` block above will return a deferred of its result. This 65 | deferred will get fulfilled when either the original call or one of the three 66 | retries completes. The execution will happen on the specified 67 | `unreliable-service-executor`. 68 | 69 | Given the above code, a retry will happen if: 70 | 71 | * An exception is thrown which is not an `UnrecoverableException` 72 | * The `result` has an `:error` key 73 | 74 | and the resulting deferred will be completed when: 75 | 76 | * A non `:error` response is returned 77 | * An `UnrecoverableException` happens and the execution is aborted. In this 78 | case a `manifail.Aborted` exception is thrown with the cause set to `e` 79 | * A `manifail.RetriesExceeded` exception is thrown when there are no more retry 80 | attempts. 81 | 82 | In case a `SessionExpiredException` is thrown the whole execution cycle will 83 | begin anew with the supplied sequence of `retry-delays`. In the example above 84 | it's the same sequence as the one provided to `with-retries` originally. 85 | 86 | ### Retry policies 87 | 88 | The simplest possible retry logic that will perform 3 retries with delays of 89 | 50, 100 and 150 ms: 90 | 91 | ```clojure 92 | (with-retries [50 100 150] 93 | (unreliable-service)) 94 | ``` 95 | 96 | The following code will retry the call 5 times with 50 ms delay: 97 | 98 | ```clojure 99 | (with-retries (delay (retries 5) 50) 100 | (unreliable-service)) 101 | ``` 102 | 103 | same, but with a backoff factor of 2.0: 104 | 105 | ```clojure 106 | (with-retries (delay (retries 5) 50 {:backoff-factor 2.0}) 107 | (unreliable-service)) 108 | ``` 109 | 110 | with a random jitter of 25 ms: 111 | 112 | ```clojure 113 | (with-retries (delay (retries 5) 50 {:backoff-factor 2.0, :jitter-ms 25}) 114 | (unreliable-service)) 115 | ``` 116 | 117 | with a random jitter scaling the delay up to 1/2: 118 | 119 | ```clojure 120 | (with-retries (delay (retries 5) 50 {:backoff-factor 2.0, :jitter-factor 0.5}) 121 | (unreliable-service)) 122 | ``` 123 | 124 | with a total call and retry duration limited to 500 ms: 125 | 126 | ```clojure 127 | (with-retries (-> (retries 5) 128 | (delay 50 {:backoff-factor 2.0, :jitter-factor 0.5}) 129 | (limit-duration 500)) 130 | (unreliable-service)) 131 | ``` 132 | 133 | unlimited equally spaced out retries with duration limited to 500 ms: 134 | 135 | ```clojure 136 | (with-retries (-> (forever) (delay 50) (limit-duration 500)) 137 | (unreliable-service)) 138 | ``` 139 | 140 | ### Retry context 141 | 142 | There are several dynamic bindings available in the retry block: 143 | 144 | * `*retry-count*` - the current retry count 145 | * `*elapsed-ms*` - milliseconds elapsed since entering the retry block 146 | * `*last-result*` - last result/exception that caused the retry 147 | 148 | ### A complete example 149 | 150 | Below I tried to map callbacks available in Failsafe retry policy to the code 151 | that you could write in order to get equivalent behaviour. We assume that 152 | `call-some-service` blocks until completed: 153 | 154 | ```clojure 155 | (-> 156 | (with-retries (delay (retries 5) 50) ;; retry on any exception by default 157 | (println "-- on before execution") 158 | (when (> *retry-count* 0) 159 | (println "-- on before retry")) 160 | (try (let [result (call-some-service)] 161 | (when (> result 5) ;; abort condition 162 | (println "-- on failed attempt 1") 163 | (println "-- on before abort") 164 | (abort! result)) ;; set the value of Abort to `result` 165 | (when (= result ::bad) ;; retry condition 166 | (println "-- on failed attempt 2") 167 | (retry!)) 168 | result) 169 | (catch OkException _ :ok) ;; do not retry on this exception 170 | (catch UnrecoverableException e 171 | (abort! e)) ;; set the cause of Abort to `e` 172 | (catch RecoverableException _ 173 | (do-some-recovery) 174 | (reset! (delay (retries 5) 50))) 175 | (catch Throwable e 176 | (when-not (marker? e) 177 | (println "-- on failed attempt any")) 178 | (throw e)))) 179 | (d/chain #(println "-- on complete" %)) 180 | (d/catch manifail.Aborted #(println "-- on after abort" (unwrap %)) 181 | (d/catch manifail.RetriesExceeded #(println "-- on retries exceeded" (unwrap %)) 182 | (d/catch #(println "-- on failure" %))) 183 | ``` 184 | 185 | `call-some-service` might also be asynchronous and return something derefable. 186 | In this case the code within `with-retries` can `chain` the logic: 187 | 188 | ```clojure 189 | (with-retries (delay (retries 5) 50) 190 | (-> (call-some-service) 191 | (d/chain (fn [result] ...)))) 192 | ``` 193 | 194 | ## License 195 | 196 | Copyright © 2016 Tulos Capital 197 | 198 | Distributed under the Eclipse Public License either version 1.0 or (at 199 | your option) any later version. 200 | -------------------------------------------------------------------------------- /build.boot: -------------------------------------------------------------------------------- 1 | (def project 'tulos/manifail) 2 | (def version "0.4.0") 3 | 4 | (set-env! :resource-paths #{"src/clj"} 5 | :source-paths #{"test" "src/java"} 6 | :dependencies '[[org.clojure/clojure "1.8.0"] 7 | [manifold "0.1.5"] 8 | [adzerk/bootlaces "0.1.13" :scope "test"] 9 | [adzerk/boot-test "RELEASE" :scope "test"]]) 10 | 11 | (task-options! 12 | pom {:project project 13 | :version version 14 | :description "Failure handling with Manifold" 15 | :url "https://github.com/tulos/manifail" 16 | :scm {:url "https://github.com/tulos/manifail"} 17 | :license {"Eclipse Public License" 18 | "http://www.eclipse.org/legal/epl-v10.html"}}) 19 | 20 | (require '[adzerk.bootlaces :as l :refer [push-release]] 21 | '[adzerk.boot-test :as t]) 22 | (l/bootlaces! version :dont-modify-paths? true) 23 | 24 | (deftask build-jar [] 25 | (comp (javac) (l/build-jar))) 26 | 27 | (deftask release [] 28 | (comp (build-jar) (push-release))) 29 | 30 | (deftask test [] 31 | (comp (javac) (t/test))) 32 | 33 | (set! *warn-on-reflection* true) 34 | -------------------------------------------------------------------------------- /examples/build.boot: -------------------------------------------------------------------------------- 1 | (def project 'tulos/manifail-examples) 2 | 3 | (set-env! :source-paths #{"src"} 4 | :dependencies '[[org.clojure/clojure "1.8.0"] 5 | [tulos/manifail "0.4.0"] 6 | [clojure.java-time "0.2.1"]]) 7 | 8 | (require '[manifail :as f] 9 | '[example :as e]) 10 | -------------------------------------------------------------------------------- /examples/src/example.clj: -------------------------------------------------------------------------------- 1 | (ns example 2 | (:require [java-time :as j] 3 | [manifold.deferred :as d] 4 | [manifail :as f])) 5 | 6 | (defn failing-service [] 7 | (let [r (rand-int 20)] 8 | (cond 9 | (> r 18) {:status :abort, :result r} 10 | (> r 12) {:status :failure, :result r} 11 | (> r 8) (throw (Exception. "Failed!")) 12 | (> r 3) {:status :unauthorized} 13 | (> r 2) (throw (Exception. "Fine!")) 14 | :else {:status :ok, :result r}))) 15 | 16 | (defn authorize [] 17 | (println "Authorized!")) 18 | 19 | (defn call-with-retries [] 20 | (let [retry-delays #(f/delay (f/retries 5) 50 {:jitter-factor 0.5})] 21 | (-> (f/with-retries (retry-delays) 22 | (println "Execution n." (inc f/*retry-count*) "after" f/*elapsed-ms* "ms...") 23 | (when f/*last-result* 24 | (println "Last result:" f/*last-result*)) 25 | (try (let [{:keys [status result]} (failing-service)] 26 | (println "Got status=" status ", result=" result) 27 | (when (= status :abort) 28 | (println "-- on before abort") 29 | (f/abort! result)) 30 | (when (= status :failure) 31 | (println "-- on before retry 1") 32 | (f/retry! result)) 33 | (when (= status :unauthorized) 34 | (println "-- on recoverable error") 35 | (authorize) 36 | (println "-- on before reset") 37 | (f/reset! (retry-delays))) 38 | result) 39 | (catch Exception e 40 | (if (= (.getMessage e) "Fine!") 41 | :ok 42 | (throw e))) 43 | (catch Throwable t 44 | (when-not (f/marker? t) 45 | (println "-- on before retry 2")) 46 | (throw t)))) 47 | (d/chain #(println "-- on complete" %)) 48 | (d/catch manifail.Aborted #(println "-- on after abort" (f/unwrap %))) 49 | (d/catch manifail.RetriesExceeded #(println "-- on retries exceeded" (f/unwrap %))) 50 | (d/catch #(println "-- on failure" %))))) 51 | -------------------------------------------------------------------------------- /src/clj/manifail.clj: -------------------------------------------------------------------------------- 1 | (ns manifail 2 | (:refer-clojure :exclude [delay reset!]) 3 | (:require [manifold 4 | [deferred :as d] 5 | [executor :as ex]]) 6 | (:import [clojure.lang ExceptionInfo] 7 | [manifail Aborted Retried RetriesExceeded Reset])) 8 | 9 | (defn forever 10 | "A sequence of infinite retries with zero delay." 11 | [] (repeat 0)) 12 | 13 | (defn retries 14 | "Creates a sequence of `n` retries with zero delay." 15 | [n] (take n (forever))) 16 | 17 | (defn- verify-factor [v min max name] 18 | (assert (and v (> v min) (<= v max)) 19 | (format "%s factor must be between %s and %s!" name min max))) 20 | 21 | (defn- delay-opts [delay-ms {:keys [max-delay-ms backoff-factor jitter-ms jitter-factor]}] 22 | (assert (not (and jitter-ms jitter-factor)) 23 | "Cannot have both jitter-ms and jitter-factor specified!") 24 | (when max-delay-ms (assert (> max-delay-ms 0) "Max delay must be positive!")) 25 | (when jitter-ms (assert (> jitter-ms 0) "Jitter delay must be positive!")) 26 | (when jitter-factor (verify-factor jitter-factor 0.0 1.0 "Jitter")) 27 | (when backoff-factor (verify-factor backoff-factor 0.0 Double/MAX_VALUE "Backoff")) 28 | {:max-delay-ms (long (or max-delay-ms Long/MAX_VALUE)) 29 | :backoff-factor (if backoff-factor (double backoff-factor) 1.0) 30 | :jitter-ms (if jitter-ms (long jitter-ms) 0) 31 | :jitter-factor (if jitter-factor (double jitter-factor) 0.0)}) 32 | 33 | (defn- compute-jitter [{:keys [jitter-ms jitter-factor]}] 34 | (let [rand-move #(- 1 (* 2 (rand)))] 35 | (cond 36 | (> jitter-ms 0) #(long (+ % (* jitter-ms (rand-move)))) 37 | (> jitter-factor 0) #(long (* % (inc (* jitter-factor (rand-move))))) 38 | :else identity))) 39 | 40 | (defn delay 41 | "Computes a delay for each retry in the `src` seq according to the `opts`. 42 | 43 | When no `opts` given, every delay will be the same `delay-ms`. 44 | 45 | Possible `opts`: 46 | * `max-delay-ms` - the maximum delay 47 | * `backoff-factor` - the factor to multiply each delay with, 48 | e.g. a factor of 2.0 and delay of 50ms will produce delays of 49 | 50, 100, 200, 400, ... 50 | * `jitter-ms` - the amount of time to jitter each delay with, 51 | e.g. a jitter of 25ms and a delay of 50ms will result in delays of 52 | 50, [25; 75], [0; 100], ... or concretely 50, 60, 45, 53, ... 53 | * `jitter-factor` - the factor between 0 and 1 to jitter each delay with, 54 | e.g. a factor of 0.5 and a delay of 50ms will result in delays of 55 | 50, [25; 75], [12; 112], ... or concretely 50, 70, 100, 65, ..." 56 | ([src delay-ms] (delay src delay-ms {})) 57 | ([src delay-ms opts] 58 | (if (> delay-ms 0) 59 | (let [{:keys [max-delay-ms backoff-factor] :as opts} (delay-opts delay-ms opts) 60 | next-delay (volatile! delay-ms) 61 | jitter (compute-jitter opts) 62 | advance #(Math/max 0 (jitter (Math/min (long (* % backoff-factor)) max-delay-ms)))] 63 | (for [_ src] 64 | (let [d @next-delay] 65 | (vswap! next-delay advance) 66 | d))) 67 | src))) 68 | 69 | (defn limit-retries 70 | "Limits the number of retries available through seq `src` to 71 | `max-retries` - a non-negative integer." 72 | [src max-retries] 73 | (let [max-retries (Math/max max-retries (int 0))] 74 | (if (> max-retries 0) 75 | (take max-retries src) 76 | []))) 77 | 78 | (defn limit-duration 79 | "The retries available through seq `src` will be available to take for 80 | `max-duration-ms` milliseconds - a non-negative float. The countdown starts 81 | once `limit-duration` is called. 82 | 83 | The last delay will be the minimum of the remaining time and the delay 84 | received from `src`. E.g. if execution takes 30 ms and we have a source of 85 | constant 30 ms delays with duration limited to 140 ms, then: 86 | 87 | execution: #1 #2 #3 #4 88 | execution takes: 30 30 30 30 89 | time remaining before running delay: - 80 20 - 90 | delay: - 30 20 - 91 | time remaining when next execution started: 110 50 0 -" 92 | [src max-duration-ms] 93 | (let [max-duration-ms (Math/max max-duration-ms 0) 94 | elapsed-since #(- (System/currentTimeMillis) %)] 95 | (if (> max-duration-ms 0) 96 | (let [start-ms (System/currentTimeMillis) 97 | this (fn this [items] 98 | (lazy-seq 99 | (let [remaining (- max-duration-ms (elapsed-since start-ms))] 100 | (when (and (seq items) (> remaining 0)) 101 | (cons (min (first items) remaining) 102 | (this (rest items)))))))] 103 | (this src)) 104 | []))) 105 | 106 | (def ^{:doc "The return value to be used in case the execution needs to be 107 | retried"} 108 | retry ::retry) 109 | (def ^{:doc "The return value to be used in case the execution needs to be 110 | aborted (stopped from being retried)"} 111 | abort ::abort) 112 | 113 | (let [e (Aborted.)] 114 | (defn abort! 115 | "Throws an exception which short-circuits the execution. 116 | 117 | Should only be used inside the retriable code. No more retries will be 118 | performed after an abort." 119 | ([] (throw e)) 120 | ([v] 121 | (throw (if (instance? Throwable v) 122 | (Aborted. ^Throwable v) 123 | (Aborted. ^Object v)))))) 124 | 125 | (let [e (Retried.)] 126 | (defn retry! 127 | "Short-circuits the execution for the next retry. 128 | 129 | Should only be used inside the retriable code." 130 | ([] (throw e)) 131 | ([v] 132 | (throw (if (instance? Throwable v) 133 | (Retried. ^Throwable v) 134 | (Retried. ^Object v)))))) 135 | 136 | (defn reset! 137 | "Short-circuits the execution and restarts the retry cycle with the new seq 138 | of `delays`." 139 | [delays] 140 | (throw (Reset. delays))) 141 | 142 | (defn marker? 143 | "True if the value `v` is an abort/retry/reset marker. 144 | 145 | Useful in case you are catching an `Throwable/Error` in the retriable code 146 | block and want to make sure you only act on a non-marker value, e.g.: 147 | 148 | ``` 149 | (with-retries ... 150 | ... 151 | (catch Throwable t 152 | (when-not (marker? t) 153 | (do-something-on-failure) 154 | (throw t)))) 155 | ```" 156 | [v] 157 | (or (identical? v abort) 158 | (identical? v retry) 159 | (instance? Reset v) 160 | (instance? Aborted v) 161 | (instance? Retried v) 162 | (instance? RetriesExceeded v))) 163 | 164 | (defn- unwrap-marker [v] 165 | (cond (instance? Aborted v) (or (.value ^Aborted v) (.getCause ^Throwable v)) 166 | (instance? Retried v) (or (.value ^Retried v) (.getCause ^Throwable v)) 167 | :else v)) 168 | 169 | (defn- retries-exceeded [retried result] 170 | (let [result' (unwrap-marker result)] 171 | (if (instance? Throwable result') 172 | (doto (RetriesExceeded. retried) 173 | (.initCause result')) 174 | (RetriesExceeded. retried result')))) 175 | 176 | (defn unwrap 177 | "Unwraps a value/cause of a `RetriesExceeded`/`Aborted`/`Retried`." 178 | [v] 179 | (cond (instance? RetriesExceeded v) 180 | (or (.value ^RetriesExceeded v) (.getCause ^Throwable v)) 181 | :else (unwrap-marker v))) 182 | 183 | (def ^:private current-thread-executor 184 | (reify java.util.concurrent.Executor 185 | (execute [_ r] 186 | (r)))) 187 | 188 | (def ^{:dynamic true 189 | :doc "The result/exception which happened during the last 190 | execution. `:manifail/none` during the first execution of if the 191 | execution was reset."} 192 | *last-result* nil) 193 | 194 | (def ^{:dynamic true 195 | :doc "The retry count of the current execution. 196 | Zero during the first one or after a reset."} 197 | *retry-count* nil) 198 | 199 | (def ^{:dynamic true 200 | :doc "The number of milliseconds spent on executions and retries up to 201 | the current execution."} 202 | *elapsed-ms* nil) 203 | 204 | (defn with-retries* 205 | "Executes the given function `f` at least once and then as long as retry 206 | seq `delays` isn't drained and execution is deemed retriable. 207 | 208 | A retriable execution is one which fails due to a (non-abort) exception or 209 | returns a special `retry` value. 210 | 211 | `delays` is a seq producing numbers of milliseconds to wait after each 212 | execution. Zero means no delay, a negative delay means that execution has to 213 | stop. 214 | 215 | If you want to specify an executor which should be used to run `f`, use 216 | `manifold.executor/with-executor`, e.g.: 217 | 218 | ``` 219 | (manifold.executor/with-executor my-executor 220 | (with-retries* (retries 5) 221 | call-service)) 222 | ```" 223 | [delays f] 224 | (let [reset-delays (fn [^Reset r] (.retryDelays r)) 225 | ex (or (ex/executor) current-thread-executor) 226 | started (System/nanoTime)] 227 | (d/loop [retried 0, last-result ::none, delays' delays] 228 | (-> (d/future-with ex 229 | (binding [*last-result* last-result 230 | *retry-count* retried 231 | *elapsed-ms* (/ (- (System/nanoTime) started) 1e6)] 232 | (f))) 233 | (d/catch' identity) 234 | (d/chain' 235 | (fn [result] 236 | (cond (instance? Reset result) (d/recur 0 ::none (reset-delays result)) 237 | (instance? Aborted result) (throw result) 238 | (identical? result abort) (abort!) 239 | 240 | (or (instance? Throwable result) 241 | (identical? result retry)) 242 | (-> (let [d (first delays')] 243 | (when (or (nil? d) (< d 0)) 244 | (throw (retries-exceeded retried result))) 245 | (-> (d/deferred ex) 246 | (d/timeout! d ::run))) 247 | (d/chain' (fn [_] (d/recur (inc retried) 248 | (unwrap-marker result) 249 | (rest delays'))))) 250 | :else result))))))) 251 | 252 | (defmacro with-retries 253 | "Macro wrapper over `with-retries*`. 254 | 255 | See docs for `with-retries*`." 256 | [delays & body] 257 | `(with-retries* ~delays (fn [] ~@body))) 258 | -------------------------------------------------------------------------------- /src/java/manifail/Aborted.java: -------------------------------------------------------------------------------- 1 | package manifail; 2 | 3 | /** 4 | * Exception thrown when the execution is aborted. 5 | */ 6 | public class Aborted extends Throwable { 7 | private static final long serialVersionUID = -3120638907441644021L; 8 | public final Object value; 9 | public Aborted() { 10 | this.value = null; 11 | } 12 | public Aborted(Object value) { 13 | this.value = value; 14 | } 15 | public Aborted(Throwable cause) { 16 | super(cause); 17 | this.value = null; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/java/manifail/Reset.java: -------------------------------------------------------------------------------- 1 | package manifail; 2 | 3 | /** 4 | * Exception thrown when the execution is reset. 5 | */ 6 | public class Reset extends Throwable { 7 | private static final long serialVersionUID = -1234543212343211234L; 8 | public final Object retryDelays; 9 | public Reset(Object retryDelays) { 10 | this.retryDelays = retryDelays; 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/java/manifail/Retried.java: -------------------------------------------------------------------------------- 1 | package manifail; 2 | 3 | /** 4 | * Exception thrown when the execution is retried. 5 | */ 6 | public class Retried extends Throwable { 7 | private static final long serialVersionUID = -5238638352410341922L; 8 | public final Object value; 9 | public Retried() { 10 | this.value = null; 11 | } 12 | public Retried(Object value) { 13 | this.value = value; 14 | } 15 | public Retried(Throwable cause) { 16 | super(cause); 17 | this.value = null; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/java/manifail/RetriesExceeded.java: -------------------------------------------------------------------------------- 1 | package manifail; 2 | 3 | /** 4 | * Exception thrown when the number of retries is exceeded. 5 | */ 6 | public class RetriesExceeded extends Throwable { 7 | private static final long serialVersionUID = 1552386065262456541L; 8 | public final int retries; 9 | public final Object value; 10 | public RetriesExceeded(int retries) { 11 | this.retries = retries; 12 | this.value = null; 13 | } 14 | public RetriesExceeded(int retries, Object value) { 15 | this.retries = retries; 16 | this.value = value; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test/manifail_test.clj: -------------------------------------------------------------------------------- 1 | (ns manifail-test 2 | (:require [clojure.test :refer :all] 3 | [manifold 4 | [deferred :as d] 5 | [stream :as s] 6 | [executor :as ex]] 7 | [manifail :as sut]) 8 | (:import [java.util.concurrent Executors])) 9 | 10 | (defn- ->seq [s] (vec s)) 11 | 12 | (deftest fixed-retry-times 13 | (is (= 1 (count (->seq (sut/retries 1))))) 14 | (is (= 5 (count (->seq (sut/retries 5)))))) 15 | 16 | (deftest limited-times 17 | (is (= 0 (-> 5 sut/retries (sut/limit-retries 0) ->seq count))) 18 | (is (= 1 (-> 5 sut/retries (sut/limit-retries 1) ->seq count))) 19 | (is (= 5 (-> 5 sut/retries (sut/limit-retries 6) ->seq count)))) 20 | 21 | (deftest limited-duration 22 | (is (= 0 (-> 1 sut/retries (sut/limit-duration 0) ->seq count))) 23 | 24 | (let [s (sut/limit-duration (sut/retries 100) 5) 25 | cnt (atom 0)] 26 | (d/loop [s' s] 27 | (Thread/sleep 1) 28 | (-> (or (first s') (d/success-deferred ::done)) 29 | (d/chain 30 | (fn [result] 31 | (swap! cnt inc) 32 | (when-not (= ::done result) (d/recur (rest s'))))))) 33 | (is (>= 5 @cnt)))) 34 | 35 | (defn around [expected-ms took-ms] 36 | (<= (- expected-ms 1) took-ms (+ expected-ms 1))) 37 | 38 | (defn between [lower-ms upper-ms took-ms] 39 | (<= (- lower-ms 1) took-ms (+ upper-ms 1))) 40 | 41 | (defn- realized-seq [s] 42 | s) 43 | 44 | (deftest delayed 45 | (testing "simple delay" 46 | (is (= [50] (realized-seq (sut/delay (sut/retries 1) 50))))) 47 | 48 | (testing "backoff factor" 49 | (is (= [10 20 40] (realized-seq (sut/delay (sut/retries 3) 10 {:backoff-factor 2.0}))))) 50 | 51 | (testing "backoff factor and max delay" 52 | (is (= [10 20 20] 53 | (realized-seq (sut/delay (sut/retries 3) 10 54 | {:backoff-factor 2.0, :max-delay-ms 20}))))) 55 | 56 | (testing "jitter-ms" 57 | (let [[a b c] (realized-seq 58 | (sut/delay (sut/retries 3) 20 {:jitter-ms 5}))] 59 | (is (around 20 a)) 60 | (is (between 15 25 b)) 61 | (is (between 10 30 c)))) 62 | 63 | (testing "jitter-factor" 64 | (let [[a b c] (realized-seq 65 | (sut/delay (sut/retries 3) 20 {:jitter-factor 0.5}))] 66 | (is (around 20 a)) 67 | (is (between 10 30 b)) 68 | (is (between 5 45 c))))) 69 | 70 | (defn- aborted? 71 | ([r] (aborted? (constantly true) r)) 72 | ([ex-pred r] 73 | (try @r 74 | false 75 | (catch manifail.Aborted e (ex-pred e))))) 76 | 77 | (defn- retries-exceeded? 78 | ([r] (retries-exceeded? (constantly true) r)) 79 | ([ex-pred r] 80 | (try @r 81 | false 82 | (catch manifail.RetriesExceeded e (ex-pred e))))) 83 | 84 | (deftest with-retries 85 | (let [executed (atom 0)] 86 | 87 | (testing "executes once" 88 | (is 89 | (= ::done 90 | @(sut/with-retries* (sut/retries 1) 91 | #(do (swap! executed inc) ::done)))) 92 | (is (= 1 @executed))) 93 | 94 | (reset! executed 0) 95 | 96 | (testing "executes several times, retries using value" 97 | (is 98 | (retries-exceeded? #(= (sut/unwrap %) sut/retry) 99 | (sut/with-retries* (sut/retries 2) 100 | #(do (swap! executed inc) 101 | sut/retry)))) 102 | (is (= 3 @executed))) 103 | 104 | (reset! executed 0) 105 | 106 | (testing "executes several times, retries using retry!" 107 | (is 108 | (retries-exceeded? #(nil? (sut/unwrap %)) 109 | (sut/with-retries* (sut/retries 2) 110 | #(do (swap! executed inc) 111 | (sut/retry!))))) 112 | (is (= 3 @executed))) 113 | 114 | (reset! executed 0) 115 | 116 | (testing "retries exceeded carries cause" 117 | (is 118 | (retries-exceeded? #(= (.getMessage (sut/unwrap %)) "boom3") 119 | (sut/with-retries* (sut/retries 2) 120 | #(do (swap! executed inc) 121 | (sut/retry! (Exception. (str "boom" @executed))))))) 122 | (is (= 3 @executed))) 123 | 124 | (reset! executed 0) 125 | 126 | (testing "executes several times, retries using generic exception" 127 | (is 128 | (retries-exceeded? #(= (.getMessage (sut/unwrap %)) "boom") 129 | (sut/with-retries* (sut/retries 2) 130 | #(do (swap! executed inc) 131 | (throw (Exception. "boom")))))) 132 | (is (= 3 @executed))) 133 | 134 | (reset! executed 0) 135 | 136 | (testing "aborts execution using return value" 137 | (is 138 | (aborted? 139 | (sut/with-retries* (sut/retries 3) 140 | #(do (swap! executed inc) 141 | (if (= @executed 2) 142 | sut/abort 143 | sut/retry))))) 144 | (is (= 2 @executed))) 145 | 146 | (reset! executed 0) 147 | 148 | (testing "aborts execution after some time" 149 | (is 150 | (aborted? 151 | (sut/with-retries* (sut/retries 3) 152 | #(do (swap! executed inc) 153 | (if (= @executed 2) 154 | (sut/abort!) 155 | (sut/retry!)))))) 156 | (is (= 2 @executed))) 157 | 158 | (reset! executed 0) 159 | 160 | (testing "sets abort! cause" 161 | (is 162 | (aborted? #(= (.getMessage (sut/unwrap %)) "boom") 163 | (sut/with-retries* (sut/retries 3) 164 | #(sut/abort! (Exception. "boom"))))) 165 | (is (= 0 @executed))) 166 | 167 | (reset! executed 0) 168 | 169 | (testing "sets abort! value" 170 | (is 171 | (aborted? #(= (sut/unwrap %) "boom") 172 | (sut/with-retries* (sut/retries 3) 173 | #(sut/abort! "boom")))) 174 | (is (= 0 @executed))) 175 | 176 | (reset! executed 0) 177 | 178 | (testing "limits duration" 179 | (is 180 | (retries-exceeded? 181 | (sut/with-retries* (sut/limit-duration (sut/retries 3) 50) 182 | #(do (swap! executed inc) 183 | (Thread/sleep 30) 184 | (sut/retry!))))) 185 | (is (= 2 @executed))) 186 | 187 | (reset! executed 0) 188 | 189 | (testing "limits duration with delay" 190 | (is 191 | (retries-exceeded? 192 | (sut/with-retries* (sut/limit-duration (sut/delay (sut/retries 3) 20) 50) 193 | #(do (swap! executed inc) 194 | (Thread/sleep 30) 195 | (sut/retry!))))) 196 | (is (= 2 @executed))))) 197 | 198 | (deftest with-retries-reset 199 | (let [executed (atom 0)] 200 | 201 | (testing "resets" 202 | (is 203 | (retries-exceeded? #(= (sut/unwrap %) 5) 204 | (sut/with-retries* (sut/retries 2) 205 | #(do (swap! executed inc) 206 | (if (= @executed 3) 207 | (sut/reset! (sut/retries 1)) 208 | (sut/retry! @executed)))))) 209 | ;; (1 execution + 2 retries initially) + (1 execution + 1 retry after reset) 210 | (is (= 5 @executed))))) 211 | 212 | (deftest dynamic-vars 213 | (let [states (atom []) 214 | eps-ms 15, execution-ms 50, delay-ms 100] 215 | (is 216 | (retries-exceeded? 217 | (sut/with-retries* (repeat 2 delay-ms) 218 | #(do (swap! states conj {:elapsed sut/*elapsed-ms* 219 | :last-result sut/*last-result* 220 | :retrying sut/*retry-count*}) 221 | (Thread/sleep execution-ms) 222 | (sut/retry! (count @states)))))) 223 | 224 | (is (= 3 (count @states))) 225 | (let [{:keys [elapsed last-result retrying]} (first @states)] 226 | (is (<= elapsed eps-ms)) 227 | (is (= retrying 0)) 228 | (is (= last-result ::sut/none))) 229 | 230 | (let [{:keys [elapsed last-result retrying]} (second @states)] 231 | (is (<= elapsed (+ execution-ms delay-ms eps-ms))) 232 | (is (= retrying 1)) 233 | (is (= last-result 1))) 234 | 235 | (let [{:keys [elapsed last-result retrying]} (nth @states 2)] 236 | (is (<= elapsed (+ (* 2 (+ execution-ms delay-ms eps-ms))))) 237 | (is (= retrying 2)) 238 | (is (= last-result 2))))) 239 | --------------------------------------------------------------------------------