├── .gitignore ├── bin └── kaocha ├── .dir-locals.el ├── tests.edn ├── Makefile ├── src └── courier │ ├── client.cljc │ ├── time.cljc │ ├── fingerprint.cljc │ ├── fs.clj │ ├── fs.cljs │ ├── cache.cljc │ └── http.cljc ├── deps.edn ├── pom.xml ├── test └── courier │ ├── cache_test.cljc │ └── http_test.cljc ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | /.nrepl-port 3 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | clojure -A:dev:test "$@" 3 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil 2 | (cider-clojure-cli-global-options . "-A:dev"))) 3 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:kaocha.watch/ignore ["**/#*" "**/.#*"] 3 | :plugins [:noyoda.plugin/swap-actual-and-expected] 4 | :tests [{:id :unit 5 | :test-paths ["test"] 6 | :source-paths ["src"]}]} 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | clojure -A:dev:test 3 | 4 | courier.jar: src/courier/*.* 5 | clojure -A:jar 6 | 7 | clean: 8 | rm -fr target courier.jar 9 | 10 | deploy: courier.jar 11 | mvn deploy:deploy-file -Dfile=courier.jar -DrepositoryId=clojars -Durl=https://clojars.org/repo -DpomFile=pom.xml 12 | 13 | .PHONY: test deploy clean 14 | -------------------------------------------------------------------------------- /src/courier/client.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.client 2 | (:require #?(:clj [clj-http.client :as http] 3 | :cljs [cljs-http.client :as http]))) 4 | 5 | (defn success? [res] 6 | (http/success? res)) 7 | 8 | (defmulti request (fn [req] [(:method req) (:url req)])) 9 | 10 | (defmethod request :default [req] 11 | (http/request req)) 12 | -------------------------------------------------------------------------------- /src/courier/time.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.time 2 | #?(:clj (:import java.time.Instant))) 3 | 4 | (defn now [] 5 | #?(:clj (java.time.Instant/now) 6 | :cljs (.getTime (js/Date.)))) 7 | 8 | (defn ->inst [i] 9 | (if (number? i) 10 | #?(:clj (java.time.Instant/ofEpochMilli i) 11 | :cljs (js/Date. i)) 12 | i)) 13 | 14 | (defn before? [a b] 15 | #?(:clj (.isBefore (->inst a) (->inst b)) 16 | :cljs (< a b))) 17 | 18 | (defn millis [t] 19 | (if (number? t) 20 | t 21 | #?(:clj (.toEpochMilli t) 22 | :cljs (.getTime t)))) 23 | 24 | (defn add-millis [t ms] 25 | #?(:clj (millis (.plusMillis t ms)) 26 | :cljs (+ t ms))) 27 | -------------------------------------------------------------------------------- /src/courier/fingerprint.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.fingerprint 2 | "Create unique string fingerprints for arbitrary data structures" 3 | (:require [clojure.walk :as walk] 4 | [clojure.string :as str]) 5 | #?(:clj (:import [java.security MessageDigest]))) 6 | 7 | (defn sorted 8 | "Recursively sort maps and sets in a data structure" 9 | [data] 10 | (walk/postwalk 11 | (fn [x] 12 | (cond 13 | (map? x) (apply sorted-map (apply concat x)) 14 | (set? x) (apply sorted-set x) 15 | :else x)) 16 | data)) 17 | 18 | (defn md5 [s] 19 | #?(:clj 20 | (let [md (MessageDigest/getInstance "MD5")] 21 | (.update md (.getBytes s)) 22 | (str/join (map #(format "%x" %) (.digest md)))) 23 | :cljs s)) 24 | 25 | (defn fingerprint [data] 26 | (binding [*print-length* -1] 27 | (md5 (pr-str (sorted data))))) 28 | 29 | (comment 30 | (fingerprint [1 2 3]) 31 | (fingerprint (list 1 2 3)) 32 | 33 | 34 | ) 35 | -------------------------------------------------------------------------------- /src/courier/fs.clj: -------------------------------------------------------------------------------- 1 | (ns courier.fs 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as str]) 4 | (:import (java.io File) 5 | (java.nio.file Files StandardCopyOption))) 6 | 7 | (defn read-file [file] 8 | (let [java-file (io/file file)] 9 | (when (.exists java-file) 10 | (slurp java-file)))) 11 | 12 | (defn delete-file [file] 13 | (io/delete-file file true)) 14 | 15 | (defn write-file 16 | "Writes the file to a temporary file, then performs an atomic move, ensuring 17 | that cache files are never partial files." 18 | [file content] 19 | (let [end-file (if (isa? File file) file (File. ^String file)) 20 | tmp-f (File/createTempFile (.getName end-file) ".tmp" (.getParentFile end-file))] 21 | (spit tmp-f content) 22 | (Files/move (.toPath tmp-f) (.toPath end-file) (into-array [StandardCopyOption/ATOMIC_MOVE])))) 23 | 24 | (defn ensure-dir [dir] 25 | (.mkdirs (File. dir))) 26 | 27 | (defn dirname [path] 28 | (str/join "/" (drop-last (str/split path #"/")))) 29 | -------------------------------------------------------------------------------- /src/courier/fs.cljs: -------------------------------------------------------------------------------- 1 | (ns courier.fs 2 | (:require [fs] 3 | [path])) 4 | 5 | (defn read-file [file] 6 | (try 7 | (.readFileSync fs file "utf-8") 8 | (catch :default e 9 | nil))) 10 | 11 | (defn delete-file [file] 12 | (try 13 | (.unlinkSync fs file) 14 | (catch :default e nil))) 15 | 16 | (defn write-file [file str] 17 | (.writeFileSync fs file str "utf-8")) 18 | 19 | (def ^:private mode (js/parseInt "0755" 8)) 20 | 21 | (defn mkdirs [dir & [made]] 22 | (let [dir (.resolve path dir)] 23 | (try 24 | (.mkdirSync fs dir mode) 25 | (or made dir) 26 | (catch :default root-error 27 | (if (= "ENOENT" (.-code root-error)) 28 | (->> made 29 | (mkdirs (.dirname path dir)) 30 | (mkdirs dir)) 31 | (when-not (.isDirectory (try 32 | (.statSync fs dir) 33 | (catch :default e 34 | (throw root-error)))) 35 | (throw root-error))))))) 36 | 37 | (defn ensure-dir [dir] 38 | (mkdirs dir)) 39 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"] 2 | :deps {org.clojure/clojure {:mvn/version "1.10.1"} 3 | org.clojure/core.async {:mvn/version "1.3.610"} 4 | clj-http/clj-http {:mvn/version "3.10.1"}} 5 | :aliases {:dev {:extra-paths ["test"] 6 | :extra-deps {org.clojure/clojurescript {:mvn/version "1.10.773"} 7 | org.clojure/tools.namespace {:mvn/version "0.3.0-alpha4"} 8 | org.clojure/test.check {:mvn/version "0.10.0-alpha4"} 9 | com.taoensso/carmine {:mvn/version "3.1.0"} 10 | cheshire/cheshire {:mvn/version "5.10.0"} 11 | lambdaisland/kaocha {:mvn/version "1.0.700"} 12 | kaocha-noyoda/kaocha-noyoda {:mvn/version "2019-06-03"}}} 13 | :test {:main-opts ["-m" "kaocha.runner"]} 14 | :jar {:extra-deps {pack/pack.alpha {:git/url "https://github.com/juxt/pack.alpha.git" 15 | :sha "e518d9b2b70f4292c9988d2792b8667d88a6f4df"}} 16 | :main-opts ["-m" "mach.pack.alpha.skinny" "--no-libs" "--project-path" "courier.jar"]}}} 17 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | jar 5 | cjohansen 6 | courier 7 | 2021.01.26 8 | courier 9 | 10 | 11 | org.clojure 12 | clojure 13 | 1.10.1 14 | 15 | 16 | clj-http 17 | clj-http 18 | 3.10.1 19 | 20 | 21 | org.clojure 22 | core.async 23 | 1.3.610 24 | 25 | 26 | 27 | src 28 | 29 | 30 | 31 | clojars 32 | https://repo.clojars.org/ 33 | 34 | 35 | A high-level HTTP client with built-in caching, retries, and "http request dependencies". 36 | https://github.com/cjohansen/courier 37 | 38 | 39 | EPL 40 | https://opensource.org/licenses/EPL-2.0 41 | 42 | 43 | 44 | scm:git:git://github.com/cjohansen/courier.git 45 | scm:git:ssh://git@github.com/cjohansen/courier.git 46 | v2021.01.26 47 | https://github.com/cjohansen/courier 48 | 49 | 50 | -------------------------------------------------------------------------------- /test/courier/cache_test.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.cache-test 2 | (:require [courier.cache :as sut] 3 | [clojure.test :refer [deftest is]])) 4 | 5 | (deftest cache-key-bare-request 6 | (is (= (sut/cache-key {:req {:method :get 7 | :url "https://example.com/"}} nil) 8 | [:courier.http/req {:method :get 9 | :url "https://example.com/"}]))) 10 | 11 | (deftest cache-key-bare-request-with-headers-query-params-and-body 12 | (is (= (sut/cache-key {:req {:method :post 13 | :as :json 14 | :content-type :json 15 | :headers {"X-Correlation-Id" 42} 16 | :form-params {:grant_type "client_credentials"} 17 | :basic-auth ["id" "secret"] 18 | :query-params {:client "someone"} 19 | :url "https://example.com/"}} nil) 20 | [:courier.http/req 21 | {:method :post 22 | :url "https://example.com/" 23 | :as :json 24 | :content-type :json 25 | :query-params {:client "someone"} 26 | :headers {"x-correlation-id" 42} 27 | :hash "f99b15885a9cf25b45b1472abec32b60"}]))) 28 | 29 | (deftest cache-key-bare-request-with-params 30 | (is (= (sut/cache-key {:req {:method :get 31 | :url "https://example.com/"}} {:id 42}) 32 | [:courier.http/req 33 | {:method :get 34 | :url "https://example.com/" 35 | :lookup-params {:id 42}}]))) 36 | 37 | (deftest inline-req-fn 38 | (is (->> (sut/cache-key {:req-fn (fn [params])} nil) 39 | first 40 | name 41 | (re-find #"fn")))) 42 | 43 | (defn stuff-request [params]) 44 | 45 | (deftest defn-req-fn 46 | (is (= (sut/cache-key {:req-fn stuff-request} {:id 42}) 47 | [:courier.cache-test/stuff-request {:id 42}]))) 48 | 49 | (deftest var-req-fn 50 | (is (= (sut/cache-key {:req-fn #'stuff-request} {:id 42}) 51 | [:courier.cache-test/stuff-request {:id 42}]))) 52 | 53 | (deftest req-fn-with-meta 54 | (is (= (sut/cache-key {:req-fn (with-meta (fn [_]) {:name "lol"})} {:id 42}) 55 | [:lol {:id 42}])) 56 | (is (= (sut/cache-key {:req-fn (with-meta (fn [_]) 57 | {:name "lol" 58 | :ns "ok"})} {:id 42}) 59 | [:ok/lol {:id 42}]))) 60 | -------------------------------------------------------------------------------- /src/courier/cache.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.cache 2 | (:require [clojure.string :as str] 3 | [courier.fs :as fs] 4 | [courier.time :as time] 5 | [courier.fingerprint :as fingerprint])) 6 | 7 | (defprotocol Cache 8 | (lookup [_ spec params]) 9 | (put [_ spec params res])) 10 | 11 | (defn fname [f] 12 | (if-let [meta-name (-> f meta :name)] 13 | (keyword (some-> f meta :ns str) (str meta-name)) 14 | (when-let [name (some-> f str (str/replace #"_" "-"))] 15 | (let [name (-> (re-find #"(?:#')?(.*)" name) 16 | second 17 | (str/replace #"-QMARK-" "?"))] 18 | #?(:clj 19 | (if-let [[_ ns n] (re-find #"(.*)\$(.*)@" name)] 20 | (keyword ns n) 21 | (keyword name)) 22 | :cljs 23 | (if-let [[_ res] (re-find #"function (.+)\(" name)] 24 | (let [[f & ns ] (-> res (str/split #"\$") reverse)] 25 | (keyword (str/join "." (reverse ns)) f)) 26 | (if (re-find #"function \(" name) 27 | (keyword (str (random-uuid))) 28 | (keyword (str/replace name #" " "-"))))))))) 29 | 30 | (defn cache-id [{:keys [lookup-id req-fn]}] 31 | (or lookup-id 32 | (when req-fn (fname req-fn)) 33 | :courier.http/req)) 34 | 35 | (defn- normalize-headers [headers] 36 | (->> headers 37 | (map (fn [[h v]] [(str/lower-case h) v])) 38 | (into {}))) 39 | 40 | (defn get-cache-relevant-params [{:keys [lookup-id req-fn req]} params] 41 | (if (or lookup-id req-fn) 42 | params 43 | (let [[url query-string] (str/split (:url req) #"\?")] 44 | (merge 45 | {:method :get} 46 | (select-keys req [:method :url :as :content-type :query-params]) 47 | (when query-string 48 | {:query-params 49 | (->> (str/split query-string #"&") 50 | (map #(let [[k & args] (str/split % #"=")] 51 | [(keyword k) (str/join args)])) 52 | (into {}))}) 53 | (when-let [headers (:headers req)] 54 | {:headers (normalize-headers headers)}) 55 | (let [sensitive (select-keys req [:body :form-params :basic-auth])] 56 | (when-not (empty? sensitive) 57 | {:hash (fingerprint/fingerprint sensitive)})) 58 | (when-not (empty? params) 59 | {:lookup-params params}))))) 60 | 61 | (defn cache-key [spec params] 62 | [(cache-id spec) (get-cache-relevant-params spec params)]) 63 | 64 | (defn expired? [res] 65 | (and (int? (:expires-at res)) 66 | (not (time/before? (time/now) (:expires-at res))))) 67 | 68 | (defn retrieve [cache spec params] 69 | (when-let [res (lookup cache spec params)] 70 | (when (not (expired? res)) 71 | res))) 72 | 73 | (defn cacheable [result] 74 | (-> (select-keys result [:req :res :success?]) 75 | (assoc :expires-at (-> result :cache :expires-at)) 76 | (update :res dissoc :http-client) 77 | (assoc :cached-at (time/millis (time/now))))) 78 | 79 | (defn store [cache spec params res] 80 | (let [cacheable-result (cacheable res) 81 | cache-data (put cache spec params cacheable-result)] 82 | (assoc cacheable-result :path (:path res) :cache-status cache-data))) 83 | 84 | (defn create-atom-map-cache [ref] 85 | (assert (instance? clojure.lang.Atom ref) 86 | (format "ref must be an atom, was %s" (type ref))) 87 | (assert (or (map? @ref) (nil? @ref)) "ref must contain nil or a map") 88 | (reify Cache 89 | (lookup [_ spec params] 90 | (get @ref (cache-key spec params))) 91 | (put [_ spec params res] 92 | (let [k (cache-key spec params)] 93 | (swap! ref assoc k res) 94 | {::key k})))) 95 | 96 | (defn str-id [spec] 97 | (let [id (cache-id spec)] 98 | (str (namespace id) "." (name id)))) 99 | 100 | (defn filename [dir spec params] 101 | (let [fingerprinted-name (fingerprint/fingerprint (get-cache-relevant-params spec params)) 102 | [_ prefix postfix] (re-find #"(..)(.+)" fingerprinted-name) 103 | dirname (str dir "/" (str-id spec) "/" prefix)] 104 | (str dirname "/" postfix ".edn"))) 105 | 106 | (defn slurp-edn [file] 107 | (try 108 | (let [content (fs/read-file file)] 109 | (if-not (empty? content) 110 | #?(:clj (read-string content) 111 | :cljs (cljs.reader/read-string content)) 112 | nil)) 113 | (catch #?(:clj Throwable 114 | :cljs :default) e 115 | nil))) 116 | 117 | (defn create-file-cache [{:keys [dir]}] 118 | (assert (string? dir) "Can't create file cache without directory") 119 | (fs/ensure-dir dir) 120 | (reify Cache 121 | (lookup [_ spec params] 122 | (when-let [file (filename dir spec params)] 123 | (when-let [val (slurp-edn file)] 124 | (if (expired? val) 125 | (do 126 | (fs/delete-file file) 127 | nil) 128 | val)))) 129 | (put [_ spec params res] 130 | (when-let [file (filename dir spec params)] 131 | (fs/ensure-dir (fs/dirname file)) 132 | (fs/write-file file (pr-str (assoc res ::file-name file))) 133 | {::file-name file})))) 134 | 135 | (def ^:private carmine-available? 136 | "Carmine/Redis is an optional dependency, so we try to load it runtime. If the 137 | dependency is available, the redis cache can be used." 138 | (try 139 | (require 'taoensso.carmine) 140 | true 141 | (catch Throwable _ false))) 142 | 143 | (defmacro wcar [& body] 144 | (when carmine-available? 145 | `(taoensso.carmine/wcar ~@body))) 146 | 147 | (defn- redis-f [f & args] 148 | (apply (ns-resolve (symbol "taoensso.carmine") (symbol (name f))) args)) 149 | 150 | (defn- redis-cache-key [spec params] 151 | (let [id (cache-id spec) 152 | params (get-cache-relevant-params spec params)] 153 | (->> [(namespace id) 154 | (name id) 155 | (fingerprint/fingerprint params)] 156 | (remove empty?) 157 | (str/join "/")))) 158 | 159 | (defn create-redis-cache [conn-opts] 160 | (assert carmine-available? "com.taoensso/carmine needs to be on the classpath") 161 | (assert (not (nil? conn-opts)) "Please provide connection options") 162 | (reify Cache 163 | (lookup [_ spec params] 164 | (wcar conn-opts (redis-f :get (redis-cache-key spec params)))) 165 | (put [_ spec params res] 166 | (let [ttl (- (time/millis (:expires-at res)) (time/millis (time/now))) 167 | cache-key (redis-cache-key spec params)] 168 | (wcar conn-opts (redis-f :psetex cache-key ttl (assoc res ::key cache-key))) 169 | {::key cache-key})))) 170 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /src/courier/http.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.http 2 | (:require #?(:clj [clojure.core.async :as a] 3 | :cljs [cljs.core.async :as a]) 4 | [courier.cache :as cache] 5 | [courier.client :as client] 6 | [courier.time :as time]) 7 | #?(:clj (:import java.time.Instant))) 8 | 9 | (defn emit [ch event exchange] 10 | (a/put! ch (-> exchange 11 | (dissoc :spec) 12 | (assoc :event event)))) 13 | 14 | (defn try-emit 15 | "Like try-catch, except emit an exception event instead of throwing exceptions." 16 | [log f & args] 17 | (try 18 | (apply f args) 19 | (catch Throwable e 20 | (emit log ::exception {:throwable e 21 | :source (cache/fname f)}) 22 | nil))) 23 | 24 | (defn with-name [f n] 25 | (with-meta f {:name n})) 26 | 27 | (defn prepare-request [log {:keys [req req-fn params]} ctx] 28 | (let [req (or req (try-emit log (with-name req-fn "req-fn") (select-keys ctx params)))] 29 | (-> req 30 | (update :method #(or % :get)) 31 | (assoc :throw-exceptions false)))) 32 | 33 | (defn success? [{:keys [spec req res]}] 34 | (let [f (or (:success? spec) (comp client/success? :res))] 35 | (f {:res res :req req}))) 36 | 37 | (defn retryable? [{:keys [req]}] 38 | (= :get (:method req))) 39 | 40 | (defn requests-for [exchanges path] 41 | (seq (filter (comp #{path} :path) exchanges))) 42 | 43 | (defn get-retry-delay [exchanges path] 44 | (-> (requests-for exchanges path) last :retry :delay)) 45 | 46 | (defn prepare-result [log exchange] 47 | (let [success? (try-emit log success? exchange)] 48 | (assoc exchange :success? (boolean success?)))) 49 | 50 | (defn select-paths [m ks] 51 | (reduce #(let [k (if (coll? %2) %2 [%2])] 52 | (assoc-in %1 k (get-in m k))) nil ks)) 53 | 54 | (defn lookup-params [{:keys [params lookup-params]} ctx] 55 | (let [required (or lookup-params params) 56 | params (select-paths ctx required)] 57 | {:required (map (fn [k] (if (coll? k) (first k) k)) required) 58 | :params params})) 59 | 60 | (defn prepare-lookup-params [{:keys [prepare-lookup-params]} params] 61 | (cond-> params 62 | (ifn? prepare-lookup-params) prepare-lookup-params)) 63 | 64 | (defn maybe-cache-result [log spec ctx cache result] 65 | (when (and cache (-> result :cache :cache?)) 66 | (let [{:keys [params]} (lookup-params spec ctx)] 67 | (try 68 | (->> (cache/store cache spec (prepare-lookup-params spec params) result) 69 | (emit log ::store-in-cache)) 70 | (catch Exception e 71 | (emit log ::exception {:throwable e 72 | :source "courier.cache/put"}) 73 | nil))))) 74 | 75 | (defn fulfill-exchange [log exchange] 76 | (a/go 77 | (let [res (a/> m 97 | (remove (comp nil? second)) 98 | (every? (fn [[k v]] 99 | (if-let [f (get validations k)] 100 | (f v) 101 | true))))) 102 | 103 | (defn get-retry-info [{:keys [spec path req res success?] :as exchange} log exchanges] 104 | (when-not success? 105 | (when-let [f (:retry-fn spec)] 106 | (let [ctx {:req req 107 | :res res 108 | ;; Increase the count to include the current attempt 109 | :num-attempts (inc (count (requests-for exchanges path)))} 110 | retry (try-emit log (with-meta f {:name "retry-fn"}) ctx)] 111 | (if (valid-keys? retry (:retry validations)) 112 | retry 113 | (do 114 | (emit log ::invalid-data (assoc exchange :retry retry)) 115 | nil)))))) 116 | 117 | (defn get-cache-info [{:keys [spec path req res success?] :as exchange} log] 118 | (when success? 119 | (when-let [f (:cache-fn spec)] 120 | (let [ctx {:req req :res res} 121 | cache (try-emit log (with-meta f {:name "cache-fn"}) ctx)] 122 | (if (valid-keys? cache (:cache validations)) 123 | cache 124 | (do 125 | (emit log ::invalid-data (assoc exchange :cache cache)) 126 | nil)))))) 127 | 128 | (defn make-request [log spec ctx path exchanges cache] 129 | (a/go 130 | (when-let [delay (get-retry-delay exchanges path)] 131 | (a/ exchanges last :retry :retry?)))) 156 | 157 | (defn find-pending [specs ctx ks exchanges] 158 | (->> (remove #(contains? ctx %) ks) 159 | (filter #(params-available? ctx (get specs %))) 160 | (filter #(eligible? % (get specs %) exchanges)))) 161 | 162 | (defn prepare-for-context [{:keys [spec res]}] 163 | (if-let [f (::select spec)] 164 | (f res) 165 | res)) 166 | 167 | (defn extract-result-data [chans] 168 | (a/go-loop [chans chans 169 | result {} 170 | exchanges []] 171 | (if (seq chans) 172 | (let [[v ch] (a/alts! chans)] 173 | (recur (remove #{ch} chans) 174 | (cond-> result 175 | (:success? v) (assoc (:path v) (prepare-for-context v))) 176 | (conj exchanges v))) 177 | {:result result 178 | :exchanges exchanges}))) 179 | 180 | (defn get-cached [log cache specs k ctx] 181 | (let [spec (k specs) 182 | {:keys [required params]} (lookup-params spec ctx)] 183 | (when (and (not (:refresh? spec)) 184 | (= (count required) (count params))) 185 | (try 186 | (when-let [cached (cache/retrieve cache spec (prepare-lookup-params spec params))] 187 | (prepare-result log (assoc cached :path k :spec spec))) 188 | (catch Exception e 189 | (emit log ::exception {:throwable e 190 | :source "courier.cache/lookup"}) 191 | nil))))) 192 | 193 | (defn lookup-cache [log specs ctx ks all-exchanges cache] 194 | (when cache 195 | (when-let [cached (seq (keep #(get-cached log cache specs % ctx) ks))] 196 | (doseq [x cached] 197 | (emit log ::cache-hit x)) 198 | (a/go 199 | (let [cached-paths (set (map :path cached))] 200 | {:specs specs 201 | :ctx (merge ctx (->> cached 202 | (map (juxt :path prepare-for-context)) 203 | (into {}))) 204 | :ks (remove cached-paths ks) 205 | :exchanges all-exchanges}))))) 206 | 207 | (defn find-stale-keys [specs exchanges] 208 | (->> exchanges 209 | (remove :success?) 210 | (mapcat (fn [{:keys [spec retry] :as result}] 211 | (when (:retry? retry) 212 | (:refresh retry)))))) 213 | 214 | (defn mark-for-refresh [specs ks] 215 | (->> ks 216 | (keep (fn [k] 217 | (when-let [spec (k specs)] 218 | [k (assoc spec :refresh? true)]))) 219 | (into {}) 220 | (merge specs))) 221 | 222 | (defn request-pending [log specs ctx ks all-exchanges cache] 223 | (when-let [pending (seq (find-pending specs ctx ks all-exchanges))] 224 | (a/go 225 | (let [chans (map #(make-request log (get specs %) ctx % all-exchanges cache) pending) 226 | {:keys [result exchanges]} (a/> (mapcat (comp :params specs) ks) 235 | (remove #(contains? ctx %)) 236 | (filter #(contains? specs %)) 237 | (remove (set ks)) 238 | seq)] 239 | (a/go {:specs specs 240 | :ctx ctx 241 | :ks (set (concat ks unresolved)) 242 | :exchanges exchanges}))) 243 | 244 | (defn unknown-host? [{:keys [exception]}] 245 | (and exception 246 | #?(:clj (instance? java.net.UnknownHostException exception) 247 | :cljs false))) 248 | 249 | (defn explain-failed-request [specs ctx exchanges k] 250 | (let [spec (k specs) 251 | reqs (requests-for exchanges k)] 252 | (cond 253 | (not (params-available? ctx spec)) 254 | {:courier.error/reason :courier.error/missing-params 255 | :courier.error/data (remove #(contains? ctx %) (:params spec))} 256 | 257 | (unknown-host? (last reqs)) 258 | {:courier.error/reason :courier.error/unknown-host 259 | :courier.error/data {:req (:req (last reqs))}} 260 | 261 | (= (:max-retries (:retry (last reqs)) 0) 0) 262 | {:courier.error/reason :courier.error/request-failed 263 | :courier.error/data (:res (last reqs))} 264 | 265 | (< (:max-retries (:retry (last reqs)) 0) (count reqs)) 266 | {:courier.error/reason :courier.error/retries-exhausted 267 | :courier.error/data (merge {:attempts (count (requests-for exchanges k)) 268 | :last-res (-> reqs last :res)} 269 | (select-keys (:retry (last reqs)) [:max-retries]))} 270 | 271 | ;; Shouldn't happen (tm) 272 | :default {:courier.error/reason :courier.error/unknown}))) 273 | 274 | (defn dep? [v] 275 | (and (map? v) 276 | (::req v))) 277 | 278 | (defn extract-specs [ctx] 279 | (->> ctx 280 | (filter (comp dep? second)) 281 | (map (fn [[k v]] [k (assoc (::req v) ::select (::select v))])) 282 | (into {}))) 283 | 284 | (defn make-requests [{:keys [cache params]} specs] 285 | (assert (or (nil? cache) (satisfies? cache/Cache cache)) 286 | "cache does not implement the courier.cache/Cache protocol") 287 | (let [log (a/chan 512) 288 | param-specs (extract-specs params) 289 | ks (keys specs)] 290 | (a/go-loop [specs (merge specs param-specs) 291 | ctx (apply dissoc params (keys param-specs)) 292 | ks ks 293 | all-exchanges []] 294 | (if-let [ch (or (lookup-cache log specs ctx ks all-exchanges cache) 295 | (request-pending log specs ctx ks all-exchanges cache) 296 | (expand-selection log specs ctx ks all-exchanges))] 297 | (let [{:keys [specs ctx ks exchanges]} (a/ (explain-failed-request specs ctx all-exchanges k) 302 | (assoc :path k)))) 303 | (a/close! log)))) 304 | log)) 305 | 306 | (defn siphon!! [in out] 307 | (a/> (keys res) 326 | (filter (comp #{"courier.cache"} namespace)) 327 | (select-keys res)))] 328 | (merge 329 | (select-keys (:res res) [:status :headers :body]) 330 | {:success? (boolean (:success? res)) 331 | :log (->> reqs 332 | (remove (comp #{::store-in-cache} :event)) 333 | (map #(dissoc % :path)))} 334 | (when (= ::cache-hit (:event res)) 335 | {:cache-status (assoc cache-status :cache-hit? true)}) 336 | (when (= ::store-in-cache (:event res)) 337 | {:cache-status (assoc cache-status :stored-in-cache? true)}) 338 | (when-let [exceptions (seq (filter (comp #{::exception} :event) events))] 339 | {:exceptions (map #(dissoc % :event :path) exceptions)}) 340 | (when (and (= ::failed (:event res)) 341 | (= :courier.error/missing-params (:courier.error/reason res))) 342 | (when-let [possibly-misplaced (some (set (keys opt)) (:courier.error/data res))] 343 | {:hint (str "Make sure you pass parameters to your request as `:params` " 344 | "in the options map, not directly in the map, e.g.: " 345 | "{:params {" possibly-misplaced " " (get opt possibly-misplaced) "}}, not " 346 | "{" possibly-misplaced " " (get opt possibly-misplaced) "}")}))))) 347 | 348 | (defn request [spec & [opt]] 349 | (->> {::req spec} 350 | (make-requests opt) 351 | collect!! 352 | (prepare-full-result-for ::req opt))) 353 | 354 | (defn request-with-log [spec & [opt]] 355 | (let [ch (a/chan 512)] 356 | [ch 357 | (a/go 358 | (->> (siphon!! (make-requests opt {::req spec}) ch) 359 | (prepare-full-result-for ::req opt)))])) 360 | 361 | (defn retry-fn [{:keys [delays refresh refresh-fn] :as opt}] 362 | (let [retryable? (:retryable? opt retryable?) 363 | retries (or (:retries opt) 0) 364 | refresh-fn (or refresh-fn 365 | (when refresh (constantly refresh)) 366 | (constantly nil))] 367 | (fn [{:keys [num-attempts] :as exchange}] 368 | (when (retryable? exchange) 369 | (merge 370 | {:retry? (<= (:num-attempts exchange) retries) 371 | :max-retries retries} 372 | (when delays 373 | {:delay (get delays (dec (min num-attempts (count delays))))}) 374 | (when-let [refresh (refresh-fn exchange)] 375 | {:refresh refresh})))))) 376 | 377 | (defn cache-fn [{:keys [ttl ttl-fn cacheable?] :as opt}] 378 | (let [ttl-fn (or ttl-fn (when ttl (constantly ttl))) 379 | cacheable? (or cacheable? (constantly true))] 380 | (fn [{:keys [res] :as exchange}] 381 | (when (and (cacheable? exchange) ttl-fn) 382 | (let [ttl (ttl-fn exchange)] 383 | {:cache? true 384 | :expires-at (time/add-millis (time/now) ttl) 385 | :ttl ttl}))))) 386 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Courier 2 | 3 | Courier is a high-level HTTP client for Clojure and ClojureScript that improves 4 | the robustness of your HTTP communications using API-specific information that 5 | goes beyond the HTTP spec - "oh this API throws a 500 on every fifth request on 6 | Sundays, just give it another try". 7 | 8 | Courier offers: 9 | 10 | - Caching 11 | - Retries 12 | - Inter-request dependencies 13 | 14 | As an example, you can declare that a request requires an OAuth token, and 15 | Courier will either find one in the cache, or make a separate request to fetch 16 | one (**or** refresh the cached one if using it implies it's expired), making 17 | sure to retry failures and handle all the nitty-gritty intricacies of this 18 | interaction for you. 19 | 20 | Courier's caching and retry mechanisms do not expect all the HTTP endpoints in 21 | the world to be perfectly spec-compliant, and allows you to tune them to 22 | out of band information about the APIs you're working with. 23 | 24 | ## Hello, Courier 25 | 26 | At its most basic, you use Courier close to how you would use 27 | [clj-http](https://github.com/dakrone/clj-http) or 28 | [cljs-http](https://github.com/r0man/cljs-http) - in fact, it uses those two 29 | libraries under the hood: 30 | 31 | ```clj 32 | (require '[courier.http :as http]) 33 | 34 | (def res 35 | (http/request 36 | {:req {:method :get 37 | :url "http://example.com/api/demo"} ;; 1 38 | :retry-fn (http/retry-fn {:retries 2})})) ;; 2 39 | 40 | (when (:success? res) 41 | (prn (:body res))) 42 | ``` 43 | 44 | 1. The `:req` map is passed on to `clj-http` or `cljs-http`. 45 | 2. The request should be retried two times, if it fails _for any reason_: 46 | network errors, any non-2xx response. This defies the HTTP spec, but anyone 47 | who has used a few APIs in the wild know that they're not all 100% spec 48 | compliant. You can add nuance to this decision with 49 | `:retry-fn`, see below. 50 | 51 | A slightly more involved example can better highlight Courier's strengths over 52 | more low-level HTTP clients: 53 | 54 | ```clj 55 | (require '[courier.http :as http] 56 | '[courier.cache :as courier-cache] 57 | '[clojure.core.cache :as cache]) 58 | 59 | (def spotify-token-request 60 | {:params [:client-id :client-secret] ;; 1 61 | 62 | :req-fn 63 | (fn [{:keys [client-id client-secret]}] ;; 2 64 | {:method :post 65 | :as :json 66 | :url "https://accounts.spotify.com/api/token" 67 | :form-params {:grant_type "client_credentials"} 68 | :basic-auth [client-id client-secret]}) 69 | 70 | :retry-fn (http/retry-fn {:retries 2}) ;; 3 71 | 72 | :cache-fn (http/cache-fn 73 | {:ttl-fn #(* 1000 (-> % :res :body :expires_in))})}) ;; 4 74 | 75 | (def spotify-playlist-request 76 | {:params [:token :playlist-id] 77 | :lookup-params [:playlist-id] ;; 5 78 | 79 | :req-fn (fn [{:keys [token playlist-id]}] 80 | {:method :get 81 | :url (format "https://api.spotify.com/playlists/%s" 82 | playlist-id) 83 | :oauth-token (:access_token token)}) 84 | 85 | :retry-fn (http/retry-fn 86 | {:retries 2 87 | :refresh-fn #(when (= 403 (-> % :res :status)) ;; 6 88 | [:token])}) 89 | 90 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) ;; 7 91 | 92 | (def cache (atom (cache/lru-cache-factory {} :threshold 8192))) ;; 8 93 | 94 | (http/request ;; 9 95 | spotify-playlist-request 96 | {:cache (courier-cache/create-atom-map-cache cache) ;; 10 97 | :params {:client-id "my-api-client" ;; 11 98 | :client-secret "api-secret" 99 | :playlist-id "3abdc" 100 | :token {::http/req spotify-token-request ;; 12 101 | ::http/select (comp :access_token :body)}}}) ;; 13 102 | ``` 103 | 104 | 1. `:params` informs Courier of which parameters are required to make this 105 | request. 106 | 2. Specifying the details of the request with a function instead of an inline 107 | map allows us to defer and externalize details. The function will be passed 108 | the parameters named in `:params`. 109 | 3. Retry any failures up to two times. 110 | 4. Cache the response for as long as specified by the `:expires_in` key in the 111 | body of the request. Multiple the number of seconds with 1000. 112 | 5. `:lookup-params` determines what parameters are required to look for a 113 | previously cached response. Since the `:token` parameter is omitted from 114 | `:lookup-params`, the token request can be skipped completely when the 115 | playlist is cached. 116 | 6. When retrying a request, we can tell Courier to refresh some parameters. In 117 | this case, if the response was a 403, we will retry the request with a fresh 118 | token. 119 | 7. Cache playlists for a fixed 10 seconds. 120 | 8. Courier provides a caching protocol and comes with an implementation for 121 | atoms with map-like data structures, like the ones provided by 122 | `clojure.core.cache`. 123 | 9. Make the request(s) and return the result of the playlist request. 124 | 10. Reifies the `courier.cache/Cache` protocol for an atom with a map-like data 125 | structure. 126 | 11. Provide inline values for deferred parameters `:client-id` and 127 | `:client-secret`. 128 | 12. `:token` is provided as another request. If the playlist request is not 129 | cached, Courier will first request a token (including retries, checking for 130 | a cached token, etc), then request playlists. If the playlist request fails 131 | with a 403, Courier will fetch a new token and retry the playlist request. 132 | 13. When passing the result of the token request to the playlist request, pass 133 | `:access_token` from the response's `:body`. In other words, in the playlist 134 | request's `:req-fn`, `:token` will be the OAuth token string. 135 | 136 | The `result` map returned from `http/request` contains `:status`, `:headers`, 137 | and `:body`, just like a normal HTTP response map. Because a Courier request can 138 | result in multiple request/response pairs (e.g. if retries are required), the 139 | map also contains other keys, see [the result map](#the-result-map). 140 | 141 | ## Table of contents 142 | 143 | - [Install](#install) 144 | - [Parameters and dependencies](#parameters-and-dependencies) 145 | - [The result map](#the-result-map) 146 | - [Retries](#retry-on-failure) 147 | - [Caching](#caching) 148 | - [Events](#events) 149 | - [Reference](#reference) 150 | - [Changelog](#changelog) 151 | 152 | ## Install 153 | 154 | Courier is a stable library - it will never change it's public API in breaking 155 | way, and will never (intentionally) introduce other breaking changes. 156 | 157 | With tools.deps: 158 | 159 | ```clj 160 | cjohansen/courier {:mvn/version "2021.01.26"} 161 | ``` 162 | 163 | With Leiningen: 164 | 165 | ```clj 166 | [cjohansen/courier "2021.01.26"] 167 | ``` 168 | 169 | **NB!** Please do not be alarmed if the version/date seems "old" - this just 170 | means that no bugs have been discovered in a while. Courier is largely 171 | feature-complete, and I expect to only rarely add to its feature set. 172 | 173 | ## Parameters and dependencies 174 | 175 | Many HTTP APIs require authentication with an OAuth 2.0 token. This means we 176 | first have to make an HTTP request for a token, then request the resource 177 | itself. Courier allows you to explicitly model this dependency. 178 | 179 | First, define the request for the token. To externalize credentials, provide a 180 | function to `:req-fn`, and declare the function's dependencies with `:params`: 181 | 182 | ```clj 183 | (def spotify-token-request 184 | {:params [:client-id :client-secret] 185 | :req-fn 186 | (fn [{:keys [client-id client-secret]}] 187 | {:url "https://accounts.spotify.com/api/token" 188 | :form-params {:grant_type "client_credentials"} 189 | :basic-auth [client-id client-secret]})}) 190 | ``` 191 | 192 | Where do the params come from? You can pass them in as you make the request: 193 | 194 | ```clj 195 | (require '[courier.http :as http]) 196 | 197 | (http/request 198 | spotify-token-request 199 | {:params {:client-id "username" 200 | :client-secret "password"}}) 201 | ``` 202 | 203 | Then define a request that uses an oauth token: 204 | 205 | ```clj 206 | (def spotify-playlist-request 207 | {:params [:token :playlist-id] 208 | :lookup-params [:playlist-id] 209 | :req-fn 210 | (fn [{:keys [token playlist-id]}] 211 | {:method :get 212 | :url (format "https://api.spotify.com/playlists/%s" 213 | playlist-id) 214 | :oauth-token token})}) 215 | ``` 216 | 217 | We _could_ manually piece the two together: 218 | 219 | ```clj 220 | (require '[courier.http :as http]) 221 | 222 | (def token 223 | (http/request 224 | spotify-token-request 225 | {:params {:client-id "username" 226 | :client-secret "password" 227 | :playlist-id "4abdc"}})) 228 | 229 | (http/request 230 | spotify-playlist-request 231 | {:params {:token (:access_token (:body token))}}) 232 | ``` 233 | 234 | Even better, let Courier manage the dependency: 235 | 236 | ```clj 237 | (require '[courier.http :as http]) 238 | 239 | (http/request 240 | spotify-playlist-request 241 | {:params {:token {::http/req spotify-token-request 242 | ::http/select (comp :access_token :body)}}}) 243 | 244 | ``` 245 | 246 | When Courier knows about the dependency, it can provide a higher level of 247 | service, especially if we also give it a means to cache results: 248 | 249 | - If requesting the token fails for some reason, retry it before requesting the 250 | playlist resource 251 | - Don't request a new token if we have one in the cache 252 | 253 | Additionally, if the cached token expires and the playlist resource fails with a 254 | 401, Courier can automatically request a new token and retry the playlist 255 | resource with it: 256 | 257 | ```clj 258 | (def spotify-playlist-request 259 | {:params [:token :playlist-id] 260 | :lookup-params [:playlist-id] 261 | :req-fn 262 | (fn [{:keys [token playlist-id]}] 263 | {:method :get 264 | :url (format "https://api.spotify.com/playlists/%s" 265 | playlist-id) 266 | :oauth-token token}) 267 | :retry-fn (http/retry-fn 268 | {:retries 3 269 | :refresh-fn (fn [{:keys [req res]}] 270 | (when (= 401 (:status res)) 271 | [:token]))})}) 272 | ``` 273 | 274 | ## The result map 275 | 276 | The map returned by Courier contains the resulting data if successful, along 277 | with information about all requests leading up to it. It contains the following 278 | keys: 279 | 280 | - `:success?` - A boolean 281 | - `:log` - A list of maps describing each attempt 282 | - `:cache-status` - A map describing the cache status of the data 283 | - `:status` - The response status of the last response 284 | - `:headers` - The headers on the last response 285 | - `:body` - The body of the last response 286 | 287 | The `:log` list contains maps with the following keys: 288 | 289 | - `:req` - The request map 290 | - `:res` - The full response 291 | - `:retry` - The result of the `:retry-fn`, if set 292 | - `:cache` - The result of the `:cache-fn`, if set 293 | - `:event` - The courier event, one of 294 | - `:courier.http/response` 295 | - `:courier.http/cache-hit` 296 | - `:courier.http/failed` 297 | 298 | `:retry` and `:cache` are only available when relevant. 299 | 300 | The `:cache-status` map contains the folowing keys: 301 | 302 | - `:cache-hit?` - A boolean, `true` if the result was pulled from the cache 303 | - `:stored-in-cache?` - A boolean, `true` if the result was stored in the cache 304 | - `:cached-at` - A timestamp (epoch milliseconds) when the object was cached 305 | - `:expires-at` - A timestamp (epoch milliseconds) when the object expires from 306 | the cache. 307 | 308 | Specific cache implementations may add additional keys in this map, with further 309 | details about the cache entry, see individual implementations. 310 | 311 | ## Retry on failure 312 | 313 | HTTP requests can fail for any number of reasons. Sometimes problems go away if 314 | you try again. By default, Courier will consider any `GET` request retryable so 315 | long as you specify a number of retries: 316 | 317 | ```clj 318 | (require '[courier.http :as http]) 319 | 320 | (http/request 321 | {:req {:method :get 322 | :url "http://example.com/api/demo"} 323 | :retry-fn (http/retry-fn {:retries 2})}) 324 | ``` 325 | 326 | With this addition, the request will be retried 2 times before causing an 327 | error - _if the result can be retried_. As mentioned, Courier considers any 328 | `GET` request retryable. If you want more fine-grained control over this 329 | decision, pass a function with the `:retryable?` keyword: 330 | 331 | ```clj 332 | (require '[courier.http :as http]) 333 | 334 | (http/request 335 | {:req {:method :get 336 | :url "http://example.com/api/demo"} 337 | :retry-fn (http/retry-fn 338 | {:retries 2 339 | :retryable? #(= :get (-> % :req :method))})}) 340 | ``` 341 | 342 | The function is passed a map with both `:req` and `:res` to help inform its 343 | decision. If this function returns `false`, the request will not be retried even 344 | if all the `:retries` haven't been exhausted. 345 | 346 | ### When to retry? 347 | 348 | By default Courier will retry failing requests immediately. If desired, you can 349 | insert a pause between retries: 350 | 351 | ```clj 352 | (require '[courier.http :as http]) 353 | 354 | (http/request 355 | {:req {:method :get 356 | :url "http://example.com/api/demo"} 357 | :retry-fn (http/retry-fn 358 | {:retries 5 359 | :retryable? #(= :get (-> % :req :method)) 360 | :delays [100 250 500]})}) 361 | ``` 362 | 363 | This will cause the first retry to happen 100ms after the initial request, the 364 | second 250ms after the first, and the remaining ones will be spaced out by 365 | 500ms. If you want the same delay between each retry, specify a vector with a 366 | single number: `[100]`. 367 | 368 | ### What is a failure? 369 | 370 | By default, Courier leans on the underlying http client library to determine if 371 | a response is a success or not. In other words, anything with a 2xx response 372 | status is a success, everything else is a failure. If this does not agree with 373 | the reality of your particular service, you can provide a custom function to 374 | determine success: 375 | 376 | ```clj 377 | (require '[courier.http :as http]) 378 | 379 | (http/request 380 | {:req {:method :get 381 | :url "http://example.com/api/demo"} 382 | :success? #(= 200 (-> % :res :status))}) 383 | ``` 384 | 385 | ### Retries with refreshed dependencies 386 | 387 | If you are using [caching](#caching), it might not be worth retrying a fetch 388 | with the same (possibly stale) set of dependencies - you might need to refresh 389 | some or all of them. To continue the example of the authentication token, a 403 390 | response from a service could be worth retrying, but only with a fresh token. 391 | 392 | `:refresh-fn` takes a function that is passed a map of `:req` and `:res`, and 393 | can return a vector of parameters that should be refreshed before retrying this 394 | one: 395 | 396 | ```clj 397 | (require '[courier.http :as http]) 398 | 399 | (def spotify-playlist-request 400 | {:params [:token :playlist-id] 401 | :req-fn (fn [{:keys [token playlist-id]}] 402 | {:method :get 403 | :url (format "https://api.spotify.com/playlists/%s" 404 | playlist-id) 405 | :oauth-token (:access_token token)}) 406 | :retry-fn (http/retry-fn 407 | {:retries 2 408 | :refresh-fn #(when (= 403 (-> % :res :status)) 409 | [:token])})}) 410 | ``` 411 | 412 | If the response to this request is an HTTP 403, Courier will grab a new `:token` 413 | by refreshing that request (bypassing the cache) and then try again. 414 | 415 | ## Caching 416 | 417 | Courier caching is provided by the `courier.cache/Cache` protocol, which defines 418 | the following two functions: 419 | 420 | ```clj 421 | (defprotocol Cache 422 | (lookup [_ spec params]) 423 | (put [_ spec params res])) 424 | ``` 425 | 426 | `spec` is the full map passed to `courier.http/request`. `params` is a map of 427 | all the lookup params - this would be the keys named in `:lookup-params`, if 428 | set, or `:params`. If neither of these are available, `params` will be empty. 429 | 430 | `lookup` attempts to load a cached response for the request. If this function 431 | returns a non-nil value, it should be a map of `{req, res}`, and `put` will 432 | never be called. 433 | 434 | If the value does not exist in the cache, the request will be made, and if 435 | successful, `put` will be called with the result. 436 | 437 | A reified instance of a cache can be passed to `http/request` as `:cache`: 438 | 439 | ```clj 440 | (require '[courier.http :as http] 441 | '[courier.cache :as courier-cache] 442 | '[clojure.core.cache :as cache]) 443 | 444 | (def cache (atom (cache/lru-cache-factory {} :threshold 8192))) 445 | 446 | (http/request 447 | spotify-playlist-request 448 | {:cache (courier-cache/create-atom-map-cache cache) 449 | :params {:client-id "my-api-client" 450 | :client-secret "api-secret" 451 | :playlist-id "3abdc" 452 | :token {::http/req spotify-token-request 453 | ::http/select (comp :access_token :body)}}}) 454 | ``` 455 | 456 | ### Lookup params 457 | 458 | Lookup params can be used in place of the full request to make more efficient 459 | use of the cache. Consider the playlist request from before: 460 | 461 | ```clj 462 | (def spotify-playlist-request 463 | {:params [:token :playlist-id] 464 | :req-fn (fn [{:keys [token playlist-id]}] 465 | {:method :get 466 | :url (format "https://api.spotify.com/playlists/%s" 467 | playlist-id) 468 | :oauth-token token}) 469 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 470 | ``` 471 | 472 | When the `:token` parameter is provided by another request, Courier might have 473 | to request a token only to find a cached version of the playlist in the cache. 474 | If the playlist is already cached, there is no need for a token. Constructing a 475 | cache key from the `:lookup-params` only, Courier will skip the token request if 476 | the playlist is cached: 477 | 478 | ```clj 479 | (def spotify-playlist-request 480 | {:params [:token :playlist-id] 481 | :lookup-params [:playlist-id] 482 | :req-fn (fn [{:keys [token playlist-id]}] 483 | {:method :get 484 | :url (format "https://api.spotify.com/playlists/%s" 485 | playlist-id) 486 | :oauth-token token}) 487 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 488 | ``` 489 | 490 | With `:lookup-params` in place, `courier.cache/lookup` won't receive the full 491 | request, only the spec and the cache parameters (the playlist ID). The `:req-fn` 492 | can be used to identify the request, but it usually won't do so in a 493 | human-friendly manner. A better approach is to include `:lookup-id` in the cache 494 | spec. `courier.cache/cache-key` can use this to construct a short, 495 | human-friendly cache key: 496 | 497 | ```clj 498 | (def spotify-playlist-request 499 | {:params [:token :playlist-id] 500 | :lookup-params [:playlist-id] 501 | :lookup-id :spotify-playlist-request 502 | :req-fn (fn [{:keys [token playlist-id]}] 503 | {:method :get 504 | :url (format "https://api.spotify.com/playlists/%s" 505 | playlist-id) 506 | :oauth-token token}) 507 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 508 | ``` 509 | 510 | With this spec, the "atom map" cache mentioned earlier will cache a request for 511 | the playlist with id `"3b5045a0-05fc-4d7f-8b61-9c6d37ab90e6"` under the 512 | following key: 513 | 514 | ```clj 515 | (def cache-key 516 | [:spotify-playlist-request 517 | {:playlist-id "3b5045a0-05fc-4d7f-8b61-9c6d37ab90e6"}]) 518 | 519 | (get @cache cache-key) ;; Playlist response 520 | ``` 521 | 522 | #### Surgical lookup params 523 | 524 | Sometimes your requests will use unwieldy data structures like configuration 525 | maps as parameters. This could lead to very large cache keys, or worse - 526 | sensitive data like credentials being used as cache keys. To avoid this, a 527 | lookup param can be expressed as a vector, which will be used to `get-in` the 528 | named parameter. 529 | 530 | Let's parameterize the Spotify API host using a configuration map: 531 | 532 | ```clj 533 | (def spotify-playlist-request 534 | {:lookup-id :spotify-playlist-request 535 | :params [:token :config :playlist-id] 536 | :req-fn (fn [{:keys [token config playlist-id]}] 537 | {:method :get 538 | :url (format "https://%s/playlists/%s" 539 | (:spotify-host config) 540 | playlist-id) 541 | :oauth-token (:access_token token)}) 542 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 543 | ``` 544 | 545 | In order to include only the relevant key in the cache key, 546 | `:lookup-params` can be expressed like so: 547 | 548 | ```clj 549 | (def spotify-playlist-request 550 | {:lookup-id :spotify-playlist-request 551 | :params [:token :config :playlist-id] 552 | :lookup-params [[:config :spotify-host] :playlist-id] 553 | :req-fn (fn [{:keys [token config playlist-id]}] 554 | {:method :get 555 | :url (format "https://%s/playlists/%s" 556 | (:spotify-host config) 557 | playlist-id) 558 | :oauth-token (:access_token token)}) 559 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 560 | ``` 561 | 562 | Which will result in the following cache key for the atom map caches: 563 | 564 | ```clj 565 | (def cache-key 566 | [:spotify-playlist-request 567 | {:config {:spotify-host "api.spotify.com"} 568 | :playlist-id "3b5045a0-05fc-4d7f-8b61-9c6d37ab90e6"}]) 569 | ``` 570 | 571 | #### Manipulating lookup params 572 | 573 | Some endpoints do not take any identifying parameters other than the token, and 574 | returns content belonging to the user for whom the token is issued. If the token 575 | contains information that's stable across tokens, you can pass the lookup 576 | parameters through a transforming function before looking up the value in the 577 | cache. In this case you will always need a token, but maybe you won't need to 578 | make the data request over again. 579 | 580 | Let's fetch all the playlists belonging to a user. This resource only uses the 581 | token to identify the user. 582 | 583 | ```clj 584 | (def spotify-playlists-request 585 | {:lookup-id :spotify-playlists 586 | :params [:token :config] 587 | :lookup-params [[:config :spotify-host] :token] 588 | :req-fn (fn [{:keys [token config]}] 589 | {:method :get 590 | :url (format "https://%s/playlists/" 591 | (:spotify-host config)) 592 | :oauth-token (:access_token token)}) 593 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 594 | ``` 595 | 596 | This caches with the token, which is no good. We can add 597 | `:prepare-lookup-params` to extract only the relevant bits: 598 | 599 | ```clj 600 | (defn base64-decode [s] 601 | (.decode (java.util.Base64/getDecoder) s)) 602 | 603 | (defn decode-jwt [token] 604 | (-> (clojure.string/split token #"\.") 605 | second 606 | base64-decode 607 | String. 608 | (cheshire.core/parse-string keyword))) 609 | 610 | (def spotify-playlists-request 611 | {:lookup-id :spotify-playlists 612 | :params [:token :config] 613 | :lookup-params [[:config :spotify-host] :token] 614 | :prepare-lookup-params (fn [params] 615 | {:host (get-in params [:config :spotify-host]) 616 | :user-id (:userId (decode-jwt (:token params)))}) 617 | :req-fn (fn [{:keys [token config]}] 618 | {:method :get 619 | :url (format "https://%s/playlists/" 620 | (:spotify-host config)) 621 | :oauth-token (:access_token token)}) 622 | :cache-fn (http/cache-fn {:ttl (* 10 1000)})}) 623 | ``` 624 | 625 | Which will result in the following cache key for the atom map caches: 626 | 627 | ```clj 628 | (def cache-key 629 | [:spotify-playlists 630 | {:host "api.spotify.com" 631 | :user-id "3b5045a0-05fc-4d7f-8b61-9c6d37ab90e6"}]) 632 | ``` 633 | 634 | ### Atom map cache 635 | 636 | The atom map cache gives you a quick and easy in-memory cache for your HTTP 637 | requests. Stick a map, or a map-like data structure, in an atom, and off you go. 638 | [clojure.core.cache](https://github.com/clojure/core.cache) has lots of nice 639 | caches that go well with this Courier cache: 640 | 641 | ```clj 642 | (require '[courier.http :as http] 643 | '[courier.cache :refer [create-atom-map-cache]] 644 | '[clojure.core.cache :as cache]) 645 | 646 | (def cache (atom (cache/lru-cache-factory {} :threshold 8192))) 647 | 648 | (http/request 649 | spotify-playlist-request 650 | {:cache (create-atom-map-cache cache) 651 | :params {:client-id "my-api-client" 652 | :client-secret "api-secret" 653 | :playlist-id "3abdc" 654 | :token {::http/req spotify-token-request 655 | ::http/select (comp :access_token :body)}}}) 656 | ``` 657 | 658 | The atom map cache adds a `:courier.cache/cache-key` to the `:cache-status` map, 659 | indicating the key under which the result is stored in the cache. 660 | 661 | ### File cache 662 | 663 | The file cache stores responses on disk. Give it a directory, and off you go. 664 | 665 | ```clj 666 | (require '[courier.http :as http] 667 | '[courier.cache :as cache]) 668 | 669 | (http/request 670 | spotify-playlist-request 671 | {:cache (cache/create-file-cache {:dir "/tmp/courier"}) 672 | :params {:client-id "my-api-client" 673 | :client-secret "api-secret" 674 | :playlist-id "3abdc" 675 | :token {::http/req spotify-token-request 676 | ::http/select (comp :access_token :body)}}}) 677 | ``` 678 | 679 | The file cache adds a `:courier.cache/file-name` key to the `:cache-status` map, 680 | containing the full path on disk to the file storing the cached response. 681 | Cache files are stored in files with UUID names, sharded by the first two 682 | characters, to avoid too many files in a single directory. 683 | 684 | ### Redis cache 685 | 686 | To cache Courier responses in Redis you must "bring your own" 687 | [Carmine](https://github.com/ptaoussanis/carmine): 688 | 689 | ```clj 690 | com.taoensso/carmine {:mvn/version "3.1.0"} 691 | ``` 692 | 693 | Then create a cache with a pool spec: 694 | 695 | ```clj 696 | (require '[courier.http :as http] 697 | '[courier.cache :as cache] 698 | '[taoensso.carmine.connections :as cc]) 699 | 700 | (def pool-spec 701 | (let [conn-spec {:spec {:uri "redis://localhost"}} 702 | [pool conn] (cc/pooled-conn conn-spec) 703 | pool-spec (assoc conn-spec :pool pool)] 704 | (.release-conn pool conn) 705 | pool-spec)) 706 | 707 | (http/request 708 | spotify-playlist-request 709 | {:cache (cache/create-redis-cache pool-spec) 710 | :params {:client-id "my-api-client" 711 | :client-secret "api-secret" 712 | :playlist-id "3abdc" 713 | :token {::http/req spotify-token-request 714 | ::http/select (comp :access_token :body)}}}) 715 | ``` 716 | 717 | ## Events 718 | 719 | Even though `(courier.http/request spec)` looks like a single request, it can 720 | actually spawn multiple requests to several endpoints. Most of the time we're 721 | only interested in the end result, in which case `request` is just what the 722 | doctor ordered. 723 | 724 | Sometimes we want more insight into the network layer of our application. Maybe 725 | you want to log each request on the way out and the response coming back. 726 | Courier does all its heavy lifting with `courier.http/make-requests`, but there 727 | is another medium-level abstraction on top of it: `request-with-log`. This 728 | function works just like `request`, except it also gives you a `core.async` 729 | channel that emits events as they occur: 730 | 731 | ```clj 732 | (require '[courier.http :as http] 733 | '[clojure.core.async :as a]) 734 | 735 | (let [[log-ch result-ch] 736 | (http/request-with-log 737 | spotify-playlist-request 738 | {:cache (courier-cache/create-atom-map-cache cache) 739 | :params {:client-id "my-api-client" 740 | :client-secret "api-secret" 741 | :playlist-id "3abdc" 742 | :token {::http/req spotify-token-request 743 | ::http/select (comp :access_token :body)}}})] 744 | 745 | ;; The result channel emits the full result, as returned by `request`: 746 | (a/go (a/ {:ok? true} 793 | 794 | ``` 795 | 796 | ## Reference 797 | 798 | ### `(courier.http/request spec opt)` 799 | 800 | `spec` is a map of the following keys: 801 | 802 | - `:req` - Inline request map 803 | - `:req-fn` - A function that computes the request map. Will be called with the 804 | parameters named by the `:params` key. 805 | - `:params` - The parameters to pass to `:req-fn`. This may contain references 806 | to other requests - if it does those will be resolved before `:req-fn` is 807 | called and this request is carried out. 808 | - `:lookup-params` - The parameters required to look this request up in the 809 | cache. Specifying this has two benefits: avoid using sensitive values like 810 | credentials as cache keys, and avoid making dependent requests if a cached 811 | response is available. 812 | - `:success?` - A function that is passed a map of `{:req :res}` and that 813 | returns a boolean indicating if the response was a success. The default 814 | implementation returns `true` for any 2XX response. 815 | - `:retry-fn` - A function that is called if the response is not a success. It 816 | is passed a map of `{:req :res :num-attempts}` (the latter being the number of 817 | attempts already made at this request) and should return a map describing if 818 | and how the request may be retried, as described by the keys: 819 | - `:retry?` - If `true`, the request will be retried 820 | - `:delay` - A number of milliseconds to wait before trying again, optional. 821 | - `:refresh` - A list of `:params` that should be fetched anew, bypassing the 822 | cache, before trying this request again. 823 | - `:cache-fn` - A function that is called if the response is a success. It is 824 | passed a map of `{:req :res}` and should return a map describing if and how 825 | the response may be cached, as described by the keys: 826 | - `:cache?` - If `true`, the response will be cached _if a ttl is specified_. 827 | - `:expires-at` - An epoch millis at which the response expires from the cache. 828 | 829 | ### `(courier.http/cache-fn {:ttl :ttl-fn :cacheable?})` 830 | 831 | Returns a function that can be passed as `:cache-fn` to `courier.http/request`. 832 | Either set `:ttl` to a number of milliseconds to cache results, or set `:ttl` to 833 | a function that will return the number of milliseconds. If set, it will be 834 | passed a map of `:req` and `:res` to aid in the decision. 835 | 836 | If you only want to cache some request/response pairs, pass a function to 837 | `:cacheable?` which takes a map of `:req` and `:res` and returns `true` if the 838 | result is cacheable. 839 | 840 | ## Changelog 841 | 842 | ### 2021.01.26 843 | 844 | Specifically handle unknown host exceptions to make it clearer why a request 845 | fails. 846 | 847 | Do not report failed responses as "retries exhausted" when there was no 848 | retries - report as failed request instead. 849 | 850 | Include the last response on failures due to exhausted retries. 851 | 852 | ### 2021.01.20 853 | 854 | Fix bug where `:prepare-lookup-params` was called before all lookup params was 855 | available. 856 | 857 | Include cache retrieval events in the `:log` in meta data returned from 858 | `request`. Also include the event name on each entry in the log. 859 | 860 | ### 2021.01.19 861 | 862 | Added support for `:prepare-lookup-params`, which allows for transformin the 863 | lookup parameters before using them to store and retrieve items from the cache. 864 | 865 | Fix a bug where `POST` requests where not cached by default when `:cache-fn` was 866 | provided. 867 | 868 | ### 2020.12.12 869 | 870 | Initial release to Clojars (after being battle-tested in production as a git 871 | dependency). 872 | 873 | ## Acknowledgements 874 | 875 | This library is my second attempt at building a more robust tool for HTTP 876 | requests in Clojure. It is a smaller and more focused version of 877 | [Pharmacist](https://github.com/cjohansen/pharmacist), which I now consider a a 878 | flawed execution of a good idea. Courier is based on a bunch of helper functions 879 | I wrote for using Pharmacist primarily for HTTP requests. It attempts to present 880 | the most useful aspects of Pharmacist in a much less ceremonious API that is 881 | closer to traditional low-level HTTP libraries. 882 | 883 | As always, [Magnar Sveen](https://github.com/magnars) has been an important 884 | contributor to the design of the API. 885 | 886 | ## License 887 | 888 | Copyright © 2020-2021 Christian Johansen 889 | 890 | Distributed under the Eclipse Public License either version 1.0 or (at your 891 | option) any later version. 892 | -------------------------------------------------------------------------------- /test/courier/http_test.cljc: -------------------------------------------------------------------------------- 1 | (ns courier.http-test 2 | (:require [clojure.core.async :as a] 3 | [clojure.test :refer [deftest is]] 4 | [courier.cache :as cache] 5 | [courier.client :as client] 6 | [courier.http :as sut] 7 | [courier.time :as time])) 8 | 9 | ;; Helper tooling 10 | 11 | (def responses (atom {})) 12 | 13 | (defmethod client/request :default [req] 14 | (let [k [(:method req) (:url req)] 15 | res (or (first (get @responses k)) 16 | {:status 200 17 | :body {:request req}})] 18 | (swap! responses update k rest) 19 | res)) 20 | 21 | (defmacro with-responses [rs & body] 22 | `(do 23 | (reset! responses ~rs) 24 | (let [res# ~@body] 25 | (reset! responses {}) 26 | res#))) 27 | 28 | (defn time-events!! [ch] 29 | (a/ event :throwable .getMessage) (:source event)] 50 | :default [(summarize-req (:req event))]))) 51 | 52 | ;; Unit tests 53 | 54 | (deftest lookup-params-uses-all-params-by-defaut 55 | (is (= (sut/lookup-params {:params [:token :id]} 56 | {:token "ejY..." 57 | :id 42}) 58 | {:required [:token :id] 59 | :params {:token "ejY..." 60 | :id 42}}))) 61 | 62 | (deftest lookup-params-uses-only-lookup-params 63 | (is (= (sut/lookup-params {:params [:token :id] 64 | :lookup-params [:id]} 65 | {:token "ejY..." 66 | :id 42}) 67 | {:required [:id] 68 | :params {:id 42}}))) 69 | 70 | (deftest lookup-params-with-surgical-selections 71 | (is (= (sut/lookup-params {:params [:token :config :id] 72 | :lookup-params [[:config :host] :id]} 73 | {:token "ejY..." 74 | :id 42 75 | :config {:host "example.com" 76 | :client-id "..." 77 | :client-secret "..."}}) 78 | {:required [:config :id] 79 | :params {:id 42 80 | :config {:host "example.com"}}}))) 81 | 82 | (deftest lookup-params-with-no-params 83 | (is (= (sut/lookup-params {} 84 | {:token "ejY..." 85 | :id 42 86 | :config {:host "example.com" 87 | :client-id "..." 88 | :client-secret "..."}}) 89 | {:required [] 90 | :params nil}))) 91 | 92 | ;; make-requests tests 93 | 94 | (deftest emits-start-and-end-events 95 | (is (= (with-responses {[:get "http://example.com/"] 96 | [{:status 200 97 | :body {:yep "Indeed"}}]} 98 | (->> {:example {:req {:url "http://example.com/"}}} 99 | (sut/make-requests {}) 100 | sut/collect!!)) 101 | [{:event ::sut/request 102 | :path :example 103 | :req {:method :get 104 | :throw-exceptions false 105 | :url "http://example.com/"}} 106 | {:event ::sut/response 107 | :path :example 108 | :req {:method :get 109 | :throw-exceptions false 110 | :url "http://example.com/"} 111 | :res {:status 200 112 | :body {:yep "Indeed"}} 113 | :success? true}]))) 114 | 115 | (deftest determines-failure-with-custom-fn 116 | (is (false? (with-responses {[:get "http://example.com/"] 117 | [{:status 200 118 | :body {:yep "Indeed"}}]} 119 | (->> {:example {:req {:url "http://example.com/"} 120 | :success? #(= 201 (-> % :res :status))}} 121 | (sut/make-requests {}) 122 | sut/collect!! 123 | second 124 | :success?))))) 125 | 126 | (deftest determines-success-with-custom-fn 127 | (is (true? (with-responses {[:get "http://example.com/"] 128 | [{:status 301 129 | :body {:yep "Indeed"}}]} 130 | (->> {:example {:req {:url "http://example.com/"} 131 | :success? #(= 301 (-> % :res :status))}} 132 | (sut/make-requests {}) 133 | sut/collect!! 134 | second 135 | :success?))))) 136 | 137 | (deftest is-failure-when-success-fn-throws 138 | (is (false? (with-responses {[:get "http://example.com/"] 139 | [{:status 201 140 | :body {:yep "Indeed"}}]} 141 | (->> {:example {:req {:url "http://example.com/"} 142 | :success? (fn [_] (throw (ex-info "Oops!" {})))}} 143 | (sut/make-requests {}) 144 | sut/collect!! 145 | (drop 2) 146 | first 147 | :success?))))) 148 | 149 | (deftest emits-retry-event 150 | (is (= (with-responses {[:get "http://example.com/"] 151 | [{:status 500} 152 | {:status 200 :body {:ok? true}}]} 153 | (->> {:example {:req {:url "http://example.com/"} 154 | :retry-fn (sut/retry-fn {:retries 2})}} 155 | (sut/make-requests {}) 156 | sut/collect!! 157 | (map summarize-event))) 158 | [[::sut/request :example [:get "http://example.com/"]] 159 | [::sut/response :example [500]] 160 | [::sut/request :example [:get "http://example.com/"]] 161 | [::sut/response :example [200 {:ok? true}]]]))) 162 | 163 | (deftest retries-with-delays 164 | (is (<= 55 165 | (with-responses {[:get "http://example.com/"] 166 | [{:status 500} 167 | {:status 500} 168 | {:status 500} 169 | {:status 500} 170 | {:status 200 :body {:ok? true}}]} 171 | (->> {:example {:req {:url "http://example.com/"} 172 | :retry-fn (sut/retry-fn {:retries 4 173 | :delays [5 10 20]})}} 174 | (sut/make-requests {}) 175 | time-events!! 176 | (map :elapsed) 177 | (reduce + 0))) 178 | 79))) 179 | 180 | (deftest bails-after-the-specified-number-of-retries 181 | (is (= (with-responses {[:get "http://example.com/"] 182 | [{:status 500} 183 | {:status 500} 184 | {:status 500} 185 | {:status 500} 186 | {:status 200 :body {:ok? true}}]} 187 | (->> {:example {:req {:url "http://example.com/"} 188 | :retry-fn (sut/retry-fn {:retries 1 189 | :delays [5 10 20]})}} 190 | (sut/make-requests {}) 191 | sut/collect!! 192 | (map summarize-event))) 193 | [[::sut/request :example [:get "http://example.com/"]] 194 | [::sut/response :example [500]] 195 | [::sut/request :example [:get "http://example.com/"]] 196 | [::sut/response :example [500]] 197 | [::sut/failed :example 198 | :courier.error/retries-exhausted 199 | {:max-retries 1 200 | :attempts 2 201 | :last-res {:status 500}}]]))) 202 | 203 | (deftest passes-named-parameters-to-req-fn 204 | (is (= (->> {:example {:req-fn (fn [params] 205 | {:url "http://example.com/" 206 | :params params}) 207 | :params [:token]}} 208 | (sut/make-requests 209 | {:params 210 | {:token "ejY-secret-..." 211 | :other "Stuff"}}) 212 | sut/collect!! 213 | second 214 | :res 215 | :body) 216 | {:request 217 | {:url "http://example.com/" 218 | :throw-exceptions false 219 | :params {:token "ejY-secret-..."} 220 | :method :get}}))) 221 | 222 | (deftest cannot-make-request-without-required-data 223 | (is (= (->> {:example {:req-fn (fn [params] 224 | {:url "http://example.com/" 225 | :params params}) 226 | :params [:token :spoken]}} 227 | (sut/make-requests 228 | {:params 229 | {:token "ejY-secret-..." 230 | :other "Stuff"}}) 231 | sut/collect!!) 232 | [{:event ::sut/failed 233 | :path :example 234 | :courier.error/reason :courier.error/missing-params 235 | :courier.error/data [:spoken]}]))) 236 | 237 | (deftest makes-dependent-request-first 238 | (is (= (with-responses {[:post "http://example.com/security/"] 239 | [{:status 200 :body {:token "ejY..."}}]} 240 | (->> {:example {:req-fn (fn [{:keys [token]}] 241 | {:url "http://example.com/" 242 | :headers {"Authorization" (str "Bearer " token)}}) 243 | :params [:token]}} 244 | (sut/make-requests 245 | {:params 246 | {:token {::sut/req {:req {:method :post 247 | :url "http://example.com/security/"}} 248 | ::sut/select (comp :token :body)} 249 | :other "Stuff"}}) 250 | sut/collect!! 251 | last 252 | :res)) 253 | {:status 200 254 | :body {:request 255 | {:url "http://example.com/" 256 | :throw-exceptions false 257 | :headers {"Authorization" "Bearer ejY..."} 258 | :method :get}}}))) 259 | 260 | (deftest makes-multiple-dependent-requests 261 | (is (= (with-responses {[:post "http://example.com/security/"] 262 | [{:status 200 :body {:clue "ejY"}}] 263 | 264 | [:get "http://example.com/super-security/ejY"] 265 | [{:status 200 :body {:token "111"}}]} 266 | (->> {:example {:req-fn (fn [{:keys [token]}] 267 | {:url "http://example.com/" 268 | :headers {"Authorization" (str "Bearer " token)}}) 269 | :params [:token]}} 270 | (sut/make-requests 271 | {:params 272 | {:clue {::sut/req {:req {:method :post 273 | :url "http://example.com/security/"}} 274 | ::sut/select (comp :clue :body)} 275 | :token {::sut/req {:req-fn (fn [{:keys [clue]}] 276 | {:method :get 277 | :url (str "http://example.com/super-security/" clue)}) 278 | :params [:clue]} 279 | ::sut/select (comp :token :body)} 280 | :other "Stuff"}}) 281 | sut/collect!! 282 | last 283 | :res)) 284 | {:status 200 285 | :body {:request 286 | {:url "http://example.com/" 287 | :throw-exceptions false 288 | :headers {"Authorization" "Bearer 111"} 289 | :method :get}}}))) 290 | 291 | (deftest loads-result-from-cache 292 | (is (= (let [cache (atom {[::sut/req {:method :get 293 | :url "http://example.com/"}] 294 | {:req {:method :get 295 | :url "http://example.com"} 296 | :res {:status 200 297 | :body "Oh yeah!"}}})] 298 | (->> {:example {:req {:url "http://example.com/"}}} 299 | (sut/make-requests {:cache (cache/create-atom-map-cache cache)}) 300 | sut/collect!!)) 301 | [{:req {:method :get 302 | :url "http://example.com"} 303 | :res {:status 200 304 | :body "Oh yeah!"} 305 | :path :example 306 | :success? true 307 | :event ::sut/cache-hit}]))) 308 | 309 | (deftest does-not-load-expired-result-from-cache 310 | (is (= (let [cache (atom {[::sut/req {:method :get 311 | :url "http://example.com/"}] 312 | {:req {:method :get 313 | :url "http://example.com"} 314 | :res {:status 200 315 | :body "Oh yeah!"} 316 | :expires-at (time/add-millis (time/now) -10)}})] 317 | (->> {:example {:req {:url "http://example.com/"}}} 318 | (sut/make-requests {:cache (cache/create-atom-map-cache cache)}) 319 | sut/collect!! 320 | (map summarize-event))) 321 | [[::sut/request :example [:get "http://example.com/"]] 322 | [::sut/response :example [200 {:request {:url "http://example.com/" 323 | :throw-exceptions false 324 | :method :get}}]]]))) 325 | 326 | (deftest uses-cached-dependent-request 327 | (is (= (let [cache (atom {[::sut/req {:method :post 328 | :url "http://example.com/security/"}] 329 | {:req {:method :post 330 | :url "http://example.com/security/"} 331 | :res {:status 200 332 | :body {:token "T0k3n"}}}})] 333 | (->> {:example {:lookup-id :example 334 | :req-fn (fn [{:keys [id token]}] 335 | {:url (str "http://example.com/" id) 336 | :headers {"Authorization" (str "Bearer " token)}}) 337 | :params [:token]}} 338 | (sut/make-requests 339 | {:cache (cache/create-atom-map-cache cache) 340 | :params 341 | {:token {::sut/req {:req {:method :post 342 | :url "http://example.com/security/"}} 343 | ::sut/select (comp :token :body)}}}) 344 | sut/collect!! 345 | (map summarize-event) 346 | )) 347 | [[::sut/cache-hit :token [200 {:token "T0k3n"}]] 348 | [::sut/request :example 349 | [:get "http://example.com/" {"Authorization" "Bearer T0k3n"}]] 350 | [::sut/response :example 351 | [200 {:request 352 | {:url "http://example.com/" 353 | :headers {"Authorization" "Bearer T0k3n"} 354 | :method :get 355 | :throw-exceptions false}}]]]))) 356 | 357 | (deftest uses-surgical-cache-key-for-lookup 358 | (is (= (let [cache (atom {[:example {:id 42 :config {:host "example.com"}}] 359 | {:req {:method :get 360 | :url "http://example.com/42"} 361 | :res {:status 200 362 | :body "I'm cached!"}}})] 363 | (->> {:example {:lookup-id :example 364 | :req-fn (fn [{:keys [id token config]}] 365 | {:url (str "http://" (:host config) "/" id) 366 | :headers {"Authorization" (str "Bearer " token)}}) 367 | :params [:id :config :token] 368 | :lookup-params [[:config :host] :id]}} 369 | (sut/make-requests 370 | {:cache (cache/create-atom-map-cache cache) 371 | :params 372 | {:token {::sut/req {:req {:method :post 373 | :url "http://example.com/security/"}} 374 | ::sut/select (comp :token :body)} 375 | :id 42 376 | :config {:host "example.com" 377 | :debug? true}}}) 378 | sut/collect!! 379 | (map summarize-event))) 380 | [[:courier.http/cache-hit :example [200 "I'm cached!"]]]))) 381 | 382 | (deftest uses-prepared-lookup-params-for-cache 383 | (is (= (let [cache (atom {[:example 42] 384 | {:req {:method :get 385 | :url "http://example.com/42"} 386 | :res {:status 200 387 | :body "I'm cached!"}}})] 388 | (->> {:example {:lookup-id :example 389 | :req-fn (fn [{:keys [id token config]}] 390 | {:url (str "http://" (:host config) "/" id) 391 | :headers {"Authorization" (str "Bearer " token)}}) 392 | :params [:id :config :token] 393 | :lookup-params [:id] 394 | :prepare-lookup-params (fn [params] 395 | (:id params))}} 396 | (sut/make-requests 397 | {:cache (cache/create-atom-map-cache cache) 398 | :params 399 | {:token {::sut/req {:req {:method :post 400 | :url "http://example.com/security/"}} 401 | ::sut/select (comp :token :body)} 402 | :id 42 403 | :config {:host "example.com" 404 | :debug? true}}}) 405 | sut/collect!! 406 | (map summarize-event))) 407 | [[:courier.http/cache-hit :example [200 "I'm cached!"]]]))) 408 | 409 | (deftest looks-up-cache-entry-without-params 410 | (is (= (let [cache (atom {[:example nil] 411 | {:req {:method :get 412 | :url "http://example.com/42"} 413 | :res {:status 200 414 | :body "I'm cached!"}}})] 415 | (->> {:example {:lookup-id :example 416 | :req-fn (fn [params] 417 | {:url (str "http://example.com/42")})}} 418 | (sut/make-requests {:cache (cache/create-atom-map-cache cache)}) 419 | sut/collect!! 420 | (map summarize-event))) 421 | [[:courier.http/cache-hit :example [200 "I'm cached!"]]]))) 422 | 423 | (deftest skips-dependent-request-when-result-is-cached 424 | (is (= (let [cache (atom {[:example {:id 42}] 425 | {:req {:method :get 426 | :url "http://example.com/42"} 427 | :res {:status 200 428 | :body "I'm cached!"}}})] 429 | (->> {:example {:lookup-id :example 430 | :req-fn (fn [{:keys [id token]}] 431 | {:url (str "http://example.com/" id) 432 | :headers {"Authorization" (str "Bearer " token)}}) 433 | :params [:id :token] 434 | :lookup-params [:id]}} 435 | (sut/make-requests 436 | {:cache (cache/create-atom-map-cache cache) 437 | :params 438 | {:token {:req {:method :post 439 | :url "http://example.com/security/"}} 440 | :id 42}}) 441 | sut/collect!! 442 | (map summarize-event))) 443 | [[:courier.http/cache-hit :example [200 "I'm cached!"]]]))) 444 | 445 | (deftest caches-successful-result-and-reuses-it 446 | (is (= (with-responses {[:get "https://example.com/"] 447 | [{:status 200 448 | :body {:content "Skontent"}}]} 449 | (let [cache (cache/create-atom-map-cache (atom {})) 450 | spec {:example {:req {:url "https://example.com/"} 451 | :cache-fn (sut/cache-fn {:ttl 100})}}] 452 | (concat 453 | (->> (sut/make-requests {:cache cache} spec) 454 | sut/collect!! 455 | (map summarize-event)) 456 | (->> (sut/make-requests {:cache cache} spec) 457 | sut/collect!! 458 | (map summarize-event))))) 459 | [[::sut/request :example [:get "https://example.com/"]] 460 | [::sut/response :example [200 {:content "Skontent"}]] 461 | [::sut/store-in-cache :example [200 {:content "Skontent"}]] 462 | [::sut/cache-hit :example [200 {:content "Skontent"}]]]))) 463 | 464 | (deftest does-not-cache-successful-result-with-no-ttl 465 | (is (= (with-responses {[:get "https://example.com/"] 466 | [{:status 200 467 | :body {:content "Skontent"}}]} 468 | (let [cache (atom {})] 469 | (sut/request 470 | {:req {:url "https://example.com/"}} 471 | {:cache (cache/create-atom-map-cache cache)}) 472 | @cache)) 473 | {}))) 474 | 475 | (deftest does-not-cache-uncacheable-successful-result 476 | (is (= (with-responses {[:get "https://example.com/"] 477 | [{:status 200 478 | :body {:content "Skontent"}}]} 479 | (let [cache (atom {})] 480 | (sut/request 481 | {:req {:url "https://example.com/"} 482 | :cache-fn (sut/cache-fn {:cacheable? (constantly false) 483 | :ttl 100})} 484 | {:cache (cache/create-atom-map-cache cache)}) 485 | @cache)) 486 | {}))) 487 | 488 | (deftest caches-and-looks-up-result-in-cache 489 | (is (= (with-responses {[:get "https://example.com/"] 490 | [{:status 200 491 | :body {:content "Skontent"}}]} 492 | (let [cache (cache/create-atom-map-cache (atom {})) 493 | spec {:lookup-id ::example 494 | :cache-fn (sut/cache-fn {:ttl 100}) 495 | :req-fn (fn [params] 496 | {:url "https://example.com/"})}] 497 | (concat 498 | (->> {:example spec} 499 | (sut/make-requests {:cache cache}) 500 | sut/collect!! 501 | (map summarize-event)) 502 | (->> {:example spec} 503 | (sut/make-requests {:cache cache}) 504 | sut/collect!! 505 | (map summarize-event))))) 506 | [[::sut/request :example [:get "https://example.com/"]] 507 | [::sut/response :example [200 {:content "Skontent"}]] 508 | [::sut/store-in-cache :example [200 {:content "Skontent"}]] 509 | [::sut/cache-hit :example [200 {:content "Skontent"}]]]))) 510 | 511 | (deftest caches-result-with-expiry 512 | (is (<= 3600000 513 | (let [now (time/now)] 514 | (with-responses {[:get "https://example.com/"] 515 | [{:status 200 516 | :body {:content "Skontent"}}]} 517 | (let [cache (atom {})] 518 | (->> {:example {:req {:url "https://example.com/"} 519 | :cache-fn (sut/cache-fn {:ttl (* 60 60 1000)})}} 520 | (sut/make-requests {:cache (cache/create-atom-map-cache cache)}) 521 | sut/collect!!) 522 | (- (-> @cache first second :expires-at time/millis) (time/millis now))))) 523 | 3600010))) 524 | 525 | (deftest caches-result-with-expiry-function 526 | (is (<= 100 527 | (let [now (time/now)] 528 | (with-responses {[:get "https://example.com/"] 529 | [{:status 200 530 | :body {:ttl 100}}]} 531 | (let [cache (atom {})] 532 | (->> {:example {:req {:url "https://example.com/"} 533 | :cache-fn (sut/cache-fn {:ttl-fn #(-> % :res :body :ttl)})}} 534 | (sut/make-requests {:cache (cache/create-atom-map-cache cache)}) 535 | sut/collect!!) 536 | (- (-> @cache first second :expires-at time/millis) (time/millis now))))) 537 | 120))) 538 | 539 | (deftest does-not-cache-if-cache-ttl-fn-throws 540 | (is (= (with-responses {[:get "https://example.com/"] 541 | [{:status 200 542 | :body {}}]} 543 | (->> {:example {:req {:url "https://example.com/"} 544 | :cache-fn (sut/cache-fn {:ttl-fn (fn [_] (throw (ex-info "Boom!" {})))})}} 545 | (sut/make-requests {:cache (cache/create-atom-map-cache (atom {}))}) 546 | sut/collect!! 547 | (map summarize-event))) 548 | [[:courier.http/request :example [:get "https://example.com/"]] 549 | [:courier.http/exception "Boom!" :cache-fn] 550 | [:courier.http/response :example [200 {}]]]))) 551 | 552 | (deftest does-not-cache-if-cacheable-fn-throws 553 | (is (= (with-responses {[:get "https://example.com/"] 554 | [{:status 200 555 | :body {}}]} 556 | (->> {:example {:req {:url "https://example.com/"} 557 | :cache-fn (sut/cache-fn {:cacheable? (fn [_] (throw (ex-info "Boom!" {})))})}} 558 | (sut/make-requests {:cache (cache/create-atom-map-cache (atom {}))}) 559 | sut/collect!! 560 | (map summarize-event))) 561 | [[:courier.http/request :example [:get "https://example.com/"]] 562 | [:courier.http/exception "Boom!" :cache-fn] 563 | [:courier.http/response :example [200 {}]]]))) 564 | 565 | (deftest does-not-cache-the-http-client-on-the-response 566 | (is (= (with-responses {[:get "https://example.com/"] 567 | [{:status 200 568 | :body {:ttl 100} 569 | :http-client {:stateful "Object"}}]} 570 | (let [cache (atom {})] 571 | (sut/request 572 | {:req {:url "https://example.com/"} 573 | :cache-fn (sut/cache-fn {:ttl 100})} 574 | {:cache (cache/create-atom-map-cache cache)}) 575 | (some-> @cache first second :res (select-keys [:http-client])))) 576 | {}))) 577 | 578 | (deftest includes-cache-info-on-store 579 | (let [cache-status 580 | (with-responses {[:get "https://example.com/"] 581 | [{:status 200 582 | :body {:ttl 100}}]} 583 | (let [cache (cache/create-atom-map-cache (atom {})) 584 | spec {:req {:url "https://example.com/"} 585 | :cache-fn (sut/cache-fn {:ttl-fn #(-> % :res :body :ttl)})}] 586 | (-> (sut/request spec {:cache cache}) 587 | :cache-status)))] 588 | (is (true? (:stored-in-cache? cache-status))) 589 | (is (number? (:cached-at cache-status))) 590 | (is (number? (:expires-at cache-status))) 591 | (is (= (::cache/key cache-status) 592 | [::sut/req {:method :get :url "https://example.com/"}])))) 593 | 594 | (deftest includes-cache-info-on-retrieve 595 | (let [cache-status 596 | (with-responses {[:get "https://example.com/"] 597 | [{:status 200 598 | :body {:ttl 100}}]} 599 | (let [cache (cache/create-atom-map-cache (atom {})) 600 | spec {:req {:url "https://example.com/"} 601 | :cache-fn (sut/cache-fn {:ttl-fn #(-> % :res :body :ttl)})}] 602 | (sut/request spec {:cache cache}) 603 | (-> (sut/request spec {:cache cache}) 604 | :cache-status)))] 605 | (is (true? (:cache-hit? cache-status))) 606 | (is (number? (:cached-at cache-status))) 607 | (is (number? (:expires-at cache-status))))) 608 | 609 | (deftest retries-bypassing-the-cache-for-refreshed 610 | (is (= (with-responses {[:post "https://example.com/security/"] 611 | [{:status 200 612 | :body {:token "ejY...."}}] 613 | 614 | [:get "https://example.com/api"] 615 | [{:status 500} 616 | {:status 200 617 | :body {:stuff "Stuff"}}]} 618 | (let [cache (atom {[::sut/req {:method :post 619 | :url "https://example.com/security/"}] 620 | {:req {:method :post 621 | :url "https://example.com/security/"} 622 | :res {:status 200 623 | :body {:token "T0k3n"}}}})] 624 | 625 | (->> {:example {:req-fn (fn [{:keys [token]}] 626 | {:url "https://example.com/api" 627 | :headers {"Authorization" (str "Bearer " token)}}) 628 | :params [:token] 629 | :retry-fn (sut/retry-fn {:retries 1 630 | :refresh [:token]})}} 631 | (sut/make-requests 632 | {:cache (cache/create-atom-map-cache cache) 633 | :params 634 | {:token {::sut/req {:req {:method :post 635 | :url "https://example.com/security/"}} 636 | ::sut/select (comp :token :body)}}}) 637 | sut/collect!! 638 | (map summarize-event)))) 639 | [[::sut/cache-hit :token [200 {:token "T0k3n"}]] 640 | [::sut/request :example 641 | [:get "https://example.com/api" {"Authorization" "Bearer T0k3n"}]] 642 | [::sut/response :example [500]] 643 | [::sut/request :token [:post "https://example.com/security/"]] 644 | [::sut/response :token [200 {:token "ejY...."}]] 645 | [::sut/request :example 646 | [:get "https://example.com/api" {"Authorization" "Bearer ejY...."}]] 647 | [::sut/response :example [200 {:stuff "Stuff"}]]]))) 648 | 649 | ;; request tests 650 | 651 | (deftest makes-basic-request 652 | (is (= (:body (sut/request {:req {:url "http://example.com/"}})) 653 | {:request {:method :get 654 | :throw-exceptions false 655 | :url "http://example.com/"}}))) 656 | 657 | (deftest communicates-success 658 | (is (-> (sut/request {:req {:url "http://example.com/"}}) 659 | :success?))) 660 | 661 | (deftest communicates-failure 662 | (is (false? (with-responses {[:get "http://example.com/"] 663 | [{:status 404 664 | :body "No"}]} 665 | (-> (sut/request {:req {:url "http://example.com/"}}) 666 | :success?))))) 667 | 668 | (deftest communicates-failure-from-custom-assessment 669 | (is (false? (with-responses {[:get "http://example.com/"] 670 | [{:status 200 671 | :body "No"}]} 672 | (-> (sut/request {:req {:url "http://example.com/"} 673 | :success? #(= 201 (:status %))}) 674 | :success?))))) 675 | 676 | (deftest communicates-success-on-cached-success 677 | (is (true? (-> (sut/request 678 | {:cache-fn (sut/cache-fn {:ttl (* 5 60 1000)}) 679 | :req {:method :get 680 | :url "http://example.com"}} 681 | {:cache (cache/create-atom-map-cache (atom {}))}) 682 | :success?)))) 683 | 684 | (deftest includes-request-log 685 | (is (= (with-responses {[:get "http://example.com/"] 686 | [{:status 200 687 | :body "Ok!"}]} 688 | (-> (sut/request {:req {:url "http://example.com/"}}) 689 | :log)) 690 | [{:req {:method :get 691 | :url "http://example.com/" 692 | :throw-exceptions false} 693 | :res {:status 200 694 | :body "Ok!"} 695 | :success? true 696 | :event :courier.http/response}]))) 697 | 698 | (deftest includes-request-log-events-for-cache-retrieval 699 | (is (= (with-responses {[:get "https://example.com/"] 700 | [{:status 200 701 | :body {:ttl 100}}]} 702 | (let [cache (cache/create-atom-map-cache (atom {})) 703 | spec {:req {:url "https://example.com/"} 704 | :cache-fn (sut/cache-fn {:ttl-fn #(-> % :res :body :ttl)})}] 705 | (sut/request spec {:cache cache}) 706 | (->> (sut/request spec {:cache cache}) 707 | :log 708 | (map #(dissoc % :expires-at :cached-at))))) 709 | [{:req {:url "https://example.com/" 710 | :method :get 711 | :throw-exceptions false} 712 | :res {:status 200 713 | :body {:ttl 100}} 714 | :success? true 715 | :event :courier.http/cache-hit}]))) 716 | 717 | (deftest includes-request-log-on-failure 718 | (is (= (with-responses {[:get "http://example.com/"] 719 | [{:status 404 720 | :body "Ok!"}]} 721 | (-> (sut/request {:req {:url "http://example.com/"}}) 722 | :log)) 723 | [{:req {:method :get 724 | :throw-exceptions false 725 | :url "http://example.com/"} 726 | :res {:status 404 727 | :body "Ok!"} 728 | :success? false 729 | :event :courier.http/response} 730 | {:courier.error/reason :courier.error/request-failed 731 | :courier.error/data {:status 404 732 | :body "Ok!"} 733 | :event :courier.http/failed}]))) 734 | 735 | (deftest includes-response-like-keys 736 | (is (= (with-responses {[:get "http://example.com/"] 737 | [{:status 200 738 | :headers {"Content-Type" "text/plain"} 739 | :body "Ok!"}]} 740 | (-> (sut/request {:req {:url "http://example.com/"}}) 741 | (select-keys [:status :headers :body]))) 742 | {:status 200 743 | :headers {"Content-Type" "text/plain"} 744 | :body "Ok!"}))) 745 | 746 | (deftest prepares-request-with-function 747 | (is (= (-> (sut/request {:req-fn (fn [_] 748 | {:url "http://example.com/"})}) 749 | :body) 750 | {:request {:method :get 751 | :throw-exceptions false 752 | :url "http://example.com/"}}))) 753 | 754 | (deftest passes-no-params-by-default 755 | (is (= (-> (sut/request 756 | {:req-fn (fn [params] 757 | {:url "http://example.com/" 758 | :params params})} 759 | {:params {:client-id "ID" 760 | :client-secret "Secret"}}) 761 | :body 762 | :request) 763 | {:method :get 764 | :throw-exceptions false 765 | :url "http://example.com/" 766 | :params {}}))) 767 | 768 | (deftest passes-specified-params-from-context 769 | (is (= (-> (sut/request 770 | {:req-fn (fn [params] 771 | {:url "http://example.com/" 772 | :params params}) 773 | :params [:client-id]} 774 | {:params {:client-id "ID" 775 | :client-secret "Secret"}}) 776 | :body 777 | :request) 778 | {:method :get 779 | :throw-exceptions false 780 | :url "http://example.com/" 781 | :params {:client-id "ID"}}))) 782 | 783 | (deftest retries-failed-request 784 | (is (= (-> (with-responses {[:get "http://example.com/"] 785 | [{:status 503 786 | :body "Uh-oh"} 787 | {:status 200 788 | :body "Yass!"}]} 789 | (sut/request 790 | {:req {:url "http://example.com/"} 791 | :retry-fn (sut/retry-fn {:retries 1})})) 792 | :body) 793 | "Yass!"))) 794 | 795 | (deftest does-not-retry-failed-request-that-is-not-retryable 796 | (is (= (-> (with-responses {[:get "http://example.com/"] 797 | [{:status 503 798 | :body "Uh-oh"} 799 | {:status 200 800 | :body "Yass!"}]} 801 | (sut/request 802 | {:req {:url "http://example.com/"} 803 | :retry-fn (sut/retry-fn {:retryable? #(= 500 (-> % :res :status)) 804 | :retries 1})})) 805 | :success?) 806 | false))) 807 | 808 | (deftest handles-exceptions-when-loading-cached-objects 809 | (is (= (->> (sut/make-requests 810 | {:cache (reify cache/Cache 811 | (lookup [_ _ _] 812 | (throw (ex-info "Boom!" {:boom? true}))) 813 | (put [_ _ _ _]))} 814 | {:example {:req {:url "http://example.com/"}}}) 815 | sut/collect!! 816 | (map summarize-event)) 817 | [[::sut/exception "Boom!" "courier.cache/lookup"] 818 | [::sut/request :example [:get "http://example.com/"]] 819 | [::sut/response :example [200 {:request {:url "http://example.com/" 820 | :method :get 821 | :throw-exceptions false}}]]]))) 822 | 823 | (deftest handles-exceptions-when-storing-cached-objects 824 | (is (= (->> (sut/make-requests 825 | {:cache (reify cache/Cache 826 | (lookup [_ _ _] 827 | nil) 828 | (put [_ _ _ _] 829 | (throw (ex-info "Boom!" {:boom? true}))))} 830 | {:example {:req {:url "http://example.com/"} 831 | :cache-fn (sut/cache-fn {:ttl 100})}}) 832 | sut/collect!! 833 | (map summarize-event)) 834 | [[:courier.http/request :example [:get "http://example.com/"]] 835 | [:courier.http/response :example [200 {:request {:url "http://example.com/" 836 | :method :get 837 | :throw-exceptions false}}]] 838 | [:courier.http/exception "Boom!" "courier.cache/put"]]))) 839 | 840 | (defmethod client/request [:get "https://explosives.com"] [req] 841 | (throw (ex-info "Boom!" {:boom? true}))) 842 | 843 | (deftest does-not-trip-on-exceptions-from-the-http-client 844 | (let [result (sut/request {:req {:url "https://explosives.com"}})] 845 | (is (not (:success? result))) 846 | (is (seq (:exceptions result))))) 847 | 848 | (deftest informs-user-of-probable-misuse 849 | (is (= (sut/request 850 | {:params [:id] 851 | :req-fn (fn [{:keys [id]}] 852 | {:url (str "http://example.com/" id)})} 853 | {:id 42}) 854 | {:success? false 855 | :log [{:courier.error/reason :courier.error/missing-params 856 | :courier.error/data [:id] 857 | :event :courier.http/failed}] 858 | :hint (str "Make sure you pass parameters to your request as " 859 | "`:params` in the options map, not directly in the map, " 860 | "e.g.: {:params {:id 42}}, not {:id 42}")}))) 861 | 862 | (defmethod client/request [:get "https://lolcathost"] [req] 863 | (throw (java.net.UnknownHostException. "Boom!"))) 864 | 865 | (deftest properly-communicates-unknown-host 866 | (is (= (-> (sut/request {:req {:url "https://lolcathost"}}) 867 | :log 868 | last) 869 | {:courier.error/reason :courier.error/unknown-host 870 | :courier.error/data {:req {:url "https://lolcathost" 871 | :method :get 872 | :throw-exceptions false}} 873 | :event :courier.http/failed}))) 874 | --------------------------------------------------------------------------------