├── .gitignore ├── LICENSE ├── README.md ├── project.clj ├── src └── pipeline │ ├── control.clj │ ├── kill_switch.clj │ ├── process.clj │ ├── process │ ├── listener.clj │ ├── messages.clj │ ├── protocols.clj │ └── worker.clj │ ├── protocols.clj │ └── utils │ ├── async.clj │ └── schema.clj └── test └── pipeline ├── kill_switch_test.clj ├── process_test.clj ├── test_utils.clj └── test_utils └── async.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hgignore 11 | .hg/ 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014-2015 Staples, Inc. 2 | 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pipeline 2 | 3 | A Clojure library for building data-processing chain pipelines 4 | 5 | ## Usage 6 | 7 | Add this dependency to your leiningen project.clj 8 | 9 | [polyglogon/pipeline "0.1.0"] 10 | 11 | ### Example code 12 | 13 | Here is a contrived example that repeatedly reuses a pipeline 14 | 15 | ``` clojure 16 | (ns pipeline-example.core 17 | (:require [clojure.core.async :as async] 18 | [clojure.pprint :refer [pprint]] 19 | [pipeline.control :as control] 20 | [pipeline.kill-switch :as kill-switch] 21 | [pipeline.protocols :refer [PipelineImpl]] 22 | [pipeline.process :as process])) 23 | 24 | (defn- square [x] 25 | (* x x)) 26 | 27 | (defrecord Squarer [_] 28 | PipelineImpl 29 | (handle [_ input-message] 30 | [(square input-message)]) 31 | (finish [_ _] 32 | [])) 33 | 34 | (defrecord Summer [accum] 35 | PipelineImpl 36 | (handle [_ input-message] 37 | (swap! accum + input-message) 38 | []) 39 | (finish [_ completed?] 40 | (if completed? 41 | [@accum]))) 42 | 43 | (defn summer-factory [ignored] 44 | (->Summer (atom 0))) 45 | 46 | (defn run [] 47 | (let [control-chan-1 (async/chan 1) 48 | control-chan-2 (async/chan 1)] 49 | (process/create 3 50 | control-chan-1 51 | ->Squarer 52 | :compute) 53 | (process/create 1 54 | control-chan-2 55 | summer-factory 56 | :compute) 57 | (doseq [i (range 100) 58 | :let [kill-switch (kill-switch/create) 59 | in-chan-1 (async/chan) 60 | in-chan-2 (async/chan) 61 | out-chan-2 (async/chan)]] 62 | (async/onto-chan in-chan-1 [1 2 3 4 5]) 63 | (control/send-message control-chan-1 64 | :input-chan in-chan-1 65 | :output-chan in-chan-2 66 | :kill-switch kill-switch 67 | :context {:foo :bar}) 68 | (control/send-message control-chan-2 69 | :input-chan in-chan-2 70 | :output-chan out-chan-2 71 | :kill-switch kill-switch 72 | :context {:spam :eggs}) 73 | (pprint (async/!! control-chan message)) 28 | 29 | (defn send-message 30 | [control-chan 31 | & {:keys [input-chan output-chan kill-switch context]}] 32 | (send-message-vec control-chan 33 | [input-chan output-chan kill-switch context])) 34 | -------------------------------------------------------------------------------- /src/pipeline/kill_switch.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "Implement a kill switch to be used by pipeline.process 2 | 3 | The kill-switch is shared by all pipeline.process instances 4 | to halt all activities across the data-processing chain when an 5 | exception occurs. The kill-switch should also be checked by code 6 | using a pipeline once the work is done (to determine if output can 7 | be considered valid)."} 8 | pipeline.kill-switch 9 | (:require [clojure.core.async :as async] 10 | [pipeline.protocols :refer :all]) 11 | (:import java.sql.SQLException)) 12 | 13 | (def ^:private kill-chan-buff-size 20) 14 | 15 | (defn- exception->map 16 | "Build a map from an exception. Copied out of staples-sparx/kits 17 | rev 67c48c5 instead of adding a lot of irrelevant deps. MIT license." 18 | [^Throwable e] 19 | (merge 20 | {:class (str (class e)) 21 | :message (.getMessage e) 22 | :stacktrace (mapv str (.getStackTrace e))} 23 | (when (.getCause e) 24 | {:cause (exception->map (.getCause e))}) 25 | (if (instance? SQLException e) 26 | (if-let [ne (.getNextException ^SQLException e)] 27 | {:next-exception (exception->map ne)})))) 28 | 29 | (defn- now 30 | "Current time in millis since the epoch 31 | Keep here for ease of mocking" 32 | [] 33 | (System/currentTimeMillis)) 34 | 35 | (defrecord AtomicKillSwitch [state listener-chan listener-mult] 36 | 37 | KillSwitch 38 | (killed? [_] 39 | (boolean (seq @state))) 40 | (kill! [_ details-m] 41 | (let [details-m2 (merge details-m {:timestamp (now)})] 42 | (swap! state (fn [x] (conj x details-m2))) 43 | (async/>!! listener-chan details-m2))) 44 | (kill-exception! [switch exception] 45 | (kill! switch (merge (ex-data exception) 46 | (exception->map exception)))) 47 | 48 | ErrorRepo 49 | (errors [_] @state) 50 | (first-error [_] (first @state)) 51 | 52 | Listener 53 | (tap [_ chan] 54 | (async/tap listener-mult chan)) 55 | (close! [_] 56 | (async/close! listener-chan))) 57 | 58 | (defn create [] 59 | (map->AtomicKillSwitch 60 | (let [listener-chan (async/chan (async/dropping-buffer kill-chan-buff-size))] 61 | {:state (atom []) 62 | :listener-chan listener-chan 63 | :listener-mult (async/mult listener-chan)}))) 64 | -------------------------------------------------------------------------------- /src/pipeline/process.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "Creates a CSP style process for composing data-processing pipelines 2 | 3 | The process is backed my multiple internal threads (green 4 | and possibly real). Work is implemented using an instance of the 5 | PipelineImpl protocol. The data-processing pipeline is generally 6 | instantiated once and is reused for multiple ad hoc tasks, which may 7 | be run concurrently. Each task has a distinct input/output channel 8 | pair that are used by the pipeline process when handling messages. 9 | The channel pairing exists only as long as the input-channel remains 10 | open. Multiple processes may be chained together (one's output 11 | channel being another's input channel) to create the data-processing 12 | pipeline. Closing the first input channel cascades closure through 13 | the pipeline until the last output channel in the chain is closed. 14 | This is how termination detection is enabled. An instance of 15 | the KillSwitch protocol is used to determine if there were any 16 | errors during processing and to quickly kill the ad hoc chain when 17 | there are errors. Input/output channel pairings are established by 18 | sending control messages over a control channel. 19 | 20 | Inspired by clojure.core.async/pipeline, but with many differences."} 21 | pipeline.process 22 | (:require [clojure.core.async :as async] 23 | [pipeline.process.listener :as listener] 24 | [pipeline.process.worker :as worker] 25 | [pipeline.utils.schema :as local-schema] 26 | [schema.core :as schema])) 27 | 28 | (defn- broadcast-control-messages 29 | "To support internal concurrency, pipeline.process is implemented 30 | with distinct listener and worker threads. Each listener thread 31 | needs a unique internal control channel as well as unique output 32 | channels. Internal control and output channels are created here and 33 | a green thread is created that intercepts control messages and sends 34 | modified control messages on the internal control channels. Control 35 | messages are modified to use the internal output channels." 36 | [n external-control-chan] 37 | (let [internal-control-chans (repeatedly n #(async/chan 1))] 38 | (async/go-loop [] 39 | (if-let [[input-chan output-chan kill-switch context] 40 | (async/! (nth internal-control-chans i) [input-chan 45 | (nth my-output-chans i) 46 | kill-switch 47 | context])) 48 | (recur)) 49 | (doseq [c internal-control-chans] 50 | (async/close! c)))) 51 | internal-control-chans)) 52 | 53 | 54 | 55 | (schema/defn create :- (schema/eq nil) 56 | "Create a pipeline.process (a link in the data-processing chain) 57 | 58 | Takes: 59 | - concurrency number (> n 0) 60 | - Control channel to listen on 61 | - Fn that takes a context map (provided in control messages) and returns 62 | an instance of the PipelineImpl protocol (eg a factory) 63 | - mode keyword, either :blocking (for real threads) or :compute (for green 64 | threads). 65 | 66 | Returns: nil" 67 | [n :- local-schema/PosInt 68 | external-control-chan :- local-schema/Chan 69 | pimpl-factory :- (schema/pred fn?) 70 | pipeline-mode :- (schema/enum :blocking :compute)] 71 | (let [internal-control-chans 72 | (broadcast-control-messages n external-control-chan) 73 | 74 | jobs-chan 75 | (async/chan n)] 76 | (doseq [control-chan internal-control-chans] 77 | (worker/work pipeline-mode jobs-chan) 78 | (listener/listen control-chan jobs-chan pimpl-factory)))) 79 | -------------------------------------------------------------------------------- /src/pipeline/process/listener.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "The listener takes messages from input channels and handles 2 | them by creating jobs and submitting them on the jobs 3 | channel. Also tracks state related to input-channels."} 4 | pipeline.process.listener 5 | (:require [clojure.core.async :as async] 6 | [clojure.core.match :refer [match]] 7 | [pipeline.process.messages :as messages] 8 | [pipeline.process.protocols :refer :all] 9 | [pipeline.protocols :as prots] 10 | [pipeline.utils.async :as local-async] 11 | [pipeline.utils.schema :as local-schema] 12 | [schema.core :as schema])) 13 | 14 | (defn- submit-a-job 15 | "Asynchronously put a job message onto the jobs channel and wait for 16 | the result. Returns a results channel." 17 | [jobs-chan pipeline-impl message state] 18 | (async/go 19 | (let [promise-chan (async/chan 1)] 20 | (async/>! jobs-chan [promise-chan pipeline-impl message state]) 21 | (async/pimpl {}] 51 | (if (= [] input-chans) 52 | (async/close! jobs) 53 | (match 54 | (async/alts! input-chans) 55 | 56 | [nil control-input-chan] 57 | (recur (filterv (partial not= control-input-chan) input-chans) 58 | input-chan->pimpl) 59 | 60 | [[input-chan output-chan kill-switch context] 61 | control-input-chan] 62 | (let [kill-chan (prots/tap kill-switch (async/chan (async/dropping-buffer 1))) 63 | pimpl (pimpl-factory-fn context)] 64 | (if (satisfies? prots/PipelineImpl pimpl) 65 | (recur (into input-chans [input-chan kill-chan]) 66 | (assoc input-chan->pimpl 67 | input-chan (map->PipelineTaskImpl 68 | {:pimpl pimpl 69 | :kill-switch kill-switch 70 | :out-chan output-chan 71 | :kill-chan kill-chan 72 | :in-chan input-chan}) 73 | kill-chan input-chan)) 74 | (do (prots/kill! kill-switch 75 | {:message "Factory did not return a PipelineImpl" 76 | :context context}) 77 | (async/close! output-chan) 78 | (recur input-chans input-chan->pimpl)))) 79 | 80 | [_ control-input-chan] 81 | (recur input-chans input-chan->pimpl) 82 | 83 | [message message-chan] 84 | (let [inchan-or-state 85 | (get input-chan->pimpl message-chan) 86 | 87 | [input-chan pipeline-task-impl] 88 | (if (local-async/channel? inchan-or-state) 89 | [inchan-or-state (get input-chan->pimpl inchan-or-state)] 90 | [message-chan inchan-or-state]) 91 | 92 | killed? 93 | (kill-chan? pipeline-task-impl message-chan) 94 | 95 | [message some-message?] 96 | (cond 97 | (nil? message) [messages/none false] 98 | killed? [messages/none false] 99 | :else [message true]) 100 | 101 | job-result 102 | (async/pimpl) 127 | (do (close-out-chan! pipeline-task-impl) 128 | (recur (filterv (complement 129 | #(some (partial = %) 130 | [input-chan (kill-chan pipeline-task-impl)])) 131 | input-chans) 132 | (dissoc input-chan->pimpl 133 | input-chan 134 | kill-chan))))))))) 135 | -------------------------------------------------------------------------------- /src/pipeline/process/messages.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "Signals shared by listener and worker"} 2 | pipeline.process.messages) 3 | 4 | (def ok ::ok) 5 | 6 | (def none ::none) 7 | 8 | (def exception ::exception) 9 | 10 | (def killed ::killed) 11 | -------------------------------------------------------------------------------- /src/pipeline/process/protocols.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "Protocols that are used by the listener and worker. 2 | Not intended for external use."} 3 | pipeline.process.protocols 4 | (:require [clojure.core.async :as async] 5 | [pipeline.protocols :as prots])) 6 | 7 | (defprotocol PipelineTaskState 8 | "State tracked by the listener that is associated with an 9 | input-channel that it is listening to" 10 | 11 | (in-chan [self]) 12 | (out-chan [self]) 13 | (out-chan? [self chan]) 14 | (close-out-chan! [self]) 15 | (kill-chan [self]) 16 | (kill-chan? [self chan])) 17 | 18 | ;; Implementation of multiple protocols, existing for the purpose of 19 | ;; tracking the state of a listener task and delegating actions to 20 | ;; wrapped instances. Used to handle messages. 21 | (defrecord PipelineTaskImpl [pimpl kill-switch in-chan kill-chan out-chan] 22 | 23 | PipelineTaskState 24 | (in-chan [_] 25 | in-chan) 26 | (out-chan [_] 27 | out-chan) 28 | (out-chan? [_ chan] 29 | (= out-chan chan)) 30 | (close-out-chan! [_] 31 | (async/close! out-chan)) 32 | (kill-chan [_] 33 | kill-chan) 34 | (kill-chan? [_ chan] 35 | (= kill-chan chan)) 36 | 37 | prots/PipelineImpl 38 | (handle [_ input-message] 39 | (prots/handle pimpl input-message)) 40 | (finish [_ completed?] 41 | (prots/finish pimpl completed?)) 42 | 43 | prots/KillSwitch 44 | (killed? [_] 45 | (prots/killed? kill-switch)) 46 | (kill! [_ details-m] 47 | (prots/kill! kill-switch details-m)) 48 | (kill-exception! [_ exception] 49 | (prots/kill-exception! kill-switch exception))) 50 | -------------------------------------------------------------------------------- /src/pipeline/process/worker.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "The worker takes messages from the jobs channel and handles them 2 | on either a green thread or a real thread (configurable)."} 3 | pipeline.process.worker 4 | (:require [clojure.core.async :as async] 5 | [pipeline.protocols :as prots] 6 | [pipeline.process.messages :as messages] 7 | [pipeline.process.protocols :as process-prots] 8 | [pipeline.utils.schema :as local-schema] 9 | [schema.core :as schema])) 10 | 11 | (def ^:private put-timeout-ms 10000) 12 | 13 | (defn- handle-closed-out-chan [kill-switch chan item] 14 | (prots/kill! kill-switch 15 | {:message "Output channel was already closed" 16 | :chan chan 17 | :item item})) 18 | 19 | (defn- call-handler-method [pimpl message state] 20 | (if (= message messages/none) 21 | (prots/finish pimpl (= state messages/ok)) 22 | (prots/handle pimpl message))) 23 | 24 | (defmacro ^:private handle-a-message 25 | [mode pipeline-impl message state] 26 | (let [alts-fn (case mode :blocking 'async/alts!! :compute 'async/alts!)] 27 | `(loop [items# (call-handler-method ~pipeline-impl ~message ~state) 28 | out-chan# (process-prots/out-chan ~pipeline-impl)] 29 | (let [next-item# (first items#) 30 | rest-items# (rest items#)] 31 | (cond 32 | (empty? items#) 33 | out-chan# 34 | 35 | (nil? next-item#) 36 | (recur rest-items# out-chan#) 37 | 38 | (async/offer! out-chan# next-item#) 39 | (recur rest-items# out-chan#) 40 | 41 | :else 42 | (let [result-chan# 43 | (loop [] 44 | (let [[put-result# ignored#] (~alts-fn [[out-chan# next-item#] 45 | (async/timeout put-timeout-ms)] 46 | :priority true)] 47 | (cond 48 | (true? put-result#) 49 | out-chan# 50 | 51 | (false? put-result#) 52 | (do (handle-closed-out-chan ~pipeline-impl next-item#) 53 | (reduced out-chan#)) 54 | 55 | (prots/killed? ~pipeline-impl) 56 | (reduced out-chan#) 57 | 58 | :else ;; put timed out 59 | (recur))))] 60 | (if (= result-chan# out-chan#) 61 | (recur rest-items# out-chan#) 62 | result-chan#))))))) 63 | 64 | (defmacro ^:private handle-a-job [mode job] 65 | (let [put-fn (case mode :blocking 'async/>!! :compute 'async/>!)] 66 | `(let [[promise-chan# pipeline-impl# message# state#] ~job] 67 | (~put-fn promise-chan# 68 | (try 69 | (handle-a-message ~mode pipeline-impl# message# state#) 70 | (catch Throwable t# t#))) 71 | (async/close! promise-chan#)))) 72 | 73 | (schema/defn work :- local-schema/Chan 74 | "Start a worker process (on either a green or real thread) 75 | 76 | Takes: 77 | - Mode keyword indicating green or real 78 | - Jobs channel 79 | 80 | Returns: Go-block results channel 81 | 82 | The worker takes jobs off of the jobs channel and uses the 83 | PipelineImpl methods to handle inputs. Successful results are put 84 | on the output-channel. Exceptions are caught and passed back to 85 | the listener on the job's promise channel. If putting on the 86 | output-channel is blocking, the kill-switch is periodically checked 87 | to see if there was an error on another thread." 88 | [mode :- (schema/enum :blocking :compute) 89 | jobs-chan :- local-schema/Chan] 90 | (case mode 91 | :blocking (async/thread 92 | (when-let [job (async/Squarer finished?))) 31 | 32 | (defrecord Sleeper [ignored-context] 33 | PipelineImpl 34 | (handle [_ input-message] 35 | (Thread/sleep 20) 36 | [input-message]) 37 | (finish [_ _] 38 | [])) 39 | 40 | (defrecord Puker [finished? counter] 41 | PipelineImpl 42 | (handle [_ input-message] 43 | (if (>= @counter 2) 44 | (throw (RuntimeException. "Blah!")) 45 | (swap! counter inc)) 46 | [input-message]) 47 | (finish [_ completed?] 48 | (reset! finished? completed?) 49 | [])) 50 | 51 | (defn puker-factory [finished?] 52 | (fn [ignored-context] 53 | (->Puker finished? (atom 0)))) 54 | 55 | (defrecord FinSquarer [finished? accum] 56 | PipelineImpl 57 | (handle [_ input-message] 58 | (swap! accum conj (square input-message)) 59 | []) 60 | (finish [_ completed?] 61 | (reset! finished? completed?) 62 | (if completed? 63 | @accum 64 | []))) 65 | 66 | (defn fin-squarer-factory [finished?] 67 | (fn [ignored-context] 68 | (->FinSquarer finished? (atom [])))) 69 | 70 | (defrecord Duplicator [finished?] 71 | PipelineImpl 72 | (handle [_ input-message] 73 | ;; Notice that nil values gets discarded by the pipeline (it is 74 | ;; not put on the output channel because that would close the 75 | ;; channel). 76 | [input-message input-message nil input-message]) 77 | (finish [_ completed?] 78 | (reset! finished? completed?) 79 | [])) 80 | 81 | (defn duplicator-factory [finished?] 82 | (fn [ignored-context] 83 | (->Duplicator finished?))) 84 | 85 | (deftest ^:unit test-parellel-compute-pipelines 86 | (let [kill-switch (kill-switch/create) 87 | 88 | ;; pipeline: squarer-1 89 | control-chan-1 (async/chan 1) 90 | squarer-1-finished? (atom ::unknown) 91 | in-chan-1 (async/chan) 92 | out-chan-1 (async/chan) 93 | _ (control/send-message control-chan-1 94 | :input-chan in-chan-1 95 | :output-chan out-chan-1 96 | :kill-switch kill-switch 97 | :context {:p 1}) 98 | _ (process/create 3 99 | control-chan-1 100 | (squarer-factory squarer-1-finished?) 101 | :compute) 102 | squarer-1-results (async-utils/go-try 103 | (async-utils/take-all! out-chan-1)) 104 | 105 | ;; pipeline: squarer-2 106 | control-chan-2 (async/chan 1) 107 | squarer-2-finished? (atom ::unknown) 108 | in-chan-2 (async/chan) 109 | out-chan-2 (async/chan) 110 | _ (control/send-message control-chan-2 111 | :input-chan in-chan-2 112 | :output-chan out-chan-2 113 | :kill-switch kill-switch 114 | :context {:p 2}) 115 | _ (process/create 3 116 | control-chan-2 117 | (squarer-factory squarer-2-finished?) 118 | :compute) 119 | squarer-2-results (async-utils/go-try 120 | (async-utils/take-all! out-chan-2))] 121 | 122 | ;; Send input asynchronously 123 | (async/onto-chan in-chan-1 [0 1 2 3 4 5 6 7 8 9]) 124 | (async/onto-chan in-chan-2 [10 11 12 13 14 15 16 17 18 19]) 125 | 126 | (testing "Correct outut for squarer-1" 127 | (is (= (set (async-utils/Sleeper 183 | :blocking) 184 | 185 | ;; pipeline: squarer-2 186 | control-chan-3 (async/chan 1) 187 | squarer-2-finished? (atom ::unknown) 188 | in-chan-3 out-chan-2 189 | out-chan-3 (async/chan) 190 | _ (control/send-message control-chan-3 191 | :input-chan in-chan-3 192 | :output-chan out-chan-3 193 | :kill-switch kill-switch 194 | :context {:p 3}) 195 | _ (process/create 3 196 | control-chan-3 197 | (squarer-factory squarer-2-finished?) 198 | :compute) 199 | 200 | ;; pipeline: duplicator 201 | control-chan-4 (async/chan 1) 202 | duplicator-finished? (atom ::unknown) 203 | in-chan-4 out-chan-3 204 | out-chan-4 (async/chan) 205 | _ (control/send-message control-chan-4 206 | :input-chan in-chan-4 207 | :output-chan out-chan-4 208 | :kill-switch kill-switch 209 | :context {:p 4}) 210 | _ (process/create 3 211 | control-chan-4 212 | (duplicator-factory duplicator-finished?) 213 | :compute) 214 | 215 | chain-results (async-utils/go-try 216 | (async-utils/take-all! out-chan-4 :timeout 500))] 217 | 218 | ;; send input asynchronously 219 | (async/onto-chan in-chan-1 [0 1 2 3 4 5 6 7 8 9]) 220 | 221 | (testing "Correct output for the pipeline chain" 222 | (let [results (async-utils/ (errors kill-switch) first :message)))))) 393 | -------------------------------------------------------------------------------- /test/pipeline/test_utils.clj: -------------------------------------------------------------------------------- 1 | (ns pipeline.test-utils) 2 | 3 | (defn attempt-until 4 | "Copied from staples-sparx/kits rev 67c48c5 here to avoid adding 5 | more deps. License MIT." 6 | [f done?-fn & {:keys [ms-per-loop timeout] 7 | :or {ms-per-loop 1000 8 | timeout 10000}}] 9 | (loop [elapsed (long 0) 10 | result (f)] 11 | (if (or (done?-fn result) 12 | (>= elapsed timeout)) 13 | result 14 | (do 15 | (Thread/sleep ms-per-loop) 16 | (recur (long (+ elapsed ms-per-loop)) (f)))))) 17 | -------------------------------------------------------------------------------- /test/pipeline/test_utils/async.clj: -------------------------------------------------------------------------------- 1 | (ns ^{:doc "Extensions to core.async used only in tests"} 2 | pipeline.test-utils.async 3 | (:require [clojure.core.async :as async] 4 | [clojure.core.async.impl.protocols :as async-proto])) 5 | 6 | (defmacro go-try 7 | "A core.async/go block, with an implicit try...catch. Exceptions are 8 | returned (put onto the go block's result channel)." 9 | [& body] 10 | `(async/go 11 | (try 12 | ~@body 13 | (catch Throwable t# 14 | t#)))) 15 | 16 | (defn throw-err 17 | "Throw element if it is Throwable, otherwise return it" 18 | [element] 19 | (when (instance? Throwable element) 20 | (throw element)) 21 | element) 22 | 23 | (defn ! results-chan results) 40 | (recur (conj results item)))))) 41 | (let [[item from-chan] 42 | (async/alts!! [results-chan (async/timeout timeout-ms)] :priority true)] 43 | (if (not= from-chan results-chan) 44 | (throw (RuntimeException. (format "Timed out after %dms" timeout-ms))) 45 | item)))) 46 | 47 | (defn closed? [chan] 48 | (async-proto/closed? chan)) 49 | --------------------------------------------------------------------------------