├── .gitignore ├── test ├── testfiles │ ├── errecho │ ├── inputdata │ ├── sloop │ └── bytes_0_through_255 └── me │ └── raynes │ ├── conch │ └── low_level_test.clj │ └── conch_test.clj ├── .travis.yml ├── project.clj ├── src └── me │ └── raynes │ ├── conch │ └── low_level.clj │ └── conch.clj └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | pom.xml 2 | .lein* 3 | *jar 4 | target 5 | -------------------------------------------------------------------------------- /test/testfiles/errecho: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | echo "$@" 1>&2 -------------------------------------------------------------------------------- /test/testfiles/inputdata: -------------------------------------------------------------------------------- 1 | we 2 | wear 3 | short 4 | shorts -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | lein: lein2 3 | script: lein2 testall -------------------------------------------------------------------------------- /test/testfiles/sloop: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | while true 4 | do 5 | echo "hi" 6 | sleep 1 7 | done 8 | -------------------------------------------------------------------------------- /test/testfiles/bytes_0_through_255: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Raynes/conch/HEAD/test/testfiles/bytes_0_through_255 -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject me.raynes/conch "0.9.0" 2 | :license {:name "Eclipse Public License - v 1.0" 3 | :url "http://www.eclipse.org/legal/epl-v10.html"} 4 | :url "https://github.com/Raynes/conch" 5 | :description "A better shell-out library for Clojure." 6 | :dependencies [[org.clojure/clojure "1.6.0"]] 7 | :aliases {"testall" ["with-profile" "dev,default:dev,1.5,default:dev,1.4,default" "test"]} 8 | :profiles {:1.5 {:dependencies [[org.clojure/clojure "1.5.0"]]} 9 | :1.4 {:dependencies [[org.clojure/clojure "1.4.0"]]} 10 | :release {:deploy-repositories {"releases" {:url "https://oss.sonatype.org/service/local/staging/deploy/maven2" 11 | :creds :gpg} 12 | "snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" 13 | :creds :gpg}}}} 14 | :repositories {"snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots"}} 15 | :pom-addition [:developers [:developer 16 | [:name "Anthony Grimes"] 17 | [:url "http://blog.raynes.me"] 18 | [:email "i@raynes.me"] 19 | [:timezone "-6"]]]) 20 | -------------------------------------------------------------------------------- /test/me/raynes/conch/low_level_test.clj: -------------------------------------------------------------------------------- 1 | (ns me.raynes.conch.low-level-test 2 | (:use clojure.test 3 | [clojure.java.io :only [file]] 4 | [clojure.string :only [trim split]]) 5 | (:require [me.raynes.conch.low-level :as c])) 6 | 7 | (defn parse-env [env] 8 | (into {} 9 | (for [[k v] (map #(split % #"=") (split env #"\r?\n"))] 10 | [k v]))) 11 | 12 | (deftest proc-test 13 | (testing "proc returns :in, :out, and :err." 14 | (let [p (c/proc "ls")] 15 | (is (instance? java.io.InputStream (:out p))) 16 | (is (instance? java.io.OutputStream (:in p))) 17 | (is (instance? java.io.InputStream (:err p))))) 18 | (testing "proc with :dir executes inside of the directory" 19 | (is (= (.getAbsolutePath (file "test/me")) 20 | (trim (c/stream-to-string (c/proc "pwd" :dir "test/me") :out))))) 21 | (testing "proc with :env executes with env vars set." 22 | (is (= "BAR" 23 | (get (parse-env 24 | (c/stream-to-string 25 | (c/proc "env" :env {"FOO" "BAR"}) 26 | :out)) 27 | "FOO"))))) 28 | 29 | (deftest stream-to-string-test 30 | (testing "output is put in a string and returned" 31 | (is (= "foo\n" (c/stream-to-string (c/proc "echo" "foo") :out))))) 32 | 33 | (deftest exit-code-test 34 | (testing "exit-code blocls until a process exists and returns an exit code." 35 | (is (= 0 (c/exit-code (c/proc "pwd")))))) 36 | 37 | (deftest read-line-test 38 | (is (= "foo" (c/read-line (c/proc "echo" "foo") :out)))) 39 | 40 | (deftest feed-from-test 41 | (testing "Can feed from a reader." 42 | (let [p (c/proc "cat")] 43 | (c/feed-from-string p "foo\n") 44 | (is (= "foo" (c/read-line p :out))) 45 | (c/destroy p)))) 46 | 47 | (deftest stream-to-test 48 | (testing "Can stream to a writer." 49 | (let [writer (java.io.StringWriter.)] 50 | (is (= "foo\n" 51 | (do (c/stream-to (c/proc "echo" "foo") :out writer) 52 | (str writer))))))) 53 | 54 | (deftest exit-code-timeout-test 55 | (testing "Returns :timeout if a timeout and destroy was necessary." 56 | (is (= :timeout (c/exit-code (c/proc "cat") 500)))) 57 | (testing "Exit code is returned if the timeout doesn't get hit." 58 | (is (= 0 (c/exit-code (c/proc "ls") 10000))))) -------------------------------------------------------------------------------- /src/me/raynes/conch/low_level.clj: -------------------------------------------------------------------------------- 1 | (ns me.raynes.conch.low-level 2 | "A simple but flexible library for shelling out from Clojure." 3 | (:refer-clojure :exclude [flush read-line]) 4 | (:require [clojure.java.io :as io]) 5 | (:import (java.util.concurrent TimeUnit TimeoutException))) 6 | 7 | (defn proc 8 | "Spin off another process. Returns the process's input stream, 9 | output stream, and err stream as a map of :in, :out, and :err keys 10 | If passed the optional :dir and/or :env keyword options, the dir 11 | and enviroment will be set to what you specify. If you pass 12 | :verbose and it is true, commands will be printed. If it is set to 13 | :very, environment variables passed, dir, and the command will be 14 | printed. If passed the :clear-env keyword option, then the process 15 | will not inherit its environment from its parent process." 16 | [& args] 17 | (let [[cmd args] (split-with (complement keyword?) args) 18 | args (apply hash-map args) 19 | builder (ProcessBuilder. (into-array String cmd)) 20 | env (.environment builder)] 21 | (when (:clear-env args) 22 | (.clear env)) 23 | (doseq [[k v] (:env args)] 24 | (.put env k v)) 25 | (when-let [dir (:dir args)] 26 | (.directory builder (io/file dir))) 27 | (when (:verbose args) (apply println cmd)) 28 | (when (= :very (:verbose args)) 29 | (when-let [env (:env args)] (prn env)) 30 | (when-let [dir (:dir args)] (prn dir))) 31 | (when (:redirect-err args) 32 | (.redirectErrorStream builder true)) 33 | (let [process (.start builder)] 34 | {:out (.getInputStream process) 35 | :in (.getOutputStream process) 36 | :err (.getErrorStream process) 37 | :process process}))) 38 | 39 | (defn destroy 40 | "Destroy a process." 41 | [process] 42 | (.destroy (:process process))) 43 | 44 | ;; .waitFor returns the exit code. This makes this function useful for 45 | ;; both getting an exit code and stopping the thread until a process 46 | ;; terminates. 47 | (defn exit-code 48 | "Waits for the process to terminate (blocking the thread) and returns 49 | the exit code. If timeout is passed, it is assumed to be milliseconds 50 | to wait for the process to exit. If it does not exit in time, it is 51 | killed (with or without fire)." 52 | ([process] (.waitFor (:process process))) 53 | ([process timeout] 54 | (try 55 | (.get (future (.waitFor (:process process))) timeout TimeUnit/MILLISECONDS) 56 | (catch Exception e 57 | (if (or (instance? TimeoutException e) 58 | (instance? TimeoutException (.getCause e))) 59 | (do (destroy process) 60 | :timeout) 61 | (throw e)))))) 62 | 63 | (defn flush 64 | "Flush the output stream of a process." 65 | [process] 66 | (.flush (:in process))) 67 | 68 | (defn done 69 | "Close the process's output stream (sending EOF)." 70 | [proc] 71 | (-> proc :in .close)) 72 | 73 | (defn stream-to 74 | "Stream :out or :err from a process to an ouput stream. 75 | Options passed are fed to clojure.java.io/copy. They are :encoding to 76 | set the encoding and :buffer-size to set the size of the buffer. 77 | :encoding defaults to UTF-8 and :buffer-size to 1024." 78 | [process from to & args] 79 | (apply io/copy (process from) to args)) 80 | 81 | (defn feed-from 82 | "Feed to a process's input stream with optional. Options passed are 83 | fed to clojure.java.io/copy. They are :encoding to set the encoding 84 | and :buffer-size to set the size of the buffer. :encoding defaults to 85 | UTF-8 and :buffer-size to 1024. If :flush is specified and is false, 86 | the process will be flushed after writing." 87 | [process from & {flush? :flush :or {flush? true} :as all}] 88 | (apply io/copy from (:in process) all) 89 | (when flush? (flush process))) 90 | 91 | (defn stream-to-string 92 | "Streams the output of the process to a string and returns it." 93 | [process from & args] 94 | (with-open [writer (java.io.StringWriter.)] 95 | (apply stream-to process from writer args) 96 | (str writer))) 97 | 98 | ;; The writer that Clojure wraps System/out in for *out* seems to buffer 99 | ;; things instead of writing them immediately. This wont work if you 100 | ;; really want to stream stuff, so we'll just skip it and throw our data 101 | ;; directly at System/out. 102 | (defn stream-to-out 103 | "Streams the output of the process to System/out" 104 | [process from & args] 105 | (apply stream-to process from (System/out) args)) 106 | 107 | (defn feed-from-string 108 | "Feed the process some data from a string." 109 | [process s & args] 110 | (apply feed-from process (java.io.StringReader. s) args)) 111 | 112 | (defn read-line 113 | "Read a line from a process' :out or :err." 114 | [process from] 115 | (binding [*in* (io/reader (from process))] 116 | (clojure.core/read-line))) 117 | -------------------------------------------------------------------------------- /test/me/raynes/conch_test.clj: -------------------------------------------------------------------------------- 1 | (ns me.raynes.conch-test 2 | (:use clojure.test) 3 | (:require [me.raynes.conch :as sh]) 4 | (:import clojure.lang.ExceptionInfo)) 5 | 6 | (deftest output-test 7 | (sh/let-programs [errecho "test/testfiles/errecho"] 8 | (sh/with-programs [echo] 9 | (testing "By default, output is accumulated into a monolitic string" 10 | (is (= "hi\n" (echo "hi")))) 11 | (testing "Output can be a lazy sequence" 12 | (is (= ["hi" "there"] (echo "hi\nthere" {:seq true})))) 13 | (testing "Can redirect output to a file" 14 | (let [output "hi\nthere\n" 15 | testfile (java.io.File/createTempFile "test-output" ".txt")] 16 | (echo "hi\nthere" {:out testfile}) 17 | (is (= output (slurp testfile))) 18 | (errecho "hi\nthere" {:err testfile}) 19 | (is (= output (slurp testfile))))) 20 | (testing "Can redirect output to a callback function" 21 | (let [x (atom []) 22 | ex (atom [])] 23 | (echo "hi\nthere" {:out (fn [line _] (swap! x conj line))}) 24 | (is (= ["hi" "there"] @x)) 25 | (errecho "hi\nthere" {:err (fn [line _] (swap! ex conj line))}) 26 | (is (= ["hi" "there"] @ex)))) 27 | (testing "Can redirect output to a writer" 28 | (let [writer (java.io.StringWriter.)] 29 | (echo "hi" {:out writer}) 30 | (is (= (str writer) "hi\n"))))))) 31 | 32 | (defn input-stream-to-byte-array [is] 33 | "Convert an input stream is to byte array" 34 | (with-open [baos (java.io.ByteArrayOutputStream.)] 35 | (let [ba (byte-array 2000)] 36 | (loop [n (.read is ba 0 2000)] 37 | (when (> n 0) 38 | (.write baos ba 0 n) 39 | (recur (.read is ba 0 2000)))) 40 | (.toByteArray baos)))) 41 | 42 | (defn file-to-bytearray [file] 43 | "Convert a file to a byte array" 44 | (let [is (clojure.java.io/input-stream file)] 45 | (input-stream-to-byte-array is))) 46 | 47 | (deftest output-test-binary 48 | (sh/let-programs [errecho "test/testfiles/errecho"] 49 | (sh/with-programs [cat] 50 | (let [binary-file-path (.getAbsolutePath (java.io.File. "test/testfiles/bytes_0_through_255")) 51 | bytes-0-through-255 (byte-array (map sh/ubyte (range 256)))] 52 | (testing "By default, output is accumulated as a single byte array" 53 | (let [result (cat binary-file-path {:binary true})] 54 | (is (sh/byte-array? result)) 55 | (is (= (seq bytes-0-through-255) (seq result))))) 56 | (testing "Output can be a lazy sequence of byte arrays" 57 | (let [result (cat binary-file-path {:binary true :seq true :buffer 10})] 58 | (is (every? sh/byte-array? result)) 59 | (is (= (seq bytes-0-through-255) (mapcat seq result))))) 60 | (testing "Can redirect output to a file" 61 | (let [temp-file (java.io.File/createTempFile "binary-file" ".bin")] 62 | (cat binary-file-path {:binary true :buffer 10 :out temp-file}) 63 | (let [result (file-to-bytearray temp-file)] 64 | (is (sh/byte-array? result)) 65 | (is (= (seq bytes-0-through-255) (seq result)))))) 66 | (testing "Can redirect output to a callback function. A small buffer will result in a seq of byte arrays." 67 | (let [result (atom [])] 68 | (cat binary-file-path {:binary true :buffer 10 :out (fn [line _] (swap! result conj line))}) 69 | (is (every? sh/byte-array? @result)) 70 | (is (= (seq bytes-0-through-255) (seq (byte-array (mapcat seq @result))))))))))) 71 | 72 | (deftest timeout-test 73 | (binding [sh/*throw* false] 74 | (sh/let-programs [sloop "test/testfiles/sloop"] 75 | (testing "Process exits and doesn't block forever" 76 | (sloop {:timeout 1000})) ; If the test doesn't sit here forever, we have won. 77 | (testing "Accumulate output before process dies from timeout" 78 | ;; We have to test a non-exact value here. We're measuring time in two 79 | ;; different places/languages, so there may be three his on some runs 80 | ;; and two on others. 81 | (is (.startsWith (sloop {:timeout 2000}) "hi\nhi\n")))))) 82 | 83 | (deftest background-test 84 | (testing "Process runs in a future" 85 | (let [f (sh/with-programs [echo] (echo "hi" {:background true}))] 86 | (is (future? f)) 87 | (is (= "hi\n" @f))))) 88 | 89 | (deftest convert-test 90 | (testing "Stringifies args" 91 | (is (= "lol\n" (sh/with-programs [echo] (echo (java.io.File. "lol"))))))) 92 | 93 | (deftest pipe-test 94 | (sh/let-programs [errecho "test/testfiles/errecho"] 95 | (sh/with-programs [echo cat] 96 | (testing "Can pipe the output of one command as the input to another." 97 | (is (= "hi\n" (cat {:in (echo "hi" {:seq true})}))) 98 | (is (= "hi\n" (cat {:in (errecho "hi" {:seq :err})}))) 99 | (is (= "hi\n" (cat (echo "hi" {:seq true})))))))) 100 | 101 | (deftest in-test 102 | (sh/with-programs [echo cat] 103 | (testing "Can input from string" 104 | (is (= "hi" (cat {:in "hi"})))) 105 | (testing "Can input a seq" 106 | (is (= "hi\nthere\n" (cat {:in ["hi" "there"]})))) 107 | (testing "Can input a file" 108 | (is (= "we\nwear\nshort\nshorts" (cat {:in (java.io.File. "test/testfiles/inputdata")})))) 109 | (testing "Can input a reader" 110 | (is (= "we\nwear\nshort\nshorts" (cat {:in (java.io.FileReader. "test/testfiles/inputdata")})))))) 111 | 112 | (deftest exception-test 113 | (sh/with-programs [ls] 114 | (testing "Throws exceptions when *throw* is true." 115 | (is sh/*throw*) 116 | (is (thrown? ExceptionInfo (ls "-2"))) 117 | (is (string? 118 | (try (ls "-2") 119 | (catch ExceptionInfo e 120 | (:stderr (ex-data e)))))) 121 | (testing "But can override" 122 | (is (= "" (ls "-2" {:throw false}))))) 123 | (testing "Can turn it off" 124 | (binding [sh/*throw* false] 125 | (is (not sh/*throw*)) 126 | (is (= "" (ls "-2"))) 127 | (testing "But can override" 128 | (is (thrown? ExceptionInfo (ls "-2" {:throw true})))))))) 129 | 130 | (deftest env-test 131 | (sh/with-programs [env] 132 | (testing "Environment is cleared if a program is called with :clear-env" 133 | (is (= "" (env {:throw true :clear-env true}))) 134 | (is (not= "" (env {:throw true})))))) 135 | -------------------------------------------------------------------------------- /src/me/raynes/conch.clj: -------------------------------------------------------------------------------- 1 | (ns me.raynes.conch 2 | (:require [me.raynes.conch.low-level :as conch] 3 | [clojure.java.io :as io] 4 | [clojure.string :as string]) 5 | (:import java.util.concurrent.LinkedBlockingQueue)) 6 | 7 | (def ^:dynamic *throw* 8 | "If set to false, exit codes are ignored. If true (default), 9 | throw exceptions for non-zero exit codes." 10 | true) 11 | 12 | (defprotocol Redirectable 13 | (redirect [this options k proc])) 14 | 15 | (defn byte? [x] 16 | (and (not (nil? x)) 17 | (= java.lang.Byte (.getClass x)))) 18 | 19 | (defn test-array 20 | [t] 21 | (let [check (type (t []))] 22 | (fn [arg] (instance? check arg)))) 23 | 24 | (def byte-array? 25 | (test-array byte-array)) 26 | 27 | 28 | (defn write-to-writer [writer s is-binary] 29 | (cond 30 | (byte? (first s)) (.write writer (byte-array s)) 31 | (or (not is-binary) 32 | (byte-array? (first s))) (if (char? (first s)) 33 | (.write writer (apply str s)) 34 | (doseq [x s] (.write writer x))))) 35 | 36 | (extend-type java.io.File 37 | Redirectable 38 | (redirect [f options k proc] 39 | (let [s (k proc) 40 | is-binary (:binary options)] 41 | (with-open [writer (if is-binary (io/output-stream f) (java.io.FileWriter. f))] 42 | (write-to-writer writer s is-binary))))) 43 | 44 | (extend-type clojure.lang.IFn 45 | Redirectable 46 | (redirect [f options k proc] 47 | (doseq [buffer (get proc k)] 48 | (f buffer proc)))) 49 | 50 | (extend-type java.io.Writer 51 | Redirectable 52 | (redirect [w options k proc] 53 | (let [s (get proc k)] 54 | (write-to-writer w s (:binary options))))) 55 | 56 | (defn seqify? [options k] 57 | (let [seqify (:seq options)] 58 | (or (= seqify k) 59 | (true? seqify)))) 60 | 61 | (extend-type nil 62 | Redirectable 63 | (redirect [_ options k proc] 64 | (let [seqify (:seq options) 65 | s (k proc)] 66 | (cond 67 | (seqify? options k) s 68 | (byte? (first s)) (byte-array s) 69 | (byte-array? (first s)) (byte-array (mapcat seq s)) 70 | :else (string/join s))))) 71 | 72 | (defprotocol Drinkable 73 | (drink [this proc])) 74 | 75 | (extend-type clojure.lang.ISeq 76 | Drinkable 77 | (drink [s proc] 78 | (with-open [writer (java.io.PrintWriter. (:in proc))] 79 | (binding [*out* writer] 80 | (doseq [x s] 81 | (println x)))) 82 | (conch/done proc))) 83 | 84 | (extend-type java.io.Reader 85 | Drinkable 86 | (drink [r proc] 87 | (conch/feed-from proc r) 88 | (conch/done proc))) 89 | 90 | (extend-type java.io.File 91 | Drinkable 92 | (drink [f proc] 93 | (drink (io/reader f) proc))) 94 | 95 | (extend-type java.lang.String 96 | Drinkable 97 | (drink [s proc] 98 | (conch/feed-from-string proc s) 99 | (conch/done proc))) 100 | 101 | (defn get-drunk [item proc] 102 | (drink 103 | (if (coll? item) 104 | (seq item) 105 | item) 106 | proc)) 107 | 108 | (defn add-proc-args [args options] 109 | (if (seq options) 110 | (apply concat args 111 | (select-keys options 112 | [:redirect-err 113 | :env 114 | :clear-env 115 | :dir])) 116 | args)) 117 | 118 | (defn queue-seq [q] 119 | (lazy-seq 120 | (let [x (.take q)] 121 | (when-not (= x :eof) 122 | (cons x (queue-seq q)))))) 123 | 124 | (defmulti buffer (fn [kind _ _] 125 | (if (number? kind) 126 | :number 127 | kind))) 128 | 129 | (defmethod buffer :number [kind reader binary] 130 | #(try 131 | (let [cbuf (make-array (if binary Byte/TYPE Character/TYPE) kind) 132 | size (.read reader cbuf)] 133 | (when-not (neg? size) 134 | (let [result (if (= size kind) 135 | cbuf 136 | (take size cbuf))] 137 | (if binary 138 | (if (seq? result) (byte-array result) result) 139 | (string/join result))))) 140 | (catch java.io.IOException _))) 141 | 142 | (defn ubyte [val] 143 | (if (>= val 128) 144 | (byte (- val 256)) 145 | (byte val))) 146 | 147 | (defmethod buffer :none [_ reader binary] 148 | #(try 149 | (let [c (.read reader)] 150 | (when-not (neg? c) 151 | (if binary 152 | ;; Return a byte (convert from unsigned value) 153 | (ubyte c) 154 | ;; Return a char 155 | (char c)))) 156 | (catch java.io.IOException _))) 157 | 158 | (defmethod buffer :line [_ reader binary] 159 | #(try 160 | (.readLine reader) 161 | (catch java.io.IOException _))) 162 | 163 | (defn queue-stream [stream buffer-type binary] 164 | (let [queue (LinkedBlockingQueue.) 165 | read-object (if binary stream (io/reader stream))] 166 | (.start 167 | (Thread. 168 | (fn [] 169 | (doseq [x (take-while identity (repeatedly (buffer buffer-type read-object binary)))] 170 | (.put queue x)) 171 | (.put queue :eof)))) 172 | (queue-seq queue))) 173 | 174 | (defn queue-output [proc buffer-type binary] 175 | (assoc proc 176 | :out (queue-stream (:out proc) buffer-type binary) 177 | :err (queue-stream (:err proc) buffer-type binary))) 178 | 179 | (defn compute-buffer [options] 180 | (update-in options [:buffer] 181 | #(if-let [buffer %] 182 | buffer 183 | (if (and (not (:binary options)) 184 | (or (:seq options) 185 | (:pipe options) 186 | (ifn? (:out options)) 187 | (ifn? (:err options)))) 188 | :line 189 | 1024)))) 190 | 191 | (defn exit-exception [verbose] 192 | (throw (ex-info (str "Program returned non-zero exit code " 193 | @(:exit-code verbose)) 194 | verbose))) 195 | 196 | (defn run-command [name args options] 197 | (let [proc (apply conch/proc name (add-proc-args (map str args) options)) 198 | options (compute-buffer options) 199 | {:keys [buffer out in err timeout verbose binary]} options 200 | proc (queue-output proc buffer binary) 201 | exit-code (future (if timeout 202 | (conch/exit-code proc timeout) 203 | (conch/exit-code proc)))] 204 | (when in (future (get-drunk in proc))) 205 | (let [proc-out (future (redirect out options :out proc)) 206 | proc-err (future (redirect err options :err proc)) 207 | proc-out @proc-out 208 | proc-err @proc-err 209 | verbose-out {:proc proc 210 | :exit-code exit-code 211 | :stdout proc-out 212 | :stderr proc-err} 213 | result (cond 214 | verbose verbose-out 215 | (= (:seq options) :err) proc-err 216 | :else proc-out)] 217 | ;; Not using `zero?` here because exit-code can be a keyword. 218 | (if (= 0 @exit-code) 219 | result 220 | (cond (and (contains? options :throw) 221 | (:throw options)) 222 | (exit-exception verbose-out) 223 | 224 | (and (not (contains? options :throw)) 225 | *throw*) 226 | (exit-exception verbose-out) 227 | 228 | :else result))))) 229 | 230 | (defn execute [name & args] 231 | (let [[[options] args] ((juxt filter remove) map? args)] 232 | (if (:background options) 233 | (future (run-command name args options)) 234 | (run-command name args options)))) 235 | 236 | (defn execute [name & args] 237 | (let [end (last args) 238 | in-arg (first (filter #(seq? %) args)) 239 | args (remove #(seq? %) args) 240 | options (when (map? end) end) 241 | args (if options (drop-last args) args) 242 | options (if in-arg (assoc options :in in-arg) options)] 243 | (if (:background options) 244 | (future (run-command name args options)) 245 | (run-command name args options)))) 246 | 247 | (defmacro programs 248 | "Creates functions corresponding to progams on the PATH, named by names." 249 | [& names] 250 | `(do ~@(for [name names] 251 | `(defn ~name [& ~'args] 252 | (apply execute ~(str name) ~'args))))) 253 | 254 | (defn- program-form [prog] 255 | `(fn [& args#] (apply execute ~prog args#))) 256 | 257 | (defn map-nth 258 | "Calls f on every nth element of coll. If start is passed, starts 259 | at that element (counting from zero), otherwise starts with zero." 260 | ([f nth coll] (map-nth f 0 nth coll)) 261 | ([f start nth coll] 262 | (map #(% %2) 263 | (concat (repeat start identity) 264 | (cycle (cons f (repeat (dec nth) identity)))) 265 | coll))) 266 | 267 | (defmacro let-programs 268 | "Like let, but expects bindings to be symbols to strings of paths to 269 | programs." 270 | [bindings & body] 271 | `(let [~@(map-nth #(program-form %) 1 2 bindings)] 272 | ~@body)) 273 | 274 | (defmacro with-programs 275 | "Like programs, but only binds names in the scope of the with-programs call." 276 | [programs & body] 277 | `(let [~@(interleave programs (map (comp program-form str) programs))] 278 | ~@body)) 279 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # conch 2 | 3 | [![Build Status](https://secure.travis-ci.org/Raynes/conch.png)](http://travis-ci.org/Raynes/conch) 4 | 5 | Conch is actually two libraries. The first, `me.raynes.conch.low-level` is a simple low-level 6 | interface to the Java process APIs. The second and more interesting library is 7 | an interface to low-level inspired by the Python 8 | [sh](http://amoffat.github.com/sh/) library. 9 | 10 | The general idea is to be able to call programs just like you would call Clojure 11 | functions. See Usage for examples. 12 | 13 | ## Installation 14 | 15 | In Leiningen: 16 | 17 | [![version](https://clojars.org/me.raynes/conch/latest-version.svg)](https://clojars.org/me.raynes/conch) 18 | 19 | ## Usage 20 | 21 | First of all, let's `require` a few things. 22 | 23 | ```clojure 24 | user> (require '[me.raynes.conch :refer [programs with-programs let-programs] :as sh]) 25 | nil 26 | ``` 27 | 28 | Let's call a program. What should we call? Let's call `echo`, because I 29 | obviously like to hear myself talk. 30 | 31 | ```clojure 32 | user> (programs echo) 33 | #'user/echo 34 | user> (echo "Hi!") 35 | "Hi!\n" 36 | ``` 37 | 38 | Cool! `programs` is a simple macro that takes a number of symbols and creates 39 | functions that calls programs on the PATH with those names. `echo` is now just a 40 | normal function defined with `defn` just like any other. 41 | 42 | `with-programs` and `let-programs` are lexical and specialized versions of 43 | `programs`. `with-programs` is exactly the same as `programs`, only it defines 44 | functions lexically: 45 | 46 | ```clojure 47 | user> (with-programs [ls] (ls)) 48 | "#README.md#\nREADME.md\nclasses\ndocs\nfoo.py\nlib\nlol\npom.xml\npom.xml.asc\nproject.clj\nsrc\ntarget\ntest\ntestfile\n" 49 | user> ls 50 | CompilerException java.lang.RuntimeException: Unable to resolve symbol: ls in 51 | this context, compiling:(NO_SOURCE_PATH:1) 52 | ``` 53 | 54 | `let-programs` is similar, but is useful for when you want to specify a path to 55 | a program that is not on the PATH: 56 | 57 | ```clojure 58 | user> (let-programs [echo "/bin/echo"] (echo "hi!")) 59 | "hi!\n" 60 | ``` 61 | 62 | Bad example since `echo` *is* on the path, but if it wasn't there already it 63 | still would have worked. I promise. 64 | 65 | ### Input 66 | 67 | You can pass input to a program easily: 68 | 69 | ```clojure 70 | user> (programs cat) 71 | #'user/cat 72 | user> (cat {:in "hi"}) 73 | "hi" 74 | user> (cat {:in ["hi" "there"]}) 75 | "hi\nthere\n" 76 | user> (cat {:in (java.io.StringReader. "hi")}) 77 | "hi" 78 | ``` 79 | 80 | `:in` is handled by a protocol and can thus be extended to support other data. 81 | 82 | ### Output 83 | 84 | So, going back to our `ls` example. Of course, `ls` gives us a bunch of 85 | lines. In a lot of cases, we're going to want to process lines individually. We 86 | can do that by telling conch that we want a lazy seq of lines instead of a 87 | monolithic string: 88 | 89 | ```clojure 90 | user> (ls {:seq true}) 91 | ("#README.md#" "README.md" "classes" "docs" "foo.py" "lib" "lol" "pom.xml" 92 | "pom.xml.asc" "project.clj" "src" "target" "test" "testfile") 93 | ``` 94 | 95 | We can also redirect output to other places. 96 | 97 | ```clojure 98 | user> (let [writer (java.io.StringWriter.)] (echo "foo" {:out writer}) (str writer)) 99 | "foo\n" 100 | user> (echo "foo" {:out (java.io.File. "conch")}) 101 | nil 102 | user> (slurp "conch") 103 | "foo\n" 104 | ``` 105 | 106 | And if that wasn't cool enough for you, `:out` is handled by a protocol and thus 107 | can be extended. 108 | 109 | ### Need Moar INNNNPUUUUUUT 110 | 111 | Need the exit code and stuff? Sure: 112 | 113 | ```clojure 114 | user> (echo "foo" {:verbose true}) 115 | {:proc {:out ("foo\n"), :in #, :err (), :process 117 | #}, :exit-code 118 | #, :stdout "foo\n", :stderr ""} 119 | ``` 120 | 121 | ### Timeouts 122 | 123 | ```clojure 124 | user> (sleep "5") 125 | ... yawn ... 126 | ``` 127 | 128 | Tired of waiting for that pesky process to exit? Make it go away! 129 | 130 | ```clojure 131 | user> (sleep "5" {:timeout 2000}) 132 | ... two seconds later ... 133 | ExceptionInfo Program returned non-zero exit code :timeout clojure.core/ex-info (core.clj:4227) 134 | ``` 135 | 136 | Much better. 137 | 138 | ### Exceptions 139 | 140 | Conch can handle exit codes pretty well. You can make it do pretty much whatever 141 | you want in failure scenarios. 142 | 143 | By default, conch throws `ExceptionInfo` exceptions for non-zero exit codes, as 144 | demonstrated here: 145 | 146 | ```clojure 147 | user> (ls "-2") 148 | ExceptionInfo Program returned non-zero exit code 1 clojure.core/ex-info 149 | (core.clj:4227) 150 | ``` 151 | 152 | This exception's data is the same result you'd get by passing the `:verbose` 153 | option: 154 | 155 | ```clojure 156 | user> (ex-data *e) 157 | {:proc {:out (), :in #, :err ("ls: illegal 158 | option -- 2\nusage: ls [-ABCFGHLOPRSTUWabcdefghiklmnopqrstuwx1] [file ...]\n"), :process #}, :exit-code #, :stdout "", :stderr "ls: illegal option -- 2\nusage: ls [-ABCFGHLOPRSTUWabcdefghiklmnopqrstuwx1] [file ...]\n"} 159 | ``` 160 | 161 | You can control this behavior in two ways. The first way is to set 162 | `me.raynes.conch/*throw*` to `false`: 163 | 164 | ```clojure 165 | user> (binding [sh/*throw* false] (ls "-2")) 166 | "" 167 | ``` 168 | 169 | You can also just override whatever `*throw*` is with the `:throw` argument to 170 | the functions themselves: 171 | 172 | ```clojure 173 | user> (ls "-2" {:throw false}) 174 | "" 175 | ``` 176 | 177 | ### Piping 178 | 179 | You can pipe the output of one program as the input to another about how you'd 180 | expect: 181 | 182 | ```clojure 183 | user> (programs grep ps) 184 | #'user/ps 185 | user> (grep "ssh" {:in (ps "-e" {:seq true})}) 186 | " 4554 ?? 0:00.77 /usr/bin/ssh-agent -l\n" 187 | ``` 188 | 189 | These functions also look for a lazy seq arg, so you can get rid of the explicit 190 | `:in` part. 191 | 192 | ```clojure 193 | user> (programs grep ps) 194 | #'user/ps 195 | user> (grep "ssh" (ps "-e" {:seq true})) 196 | " 4554 ?? 0:00.77 /usr/bin/ssh-agent -l\n" 197 | ``` 198 | 199 | ### Buffering 200 | 201 | Conch gets rid of some ugly edge-cases by **always reading process output 202 | immediately when it becomes available**. It buffers this data into a queue that 203 | you consume however you want. This is how returning lazy seqs work. Keep in mind 204 | that if you don't consume data, it is being held in memory. 205 | 206 | You can change how conch buffers data using the `:buffer` key. 207 | 208 | ```clojure 209 | user> (ls {:seq true :buffer 5}) 210 | ("#READ" "ME.md" "#\nREA" "DME.m" "d\ncla" "sses\n" "conch" "\ndocs" "\nfoo." "py\nli" "b\nlol" "\npom." "xml\np" "om.xm" "l.asc" "\nproj" "ect.c" "lj\nsr" "c\ntar" "get\nt" "est\nt" "estfi" "le\n") 211 | user> (ls {:seq true :buffer :none}) 212 | (\# \R \E \A \D \M \E \. \m \d \# \newline \R \E \A \D \M \E \. \m \d \newline 213 | \c \l \a \s \s \e \s \newline \c \o \n \c \h \newline \d \o \c \s \newline \f \o 214 | \o \. \p \y \newline \l \i \b \newline \l \o \l \newline \p \o \m \. \x \m \l 215 | \newline \p \o \m \. \x \m \l \. \a \s \c \newline \p \r \o \j \e \c \t \. \c \l 216 | \j \newline \s \r \c \newline \t \a \r \g \e \t \newline \t \e \s \t \newline \t 217 | \e \s \t \f \i \l \e \newline) 218 | ``` 219 | 220 | Another nice thing gained by the way conch consumes data is that it is able to 221 | kill a process after a timeout and keep whatever data it has already consumed. 222 | 223 | 224 | ### PTY stuff 225 | 226 | PTY stuff seems like it'd be a lot of work and would involve non-Clojure 227 | stuff. If you need a PTY for output, I suggest wrapping your programs in 228 | 'unbuffer' from the expect package. It usually does the trick for unbuffering 229 | program output by making it think a terminal is talking to it. 230 | 231 | ### Hanging 232 | 233 | You might run into an issue where your program finishes after using conch but 234 | does not exit. Conch uses futures under the hood which spin off threads that 235 | stick around for a minute or so after everything else is done. This is an 236 | unfortunate side effect, but futures are necessary to conch's functionality so 237 | I'm not sure there is much I can do about it. 238 | 239 | You can work around this by adding a `(System/exit 0)` call to the end of your program. 240 | 241 | ## Low Level Usage 242 | 243 | The low-level API is available in a separate package: 244 | 245 | ```clojure 246 | (use '[me.raynes.conch.low-level :as sh]) 247 | ``` 248 | 249 | It is pretty simple. You spin off a process with `proc`. 250 | 251 | ```clojure 252 | user=> (def p (sh/proc "cat")) 253 | #'user/p 254 | user=> p 255 | {:out #, :in #, :err #, :process #} 256 | ``` 257 | 258 | When you create a process with `proc`, you get back a map containing the 259 | keys `:out`, `:err`, `:in`, and `:process`. 260 | 261 | * `:out` is the process's stdout. 262 | * `:err` is the process's stderr. 263 | * `:in` is the process's stdin. 264 | * `:process` is the process object itself. 265 | 266 | Conch is more flexible than `clojure.java.shell` because you have direct 267 | access to all of the streams and the process object itself. 268 | 269 | So, now we have a cat process. This is a unix tool. If you 270 | run `cat` with no arguments, it echos whatever you type in. This makes it 271 | perfect for testing input and output. 272 | 273 | Conch defines a few utility functions for streaming output and feeding 274 | input. Since we want to make sure that our input is going to the right 275 | place, let's set up a way to see the output of our process in realtime: 276 | 277 | ```clojure 278 | user=> (future (sh/stream-to-out p :out)) 279 | # 280 | ``` 281 | 282 | The `stream-to-out` function takes a process and either `:out` or `:err` 283 | and streams that to `System/out`. In this case, it has the effect of printing 284 | everything we pipe into our cat process, since our cat process just 285 | outputs whatever we input. 286 | 287 | ```clojure 288 | user=> (sh/feed-from-string p "foo\n") 289 | nil 290 | foo 291 | ``` 292 | 293 | The `feed-from-string` function just feeds a string to the process. It 294 | automatically flushes (which is why this prints immediately) but you can 295 | stop it from doing that by passing `:flush false`. 296 | 297 | I think our cat process has lived long enough. Let's kill it and get its 298 | exit code. We can use the `exit-code` function to get the exit code. 299 | However, since `exit-code` stops the thread and waits for the process to 300 | terminate, we should run it in a future until we actually destroy the 301 | process. 302 | 303 | ```clojure 304 | user=> (def exit (future (sh/exit-code p))) 305 | #'user/exit 306 | ``` 307 | 308 | Now let's kill. R.I.P process. 309 | 310 | ```clojure 311 | user=> (sh/destroy p) 312 | nil 313 | ``` 314 | 315 | And the exit code, which we should be able to obtain now that the 316 | process has been terminated: 317 | 318 | ```clojure 319 | user=> @exit 320 | 0 321 | ``` 322 | 323 | Awesome! Let's go back to `proc` and see what else we can do with it. We 324 | can pass multiple strings to `proc`. The first string will be considered 325 | the executable and the rest of them the arguments to that executable. 326 | 327 | ```clojure 328 | user=> (sh/proc "ls" "-l") 329 | {:out #, :in #, :err #, :process #} 330 | ``` 331 | 332 | ## low-level 333 | 334 | ```clojure 335 | (require '[me.raynes.conch.low-level :as sh]) 336 | ``` 337 | 338 | Here is an easy way to get the output of a one-off process like this as 339 | a string: 340 | 341 | ```clojure 342 | user=> (sh/stream-to-string (sh/proc "ls" "-l") :out) 343 | "total 16\n-rw-r--r-- 1 anthony staff 2545 Jan 24 16:37 README.md\ndrwxr-xr-x 2 anthony staff 68 Jan 19 19:23 classes\ndrwxr-xr-x 3 anthony staff 102 Jan 19 19:23 lib\n-rw-r--r-- 1 anthony staff 120 Jan 20 14:45 project.clj\ndrwxr-xr-x 3 anthony staff 102 Jan 20 14:45 src\ndrwxr-xr-x 3 anthony staff 102 Jan 19 16:36 test\n" 344 | ``` 345 | 346 | Let's print that for readability: 347 | 348 | ```clojure 349 | user=> (print (sh/stream-to-string (sh/proc "ls" "-l") :out)) 350 | total 16 351 | -rw-r--r-- 1 anthony staff 2545 Jan 24 16:37 README.md 352 | drwxr-xr-x 2 anthony staff 68 Jan 19 19:23 classes 353 | drwxr-xr-x 3 anthony staff 102 Jan 19 19:23 lib 354 | -rw-r--r-- 1 anthony staff 120 Jan 20 14:45 project.clj 355 | drwxr-xr-x 3 anthony staff 102 Jan 20 14:45 src 356 | drwxr-xr-x 3 anthony staff 102 Jan 19 16:36 test 357 | nil 358 | ``` 359 | 360 | So, that's the `ls` of the current directory. I ran this REPL in the 361 | conch project directory. We can, of course, pass a directory to `ls` to 362 | get it to list the files in that directory, but that isn't any fun. We 363 | can pass a directory to `proc` itself and it'll run in the context of 364 | that directory. 365 | 366 | ```clojure 367 | user=> (print (sh/stream-to-string (sh/proc "ls" "-l" :dir "lib/") :out)) 368 | total 6624 369 | -rw-r--r-- 1 anthony staff 3390414 Jan 19 19:23 clojure-1.3.0.jar 370 | nil 371 | ``` 372 | 373 | You can also pass a `java.io.File` or anything that can be passed to 374 | `clojure.java.io/file`. 375 | 376 | We can also set environment variables: 377 | 378 | ```clojure 379 | user=> (print (sh/stream-to-string (sh/proc "env" :env {"FOO" "bar"}) :out)) 380 | FOO=bar 381 | nil 382 | ``` 383 | 384 | The map passed to `:env` completely replaces any other environment 385 | variables that were in place. 386 | 387 | Finally, there a couple of low-level functions for streaming from and 388 | feeding to a process. They are `stream-to` and `feed-from`. These 389 | functions are what the utility functions are built off of, and you can 390 | probably use them to stream to and feed from your own special places. 391 | 392 | You might want to fire off a program that listens for input until EOF. 393 | In these cases, you can feed it data for as long as you want and just 394 | tell it when you are done. Let's use `pygmentize` as an example: 395 | 396 | ```clojure 397 | user=> (def proc (sh/proc "pygmentize" "-fhtml" "-lclojure")) 398 | #'user/proc 399 | user=> (sh/feed-from-string proc "(+ 3 3)") 400 | nil 401 | user=> (sh/done proc) 402 | nil 403 | user=> (sh/stream-to-string proc :out) 404 | "
(+ 3 3)\n
\n" 405 | ``` 406 | 407 | When we call `done`, it closes the process's output stream which is 408 | like sending EOF. The process processes its input and then puts it on 409 | its input stream where we read it with `stream-to-string`. 410 | 411 | ### Other options 412 | 413 | All of conch's streaming and feeding functions (including the lower 414 | level ones) pass all of their keyword options to `clojure.java.io/copy`. 415 | It can take an `:encoding` and `:buffer-size` option. Guess what they 416 | do. 417 | 418 | ### Key names 419 | 420 | You might notice that the map that `proc` returns is mapped like so: 421 | 422 | * `:in` -> output stream 423 | * `:out` -> input stream 424 | 425 | I did this because swapping them feels counterintuitive. The output 426 | stream is what you put `:in` to and the input stream is what you pull 427 | `:out` from. 428 | 429 | ## License 430 | 431 | Copyright (C) 2012 Anthony Grimes 432 | 433 | Distributed under the Eclipse Public License, the same as Clojure. 434 | --------------------------------------------------------------------------------