├── examples ├── .gitignore ├── README.md ├── deps.edn ├── src │ └── cognitect │ │ └── transcriptor │ │ └── examples │ │ └── generators.clj └── atoms.repl ├── transcriptor.clj └── README.md /examples/.gitignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # transcriptor examples 2 | 3 | Install the [Clojure CLI](https://clojure.org/guides/deps_and_cli) and 4 | eval the .repl files one form at a time at the Clojure REPL. 5 | -------------------------------------------------------------------------------- /examples/deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"] 2 | :deps 3 | {com.cognitect/transcriptor {:mvn/version "0.1.5"}, 4 | org.clojure/test.check {:mvn/version "0.9.0"} 5 | org.clojure/clojure {:mvn/version "1.9.0-beta1"}}} 6 | -------------------------------------------------------------------------------- /examples/src/cognitect/transcriptor/examples/generators.clj: -------------------------------------------------------------------------------- 1 | (ns cognitect.transcriptor.examples.generators 2 | (:require [clojure.spec.alpha :as s])) 3 | 4 | (defn one-of 5 | [s] 6 | (-> (s/exercise s) last first)) 7 | -------------------------------------------------------------------------------- /examples/atoms.repl: -------------------------------------------------------------------------------- 1 | ;; compare with https://github.com/clojure/clojure/blob/a8f1c6436a8bfe181b0a00cf9e44845dcbbb63ee/test/clojure/test_clojure/atoms.clj 2 | (require '[cognitect.transcriptor :as xr :refer (check!)] 3 | '[cognitect.transcriptor.examples.generators :refer (one-of)] 4 | '[clojure.spec.alpha :as s] 5 | '[clojure.spec.test.alpha :as test]) 6 | 7 | (comment "exact match test") 8 | @(def a (atom 0)) 9 | (swap-vals! a inc) 10 | (check! #{[0 1]}) 11 | 12 | (comment "property test") 13 | (s/def ::inced (s/and (s/tuple int? int?) 14 | (fn [[before after]] (= (inc before) after)))) 15 | @(def a (atom (one-of int?))) 16 | (swap-vals! a inc) 17 | (check! ::inced) 18 | 19 | (comment "generative exploration") 20 | (s/fdef swap-inc-atom 21 | :args (s/cat :initial (s/int-in -1000000 1000000)) 22 | :ret ::inced 23 | :fn (fn [{:keys [args ret]}] 24 | (let [initial (:initial args)] 25 | (= (first ret) initial)))) 26 | (defn swap-inc-atom 27 | [x] 28 | (swap-vals! (atom x) inc)) 29 | (s/exercise-fn #'swap-inc-atom) 30 | 31 | (comment "generative testing") 32 | (-> (test/check `swap-inc-atom) 33 | (test/summarize-results)) 34 | ;; this needs a helper fn: 35 | (check! #{{:total 1 :check-passed 1}}) 36 | -------------------------------------------------------------------------------- /transcriptor.clj: -------------------------------------------------------------------------------- 1 | ;; Copyright (c) Cognitect, Inc. 2 | ;; All rights reserved. 3 | 4 | (ns cognitect.transcriptor 5 | (:import clojure.lang.LineNumberingPushbackReader java.io.File) 6 | (:require 7 | [clojure.core.server :as server] 8 | [clojure.java.io :as io] 9 | [clojure.main :as main] 10 | [clojure.pprint :as pp] 11 | [clojure.spec.alpha :as s] 12 | [clojure.string :as str])) 13 | 14 | (defmacro check! 15 | "Checks v (defaults to *1) against spec, throwing on failure. Returns nil." 16 | ([spec] 17 | `(check! ~spec *1)) 18 | ([spec v] 19 | `(let [v# ~v] 20 | (when-not (s/valid? ~spec v#) 21 | (let [ed# (s/explain-data ~spec v#) 22 | err# (ex-info (str "Transcript assertion failed! " (with-out-str (s/explain-out ed#))) 23 | ed#)] 24 | (throw err#)))))) 25 | 26 | (def ^:private ^:dynamic *exit-items* ::disabled) 27 | 28 | (defn on-exit 29 | "If running inside a call to repl, queue f to run when REPL exits." 30 | [f] 31 | (when-not (= ::disabled *exit-items*) 32 | (swap! *exit-items* conj f)) 33 | nil) 34 | 35 | (defn repl 36 | "Transcript-making REPL. Like a normal REPL except: 37 | 38 | - pretty prints inputs 39 | - prints '=> ' before pretty printing results 40 | - throws on exception 41 | 42 | Not intended for interactive use -- point this at a file to 43 | produce a transcript as-if a human had performed the 44 | interactions." 45 | [] 46 | (let [cl (.getContextClassLoader (Thread/currentThread))] 47 | (.setContextClassLoader (Thread/currentThread) (clojure.lang.DynamicClassLoader. cl))) 48 | (let [request-prompt (Object.) 49 | request-exit (Object.) 50 | read-eval-print 51 | (fn [] 52 | (let [read-eval *read-eval* 53 | input (main/with-read-known (server/repl-read request-prompt request-exit))] 54 | (if (#{request-prompt request-exit} input) 55 | input 56 | (do 57 | (pp/pprint input) 58 | (let [value (binding [*read-eval* read-eval] (eval input))] 59 | (set! *3 *2) (set! *2 *1) (set! *1 value) 60 | (print "=> ") 61 | (pp/pprint value) 62 | (println))))))] 63 | (main/with-bindings 64 | (binding [*exit-items* (atom ())] 65 | (try 66 | (loop [] 67 | (let [value (read-eval-print)] 68 | (when-not (identical? value request-exit) 69 | (recur)))) 70 | (finally 71 | (doseq [item @*exit-items*] 72 | (item)))))))) 73 | 74 | (defn- repl-on 75 | [r] 76 | (with-open [rdr (LineNumberingPushbackReader. (io/reader r))] 77 | (binding [*source-path* (str r) *in* rdr] 78 | (repl)))) 79 | 80 | (def script-counter (atom 0)) 81 | 82 | (defn run 83 | "Run script through transcripting repl in a tearoff namespace." 84 | [script] 85 | (let [ns (symbol (str "cognitect.transcriptor.t_" (swap! script-counter inc)))] 86 | (prn (list 'comment {:transcript (str script) :namespace ns})) 87 | (binding [*ns* *ns*] 88 | (in-ns ns) 89 | (clojure.core/use 'clojure.core) 90 | (repl-on script)))) 91 | 92 | (defn repl-files 93 | "Returns a seq of .repl files under dir" 94 | [dir] 95 | (->> (io/file dir) 96 | file-seq 97 | (filter (fn [^java.io.File f] 98 | (and (.isFile f) 99 | (str/ends-with? (.getName f) ".repl")))) 100 | (map #(.getPath ^File %)))) 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # transcriptor 2 | 3 | Convert REPL interactions into example-based tests. 4 | 5 | # Using 6 | 7 | transcriptor is in the Maven Central repository. 8 | 9 | Clojure deps.edn: 10 | 11 | com.cognitect/transcriptor {:mvn/version "0.1.5"} 12 | 13 | lein project.clj: 14 | 15 | [com.cognitect/transcriptor "0.1.5"] 16 | 17 | # Problem 18 | 19 | Testing frameworks often introduce their own abstractions for 20 | e.g. evaluation order, data validation, reporting, scope, code reuse, 21 | state, and lifecycle. In my experience, these abstractions are 22 | *always* needlessly different from (and inferior to) related 23 | abstractions provided by the language itself. 24 | 25 | Adapting an already-working REPL interaction to satisfy such testing 26 | abstractions is a waste of time, and it throws away the intermediate 27 | REPL results that are valuable in diagnosing a problem. 28 | 29 | So transcriptor aims to do *less*, and impose the bare minimum of 30 | cognitive load needed to convert a REPL interaction into a test. The 31 | entire API is four functions: 32 | 33 | * `xr/run` runs a REPL script and produces a transcript 34 | * `check!` validates the last returned value against a Clojure spec 35 | * `xr/on-exit` lets you register cleanup code to run after `xr/run` completes 36 | * `xr/repl-files` finds the `.repl` files in a directory tree 37 | 38 | # Approach 39 | 40 | Work at the REPL. Whenever you want to convert a chunk of work into a 41 | test, just copy it into a file with a .repl suffix. You can later call 42 | `xr/run` on a REPL file: 43 | 44 | (require '[cognitect.transcriptor :as xr :refer (check!)]) 45 | (xr/run "your-file-name-here.repl") 46 | 47 | `run` launches a REPL that consumes all forms in the file passed 48 | in. `run` will 49 | 50 | * isolate execution in a one-off namespace whose name is printed to 51 | stdout. (If the script fails, you can enter this namespace and poke around.) 52 | * pretty print every evaluation result, providing a transcript as 53 | if you had repeated the REPL interactions by hand. 54 | 55 | # Evaluation Order 56 | 57 | Clojure language (REPL) semantics. 58 | 59 | # Validation 60 | 61 | Clojure language semantics plus one function. 62 | 63 | transcriptor includes a single validation form, `check!`, that will 64 | check an argument (by default `*1`) against a provided spec, throwing 65 | an exception if the error does not match: 66 | 67 | (+ 1 1) 68 | (check! even?) 69 | 70 | Exceptions are failures and unwind the stack back to the call to `xr/run`. 71 | 72 | # Reporting 73 | 74 | Read clojure.spec error data directly, or pipe it to an error reporter 75 | or visualizer of your choice. 76 | 77 | # Code reuse 78 | 79 | Clojure language semantics. Write functions in namespaces and have 80 | .repl scripts require them as needed. 81 | 82 | # Scope 83 | 84 | Clojure language semantics. `def` vars that you need. 85 | 86 | # State 87 | 88 | Clojure language semantics. (For testing code with nontrivial state I 89 | recommend simulation-based testing instead). 90 | 91 | # Lifecycle 92 | 93 | Clojure language semantics plus one function. 94 | 95 | The `xr/on-exit` function is a no-op outside `xr/run`. Inside, it will 96 | queue a function that will be called after the REPL exits. 97 | 98 | # Test Automation 99 | 100 | Clojure language semantics plus one function. 101 | 102 | * `xr/repl-files` returns a seq of .repl files under a directory root, suitable 103 | for passing to `xr/run`. 104 | 105 | # Test Repeatability 106 | 107 | Clojure language semantics. 108 | 109 | # Keep Dumb Tests Ugly 110 | 111 | Tests that want an exact value match can use a Clojure set as a spec: 112 | 113 | (+ 1 2) 114 | (check! #{3}) ;; duh 115 | 116 | This is ugly by design, as an inducement to test properties instead of 117 | specifics. 118 | 119 | # License 120 | 121 | Eclipse Public License, same as Clojure. 122 | https://www.eclipse.org/legal/epl-v10.html 123 | 124 | # Contributing 125 | 126 | Please open a Github issue if your have feedback. 127 | --------------------------------------------------------------------------------