├── .gitignore ├── bin ├── kaocha ├── update_toc.ed ├── run_feature ├── update_feature ├── check_features ├── generate_toc ├── update_project_list.ed ├── check_feature ├── release └── prep_release ├── .dir-locals.el ├── resources └── kaocha │ ├── clojure_logo.png │ └── default_config.edn ├── fixtures ├── b-tests │ └── finn │ │ └── finn_test.clj ├── a-tests │ └── foo │ │ └── bar_test.clj ├── tests.edn ├── d-tests │ └── ddd │ │ ├── bar_test.clj │ │ ├── exception_outside_is.clj │ │ └── foo_test.clj ├── custom_config.edn ├── with_failing.edn ├── c-tests │ └── foo │ │ └── hello_test.clj └── with_exception.edn ├── src └── kaocha │ ├── jit.clj │ ├── random.clj │ ├── plugin │ ├── bindings.clj │ ├── alpha │ │ ├── info.clj │ │ └── xfail.clj │ ├── print_invocations.clj │ ├── hooks.clj │ ├── randomize.clj │ ├── version_filter.clj │ ├── capture_output.clj │ ├── profiling.clj │ └── notifier.clj │ ├── type.clj │ ├── shellwords.clj │ ├── test.clj │ ├── load.clj │ ├── history.clj │ ├── output.clj │ ├── type │ ├── clojure │ │ └── test.clj │ ├── var.clj │ └── ns.clj │ ├── core_ext.clj │ ├── stacktrace.clj │ ├── result.clj │ ├── report │ └── progress.clj │ ├── classpath.clj │ ├── assertions.clj │ ├── plugin.clj │ ├── matcher_combinators.clj │ ├── hierarchy.clj │ └── specs.clj ├── test ├── shared │ └── kaocha │ │ ├── test_factories.clj │ │ ├── test_plugins.clj │ │ ├── test_util.clj │ │ ├── test_helper.clj │ │ └── integration_helpers.clj ├── unit │ └── kaocha │ │ ├── monkey_patch_test.clj │ │ ├── runner_test.clj │ │ ├── repl_test.clj │ │ ├── history_test.clj │ │ ├── plugin_test.clj │ │ ├── core_ext_test.clj │ │ ├── random_test.clj │ │ ├── plugin │ │ ├── hooks_test.clj │ │ ├── version_filter_test.clj │ │ └── randomize_test.clj │ │ ├── api_test.clj │ │ ├── hierarchy_test.clj │ │ ├── testable_test.clj │ │ ├── output_test.clj │ │ ├── type │ │ ├── clojure │ │ │ └── test_test.clj │ │ └── ns_test.clj │ │ └── result_test.clj ├── features │ ├── plugins │ │ ├── bindings.feature │ │ ├── capture_output.feature │ │ ├── hooks_plugin.feature │ │ ├── version_filter.feature │ │ └── notifier_plugin.feature │ ├── command_line │ │ ├── fail_fast.feature │ │ ├── print_config.feature │ │ ├── reporter.feature │ │ └── suite_names.feature │ ├── pending.feature │ ├── filtering │ │ ├── focusing.feature │ │ ├── skipping.feature │ │ ├── focus_meta.feature │ │ └── skip_meta.feature │ └── clojure_test │ │ └── assertions.feature └── step_definitions │ └── kaocha_integration.clj ├── tests.edn ├── doc ├── plugins │ ├── bindings.md │ ├── capture_output.md │ ├── hooks_plugin.md │ ├── version_filter.md │ └── notifier_plugin.md ├── command_line │ ├── fail_fast.md │ ├── print_config.md │ ├── reporter.md │ └── suite_names.md ├── pending.md ├── 01_introduction.md ├── filtering │ ├── focusing.md │ ├── skipping.md │ ├── focus_meta.md │ └── skip_meta.md ├── clojure_test │ └── assertions.md ├── 06_focusing_and_skipping.md ├── 05_running_kaocha_repl.md ├── 08_plugins.md ├── 02_installing.md ├── 07_watch_mode.md └── 04_running_kaocha_cli.md ├── deps.edn ├── .circleci └── config.yml └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | .nrepl-port 3 | target 4 | repl 5 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | clojure -A:dev:test -m kaocha.runner "$@" 3 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((cider-clojure-cli-global-options . "-A:dev:test")))) 2 | -------------------------------------------------------------------------------- /bin/update_toc.ed: -------------------------------------------------------------------------------- 1 | r README.md 2 | 1 3 | / docs-toc/+,/\/docs-toc/- d 4 | - 5 | .r ! bin/generate_toc 6 | w 7 | q -------------------------------------------------------------------------------- /resources/kaocha/clojure_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magnars/kaocha/master/resources/kaocha/clojure_logo.png -------------------------------------------------------------------------------- /bin/run_feature: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin/kaocha --config-file features/$1/tests.edn `cat features/$1/args` "${@:2}" 4 | -------------------------------------------------------------------------------- /bin/update_feature: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin/run_feature "$@" 1>features/$1/out 2>features/$1/err 4 | echo $? > features/$1/exit 5 | -------------------------------------------------------------------------------- /fixtures/b-tests/finn/finn_test.clj: -------------------------------------------------------------------------------- 1 | (ns finn.finn-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest the-test 5 | (is true)) 6 | -------------------------------------------------------------------------------- /fixtures/a-tests/foo/bar_test.clj: -------------------------------------------------------------------------------- 1 | (ns foo.bar-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (def a-var :ok) 5 | 6 | (deftest a-test 7 | (is true)) 8 | -------------------------------------------------------------------------------- /fixtures/tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :a 3 | :test-paths ["fixtures/a-tests"]} 4 | {:id :b 5 | :test-paths ["fixtures/b-tests"]}]} 6 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/bar_test.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.bar-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest test-1 5 | (is true)) 6 | 7 | (deftest test-2 8 | (is true)) 9 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/exception_outside_is.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.exception-outside-is 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest exception-outside-is-test 5 | (throw (Exception. "booo"))) 6 | -------------------------------------------------------------------------------- /src/kaocha/jit.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.jit) 2 | 3 | (defmacro jit 4 | "Just in time loading of dependencies." 5 | [sym] 6 | `(do 7 | (require '~(symbol (namespace sym))) 8 | (find-var '~sym))) 9 | -------------------------------------------------------------------------------- /fixtures/custom_config.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :foo 3 | :test-paths ["test/foo"]}] 4 | :reporter [kaocha.report.progress/report] 5 | :color? false 6 | :fail-fast? true 7 | :plugins [:kaocha.plugin.alpha/xfail]} 8 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/foo_test.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.foo-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest test-1 5 | (is false)) 6 | 7 | (deftest test-2 8 | (is true)) 9 | 10 | (deftest test-3 11 | (is (throw (Exception. "fail!")))) 12 | -------------------------------------------------------------------------------- /fixtures/with_failing.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :a 3 | :test-paths ["fixtures/a-tests"]} 4 | 5 | {:id :c 6 | :test-paths ["fixtures/c-tests"]} 7 | 8 | {:id :b 9 | :test-paths ["fixtures/b-tests"]}]} 10 | -------------------------------------------------------------------------------- /bin/check_features: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RESULT=0 4 | 5 | for feature in $(find features -maxdepth 2 -mindepth 2 -printf "%P\n" | sort) 6 | do 7 | echo $feature 8 | bin/check_feature $feature 9 | if [ $? -ne 0 ]; then 10 | RESULT=1 11 | fi 12 | done 13 | 14 | exit $RESULT 15 | -------------------------------------------------------------------------------- /fixtures/c-tests/foo/hello_test.clj: -------------------------------------------------------------------------------- 1 | (ns foo.hello-test 2 | (:require [clojure.test :as t])) 3 | 4 | (t/deftest pass-1 5 | (t/is true)) 6 | 7 | (t/deftest pass-2 8 | (t/is true)) 9 | 10 | (t/deftest fail-1 11 | (t/is true) 12 | (t/is false) 13 | (t/is true)) 14 | 15 | (t/deftest pass-3 16 | (t/is true)) 17 | -------------------------------------------------------------------------------- /bin/generate_toc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | def uslug(s) 4 | s.downcase.gsub(/[^\p{L}\p{N}]+/, ' ').gsub(/[\p{Z}\s]+/, '-') 5 | end 6 | 7 | Dir['doc/**/*.md'].sort.each do |f| 8 | title = File.read(f).each_line.first.sub(/^#+/, '').strip 9 | puts "- [%s](https://cljdoc.org/d/lambdaisland/kaocha/CURRENT/doc/%s)" % [title, uslug(title)] 10 | end 11 | -------------------------------------------------------------------------------- /fixtures/with_exception.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :inside-is 3 | :test-paths ["fixtures/d-tests"] 4 | :kaocha.filter/focus [ddd.foo-test/test-3]} 5 | {:id :outside-is 6 | :test-paths ["fixtures/d-tests"] 7 | :ns-patterns [""] 8 | :kaocha.filter/focus [ddd.exception-outside-is]}]} 9 | -------------------------------------------------------------------------------- /src/kaocha/random.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.random) 2 | 3 | (defn rng [seed] 4 | (let [rng (java.util.Random. seed)] 5 | (fn [& _] (.nextInt rng)))) 6 | 7 | (defn randomize-tests [seed ns->tests] 8 | (let [next-int (rng seed) 9 | ns->tests' (->> ns->tests 10 | (map (fn [[k v]] [k (sort-by str v)])) 11 | (sort-by first))] 12 | (->> ns->tests' 13 | (map (fn [[k v]] [k (sort-by next-int v)])) 14 | (sort-by next-int)))) 15 | -------------------------------------------------------------------------------- /src/kaocha/plugin/bindings.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.bindings 2 | (:require [kaocha.plugin :refer [defplugin]])) 3 | 4 | (defplugin kaocha.plugin/bindings 5 | "Set dynamic bindings during the test run." 6 | (wrap-run [run test-plan] 7 | (if-let [bindings (:kaocha/bindings test-plan)] 8 | (let [bindings (into {} (map (fn [[k v]] [(find-var k) v])) bindings)] 9 | (fn [& args] 10 | (try 11 | (push-thread-bindings bindings) 12 | (apply run args) 13 | (finally 14 | (pop-thread-bindings))))) 15 | run))) 16 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_factories.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-factories 2 | (:require [kaocha.specs] 3 | [kaocha.api] 4 | [clojure.spec.alpha :as s] 5 | [clojure.spec.gen.alpha :as gen] 6 | [kaocha.config :as config])) 7 | 8 | (defn var-testable [m] 9 | (let [testable (gen/generate (s/gen :kaocha.type/var))] 10 | (merge testable 11 | {:kaocha.testable/type :kaocha.type/var 12 | :kaocha.testable/desc (name (:kaocha.testable/id testable))} 13 | m))) 14 | 15 | (defn test-plan [m] 16 | (merge 17 | (config/default-config) 18 | 19 | m)) 20 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:plugins [:kaocha.plugin/profiling 3 | :kaocha.plugin/print-invocations 4 | :kaocha.plugin/hooks 5 | :kaocha.plugin/notifier] 6 | 7 | :tests [{:id :unit 8 | :test-paths ["test/shared" 9 | "test/unit"]} 10 | {:id :integration 11 | :type :kaocha.type/cucumber 12 | :test-paths ["test/shared" 13 | "test/features"] 14 | :cucumber/glue-paths ["test/step_definitions"]}] 15 | 16 | :kaocha.hooks/pre-load [kaocha.assertions/load-assertions]} 17 | -------------------------------------------------------------------------------- /resources/kaocha/default_config.edn: -------------------------------------------------------------------------------- 1 | {:kaocha/reporter [kaocha.report/dots] 2 | :kaocha/color? true 3 | :kaocha/fail-fast? false 4 | :kaocha/plugins [:kaocha.plugin/randomize 5 | :kaocha.plugin/filter 6 | :kaocha.plugin/capture-output] 7 | :kaocha/tests [{:kaocha.testable/type :kaocha.type/clojure.test 8 | :kaocha.testable/id :unit 9 | :kaocha/ns-patterns ["-test$"] 10 | :kaocha/source-paths ["src"] 11 | :kaocha/test-paths ["test"] 12 | :kaocha.filter/skip-meta [:kaocha/skip]}]}} 13 | -------------------------------------------------------------------------------- /src/kaocha/type.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type 2 | "Utilities for writing new test types." 3 | (:require [clojure.test :as t] 4 | [kaocha.result :as result])) 5 | 6 | (def initial-counters {:test 0, :pass 0, :fail 0, :error 0, :pending 0}) 7 | 8 | (def ^:dynamic *intermediate-report*) 9 | 10 | (defmacro with-report-counters 11 | {:style/indent [0]} 12 | [& body] 13 | `(binding [*intermediate-report* (or (some-> t/*report-counters* deref) ~initial-counters)] 14 | (binding [t/*report-counters* (ref *intermediate-report*)] 15 | ~@body))) 16 | 17 | (defn report-count [] 18 | (result/diff-test-result *intermediate-report* @t/*report-counters*)) 19 | -------------------------------------------------------------------------------- /test/unit/kaocha/monkey_patch_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.monkey-patch-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.monkey-patch :as monkey-patch] 4 | [kaocha.plugin :as plugin])) 5 | 6 | ;; Note: this test is doing some somersaults to avoid crashing into clojure.test 7 | ;; while it is itself running. 8 | (deftest pre-report-hook-is-used 9 | (let [result (atom nil)] 10 | (binding [plugin/*current-chain* [{:kaocha.hooks/pre-report (fn [event] (assoc event :been-here true))}]] 11 | (with-redefs [monkey-patch/report (fn [event] (reset! result event))] 12 | (monkey-patch/do-report {:type :pass}))) 13 | (is (:been-here @result)))) 14 | -------------------------------------------------------------------------------- /src/kaocha/shellwords.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.shellwords) 2 | 3 | (def shellwords-pattern #"[^\s'\"]+|[']([^']*)[']|[\"]([^\"]*)[\"]") 4 | 5 | ;; ported from cucumber.runtime.ShellWords, which was ported from Ruby 6 | (defn shellwords [cmdline] 7 | (let [matcher (re-matcher shellwords-pattern cmdline)] 8 | (loop [res []] 9 | (if (.find matcher) 10 | (recur 11 | (if-let [word (.group matcher 1)] 12 | (conj res word) 13 | (let [word (.group matcher)] 14 | (if (and (= \" (first word)) 15 | (= \" (last word))) 16 | (conj res (subs word 1 (dec (count word)))) 17 | (conj res word))))) 18 | res)))) 19 | -------------------------------------------------------------------------------- /test/unit/kaocha/runner_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.runner-test 2 | (:require [clojure.test :as t :refer :all] 3 | [kaocha.runner :as runner] 4 | [kaocha.test-util :refer [with-out-err]])) 5 | 6 | (defn -main [& args] 7 | (with-out-err 8 | (apply #'runner/-main* args))) 9 | 10 | (deftest main-test 11 | (testing "--test-help" 12 | (let [{:keys [out err result]} (-main "--test-help")] 13 | (is (re-find #"USAGE:" out)) 14 | (is (= 0 result)))) 15 | 16 | (testing "unknown command line options" 17 | (let [{:keys [out err result]} (-main "--foo")] 18 | (is (= -1 result)) 19 | (is (re-find #"Unknown option: \"--foo\"\n" out)) 20 | (is (re-find #"USAGE:" out))))) 21 | -------------------------------------------------------------------------------- /test/unit/kaocha/repl_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.repl-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.repl :as repl] 4 | [kaocha.config :as config])) 5 | 6 | (deftest config-test 7 | (is (match? 8 | '{:kaocha/tests [{:kaocha.testable/id :foo 9 | :kaocha/test-paths ["test/foo"]}] 10 | :kaocha/reporter [kaocha.report.progress/report] 11 | :kaocha/color? false 12 | :kaocha/fail-fast? true 13 | :kaocha/plugins [:kaocha.plugin/randomize 14 | :kaocha.plugin/filter 15 | :kaocha.plugin/capture-output 16 | :kaocha.plugin.alpha/xfail]} 17 | 18 | (repl/config {:config-file "fixtures/custom_config.edn"})))) 19 | -------------------------------------------------------------------------------- /bin/update_project_list.ed: -------------------------------------------------------------------------------- 1 | /-- projects --/+,/-- \/projects --/- d 2 | i 3 | | Project | CI | Docs | Release | Coverage | 4 | |---------|----|------|---------|----------| 5 | . 6 | .r ! for X in kaocha kaocha-cljs kaocha-cucumber kaocha-junit-xml kaocha-cloverage kaocha-boot deep-diff; do echo "| [${X}](https://github.com/lambdaisland/${X}) | [![CircleCI](https://circleci.com/gh/lambdaisland/${X}.svg?style=svg)](https://circleci.com/gh/lambdaisland/${X}) | [![cljdoc badge](https://cljdoc.org/badge/lambdaisland/${X})](https://cljdoc.org/d/lambdaisland/${X}) | [![Clojars Project](https://img.shields.io/clojars/v/lambdaisland/${X}.svg)](https://clojars.org/lambdaisland/${X}) | [![codecov](https://codecov.io/gh/lambdaisland/${X}/branch/master/graph/badge.svg)](https://codecov.io/gh/lambdaisland/${X}) |" ; done 7 | wq 8 | -------------------------------------------------------------------------------- /test/unit/kaocha/history_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.history-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.history :refer :all])) 4 | 5 | (deftest clojure-test-summary-test 6 | (is (= (clojure-test-summary []) 7 | {:type :summary :test 0, :pass 0, :fail 0, :error 0})) 8 | 9 | (is (= (clojure-test-summary [{:type :pass}]) 10 | {:type :summary :test 0, :pass 1, :fail 0, :error 0})) 11 | 12 | (is (= (clojure-test-summary [{:type :begin-test-var} 13 | {:type :pass} 14 | {:type :fail} 15 | {:type :pass} 16 | {:type :begin-test-ns} 17 | {:type :error}]) 18 | {:type :summary :test 1, :pass 2, :fail 1, :error 1}))) 19 | -------------------------------------------------------------------------------- /test/features/plugins/bindings.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Bindings 2 | 3 | The `bindings` plugin allows you to configure dynamic var bindings from 4 | `tests.edn`, so they are visible during the test run. 5 | 6 | Scenario: Implementing a hook 7 | Given a file named "tests.edn" with: 8 | """ clojure 9 | #kaocha/v1 10 | {:plugins [:kaocha.plugin/bindings] 11 | :kaocha/bindings {my.foo-test/*a-var* 456}} 12 | """ 13 | And a file named "test/my/foo_test.clj" with: 14 | """ clojure 15 | (ns my.foo-test 16 | (:require [clojure.test :refer :all])) 17 | 18 | (def ^:dynamic *a-var* 123) 19 | 20 | (deftest var-test 21 | (is (= 456 *a-var*))) 22 | """ 23 | When I run `bin/kaocha` 24 | Then the output should contain: 25 | """ 26 | 1 tests, 1 assertions, 0 failures. 27 | """ 28 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin-test 2 | (:require [kaocha.plugin :as plugin] 3 | [clojure.test :refer :all] 4 | [kaocha.test-util :as util] 5 | [kaocha.output :as output]) 6 | (:import (clojure.lang ExceptionInfo))) 7 | 8 | (deftest missing-plugin-test 9 | (is (thrown-with-msg? ExceptionInfo 10 | #"Couldn't load plugin :kaocha.missing.plugin/gone" 11 | (plugin/load-all [:kaocha.missing.plugin/gone]))) 12 | (is (= {:err "ERROR: Couldn't load plugin :kaocha.missing.plugin/gone\n" :out "" :result nil} 13 | (binding [output/*colored-output* false] 14 | (util/with-out-err 15 | (try 16 | (plugin/load-all [:kaocha.missing.plugin/gone]) 17 | (catch ExceptionInfo e 18 | nil))))))) 19 | -------------------------------------------------------------------------------- /doc/plugins/bindings.md: -------------------------------------------------------------------------------- 1 | # Plugin: Bindings 2 | 3 | The `bindings` plugin allows you to configure dynamic var bindings from 4 | `tests.edn`, so they are visible during the test run. 5 | 6 | ## Implementing a hook 7 | 8 | - Given a file named "tests.edn" with: 9 | 10 | ``` clojure 11 | #kaocha/v1 12 | {:plugins [:kaocha.plugin/bindings] 13 | :kaocha/bindings {my.foo-test/*a-var* 456}} 14 | ``` 15 | 16 | 17 | - And a file named "test/my/foo_test.clj" with: 18 | 19 | ``` clojure 20 | (ns my.foo-test 21 | (:require [clojure.test :refer :all])) 22 | 23 | (def ^:dynamic *a-var* 123) 24 | 25 | (deftest var-test 26 | (is (= 456 *a-var*))) 27 | ``` 28 | 29 | 30 | - When I run `bin/kaocha` 31 | 32 | - Then the output should contain: 33 | 34 | ``` nil 35 | 1 tests, 1 assertions, 0 failures. 36 | ``` 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /bin/check_feature: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin/run_feature "$@" 1>/tmp/kaocha_actual_out 2>/tmp/kaocha_actual_err 4 | 5 | KAOCHA_EXIT_CODE=$? 6 | RESULT=0 7 | 8 | if [[ -f "features/$1/out" ]]; then 9 | colordiff -u "features/$1/out" /tmp/kaocha_actual_out 10 | else 11 | colordiff -u <(echo -n) /tmp/kaocha_actual_out 12 | fi 13 | 14 | if [ $? -ne 0 ]; then 15 | RESULT=1 16 | fi 17 | 18 | if [[ -f "features/$1/err" ]]; then 19 | colordiff -u "features/$1/err" /tmp/kaocha_actual_err 20 | else 21 | colordiff -u <(echo -n) /tmp/kaocha_actual_err 22 | fi 23 | 24 | if [[ -f "features/$1/exit" ]]; then 25 | EXPECTED_EXIT_CODE=$(cat features/$1/exit) 26 | if [[ $KAOCHA_EXIT_CODE -ne $EXPECTED_EXIT_CODE ]]; then 27 | echo "Bad exit code, expected " $EXPECTED_EXIT_CODE ", got " $KAOCHA_EXIT_CODE "." 28 | RESULT=1 29 | fi 30 | fi 31 | 32 | if [ $? -ne 0 ]; then 33 | RESULT=1 34 | fi 35 | 36 | exit $RESULT 37 | -------------------------------------------------------------------------------- /src/kaocha/plugin/alpha/info.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.alpha.info 2 | (:require [kaocha.api :as api] 3 | [kaocha.plugin :refer [defplugin]] 4 | [kaocha.testable :as testable] 5 | [slingshot.slingshot :refer [throw+]])) 6 | 7 | (def cli-opts 8 | [[nil "--print-test-ids" "Print all known test ids"] 9 | [nil "--print-env" "Print Clojure and Java version."]]) 10 | 11 | (defplugin kaocha.plugin.alpha/info 12 | (cli-options [opts] 13 | (into opts cli-opts)) 14 | 15 | (main [config] 16 | (cond 17 | (:print-test-ids (:kaocha/cli-options config)) 18 | (let [test-plan (api/test-plan config)] 19 | (doseq [test (testable/test-seq test-plan)] 20 | (println (:kaocha.testable/id test))) 21 | (throw+ {:kaocha/early-exit 0})) 22 | 23 | (:print-env (:kaocha/cli-options config)) 24 | (do 25 | (println "Clojure" (clojure-version)) 26 | (println (System/getProperty "java.runtime.name") (System/getProperty "java.runtime.version")) 27 | config) 28 | 29 | :else 30 | config))) 31 | -------------------------------------------------------------------------------- /doc/command_line/fail_fast.md: -------------------------------------------------------------------------------- 1 | # CLI: `--fail-fast` option 2 | 3 | Kaocha by default runs all tests it can find, providing a final summary on 4 | failures and errors when all tests have finished. With the `--fail-fast` 5 | option the test run will be interrupted as soon as a single failure or error 6 | has occured. Afterwards a summary of the test run so far is printed. 7 | 8 | ## Failing fast 9 | 10 | - Given a file named "test/my/project/fail_fast_test.clj" with: 11 | 12 | ``` clojure 13 | (ns my.project.fail-fast-test 14 | (:require [clojure.test :refer :all])) 15 | 16 | (deftest test-1 17 | (is true)) 18 | 19 | (deftest test-2 20 | (is true) 21 | (is false) 22 | (is true)) 23 | 24 | (deftest test-3 25 | (is true)) 26 | ``` 27 | 28 | 29 | - When I run `bin/kaocha --fail-fast` 30 | 31 | - Then the exit-code should be 1 32 | 33 | - And the output should contain: 34 | 35 | ``` nil 36 | [(..F)] 37 | 38 | FAIL in my.project.fail-fast-test/test-2 (fail_fast_test.clj:9) 39 | expected: false 40 | actual: false 41 | 2 tests, 3 assertions, 1 failures. 42 | ``` 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /test/features/command_line/fail_fast.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: `--fail-fast` option 2 | 3 | Kaocha by default runs all tests it can find, providing a final summary on 4 | failures and errors when all tests have finished. With the `--fail-fast` 5 | option the test run will be interrupted as soon as a single failure or error 6 | has occured. Afterwards a summary of the test run so far is printed. 7 | 8 | Scenario: Failing fast 9 | Given a file named "test/my/project/fail_fast_test.clj" with: 10 | """clojure 11 | (ns my.project.fail-fast-test 12 | (:require [clojure.test :refer :all])) 13 | 14 | (deftest test-1 15 | (is true)) 16 | 17 | (deftest test-2 18 | (is true) 19 | (is false) 20 | (is true)) 21 | 22 | (deftest test-3 23 | (is true)) 24 | """ 25 | When I run `bin/kaocha --fail-fast` 26 | Then the exit-code should be 1 27 | And the output should contain: 28 | """ 29 | [(..F)] 30 | 31 | FAIL in my.project.fail-fast-test/test-2 (fail_fast_test.clj:9) 32 | expected: false 33 | actual: false 34 | 2 tests, 3 assertions, 1 failures. 35 | """ 36 | -------------------------------------------------------------------------------- /src/kaocha/plugin/alpha/xfail.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.alpha.xfail 2 | "This plugin inverses fail & pass for tests marked with the ^:kaocha/xfail 3 | metadata. A failing test can be marked with this metadata, and will now be 4 | considered passing. Once the test passes again, it fails. 5 | 6 | Note that this current implementation inverses each assertion in turn, so if 7 | you have multiple assertions in a xfail test, then they all must fail for the 8 | test to pass." 9 | (:require [kaocha.plugin :as plugin] 10 | [kaocha.testable :as testable] 11 | [kaocha.hierarchy :as hierarchy])) 12 | 13 | (plugin/defplugin kaocha.plugin.alpha/xfail 14 | (post-load [test-plan] 15 | (update 16 | test-plan 17 | :kaocha/reporter 18 | (fn [reporter] 19 | (fn [event] 20 | (reporter 21 | (if (-> event :kaocha/testable ::testable/meta :kaocha/xfail) 22 | (cond 23 | (hierarchy/fail-type? event) 24 | (assoc event :type :pass) 25 | 26 | (= (:type event) :pass) 27 | (assoc event :type :fail) 28 | 29 | :else 30 | event) 31 | event))))))) 32 | -------------------------------------------------------------------------------- /src/kaocha/test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test 2 | "Work in progress. 3 | 4 | This is intended to eventually be a drop-in replacement for clojure.test, with 5 | some improvements. 6 | 7 | - deftest will kick-off a test run unless one is already in progress, for easy 8 | in-buffer eval 9 | " 10 | (:require [clojure.test :as t] 11 | [kaocha.testable :as testable] 12 | [kaocha.api :as api])) 13 | 14 | (defmacro deftest 15 | {:doc (:doc (meta #'clojure.test/deftest)) 16 | :arglists (:arglists (meta #'clojure.test/deftest))} 17 | [name & body] 18 | (if kaocha.api/*active?* 19 | `(clojure.test/deftest ~name ~@body) 20 | `(do 21 | (let [var# (clojure.test/deftest ~name ~@body)] 22 | (or (find-ns 'kaocha.repl) (require 'kaocha.repl)) 23 | ((find-var 'kaocha.repl/run) var#))))) 24 | 25 | (defmacro is 26 | {:doc (:doc (meta #'clojure.test/is)) 27 | :arglists (:arglists (meta #'clojure.test/is))} 28 | [& args] 29 | `(clojure.test/is ~@args)) 30 | 31 | (defmacro testing 32 | {:doc (:doc (meta #'clojure.test/testing)) 33 | :arglists (:arglists (meta #'clojure.test/testing))} 34 | [& args] 35 | `(clojure.test/testing ~@args)) 36 | -------------------------------------------------------------------------------- /doc/command_line/print_config.md: -------------------------------------------------------------------------------- 1 | # CLI: Print the Kaocha configuration 2 | 3 | A Kaocha test run starts with building up a Kaocha configuration map, based on 4 | default values, the contents of `tests.edn`, command line flags, and active 5 | plugins. 6 | 7 | Debugging issues with Kaocha often starts with inspecting the configuration, 8 | which is why a `--print-config` flag is provided. This builds up the 9 | configuration from any available sources, runs it through any active plugins, 10 | and then pretty prints the result, an EDN map. 11 | 12 | ## Using `--print-config` 13 | 14 | - When I run `bin/kaocha --print-config` 15 | 16 | - Then the output should contain: 17 | 18 | ``` clojure 19 | {:kaocha.plugin.randomize/randomize? false, 20 | :kaocha/reporter [kaocha.report/dots], 21 | :kaocha/color? false, 22 | :kaocha/fail-fast? false, 23 | ``` 24 | 25 | 26 | - And the output should contain: 27 | 28 | ``` clojure 29 | :kaocha/tests 30 | [{:kaocha.testable/type :kaocha.type/clojure.test, 31 | :kaocha.testable/id :unit, 32 | :kaocha/ns-patterns ["-test$"], 33 | :kaocha/source-paths ["src"], 34 | :kaocha/test-paths ["test"], 35 | :kaocha.filter/skip-meta [:kaocha/skip]}], 36 | ``` 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/kaocha/load.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.load 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [kaocha.core-ext :refer :all] 4 | [kaocha.classpath :as classpath] 5 | [kaocha.testable :as testable] 6 | [clojure.java.io :as io] 7 | [lambdaisland.tools.namespace.find :as ctn-find] 8 | [kaocha.output :as output])) 9 | 10 | (def clj ctn-find/clj) 11 | (def cljs ctn-find/cljs) 12 | 13 | (defn ns-match? [ns-patterns ns-sym] 14 | (some #(re-find % (name ns-sym)) ns-patterns)) 15 | 16 | (defn find-test-nss [test-paths ns-patterns & [platform]] 17 | (sequence (comp 18 | (map io/file) 19 | (map #(ctn-find/find-namespaces-in-dir % platform)) 20 | cat 21 | (filter (partial ns-match? ns-patterns))) 22 | test-paths)) 23 | 24 | (defn load-test-namespaces [testable ns-testable-fn & [platform]] 25 | (let [test-paths (:kaocha/test-paths testable) 26 | ns-patterns (map regex (:kaocha/ns-patterns testable)) 27 | ns-names (find-test-nss test-paths ns-patterns platform) 28 | testables (map ns-testable-fn ns-names)] 29 | (assoc testable 30 | :kaocha.test-plan/tests 31 | (testable/load-testables testables)))) 32 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | 3 | :deps 4 | {org.clojure/clojure {:mvn/version "1.10.0"} 5 | org.clojure/spec.alpha {:mvn/version "0.2.176"} 6 | org.clojure/tools.cli {:mvn/version "0.4.1"} 7 | lambdaisland/tools.namespace {:mvn/version "0.0-234"} 8 | lambdaisland/deep-diff {:mvn/version "0.0-25"} 9 | org.tcrawley/dynapath {:mvn/version "1.0.0"} 10 | slingshot {:mvn/version "0.12.2"} 11 | hawk {:mvn/version "0.2.11"} 12 | expound {:mvn/version "0.7.2"} 13 | orchestra {:mvn/version "2018.12.06-2"} 14 | aero {:mvn/version "1.1.3"} 15 | progrock {:mvn/version "0.1.2"} 16 | meta-merge {:mvn/version "1.0.0"}} 17 | 18 | :aliases 19 | {:test 20 | {:extra-deps {org.clojure/test.check {:mvn/version "0.10.0-alpha3"} 21 | lambdaisland/kaocha-cucumber {:mvn/version "0.0-36"} 22 | lambdaisland/kaocha-cloverage {:mvn/version "0.0-22"} 23 | nubank/matcher-combinators {:mvn/version "0.4.2"} 24 | akvo/fs {:mvn/version "20180904-152732.6dad3934"}}} 25 | 26 | :dev 27 | {}}} 28 | -------------------------------------------------------------------------------- /test/unit/kaocha/core_ext_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.core-ext-test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :refer :all] 4 | [kaocha.core-ext :refer :all])) 5 | 6 | (deftest regex?-test 7 | (is (regex? #"foo")) 8 | (is (not (regex? "foo")))) 9 | 10 | (deftest exception?-test 11 | (is (exception? (Exception. "oh no"))) 12 | (is (not (exception? (Throwable. "oh no"))))) 13 | 14 | (deftest error?-test 15 | (is (error? (Error. "oh no"))) 16 | (is (not (error? (Exception. "oh no")))) 17 | (is (not (error? (Throwable. "oh no"))))) 18 | 19 | (deftest throwable?-test 20 | (is (throwable? (Error. "oh no"))) 21 | (is (throwable? (Exception. "oh no"))) 22 | (is (throwable? (Throwable. "oh no")))) 23 | 24 | (deftest ns?-test 25 | (is (ns? *ns*)) 26 | (is (not (ns? {})))) 27 | 28 | (deftest regex-test 29 | (is (= "ok" (re-find (regex "[ko]+") "--ok--"))) 30 | (is (= "ok" (re-find (regex #"[ko]+") "--ok--"))) 31 | (is (= "ok" (re-find (regex "o" "k") "--ok--"))) 32 | (is (thrown? clojure.lang.ExceptionInfo (regex 123) ""))) 33 | 34 | (deftest mapply-test 35 | (let [f (fn [& {:as opts}] 36 | {:opts opts})] 37 | (is (= {:opts {:foo :bar :abc :xyz}} 38 | (mapply f {:foo :bar :abc :xyz}))))) 39 | -------------------------------------------------------------------------------- /doc/plugins/capture_output.md: -------------------------------------------------------------------------------- 1 | # Plugin: Capture output 2 | 3 | Kaocha has a plugin which will capture all output written to stdout or stderr 4 | during the test run. When tests pass this output is hidden, when they fail the 5 | output is made visible to help understand the problem. 6 | 7 | This plugin is loaded by default, but can be disabled with `--no-capture-output` 8 | 9 | ## Show output of failing test 10 | 11 | - Given a file named "test/sample_test.clj" with: 12 | 13 | ``` clojure 14 | (ns sample-test 15 | (:require [clojure.test :refer :all])) 16 | 17 | (deftest stdout-pass-test 18 | (println "You peng zi yuan fang lai") 19 | (is (= :same :same))) 20 | 21 | (deftest stdout-fail-test 22 | (println "Bu yi le hu?") 23 | (is (= :same :not-same))) 24 | ``` 25 | 26 | 27 | - When I run `bin/kaocha` 28 | 29 | - Then the output should contain: 30 | 31 | ``` nil 32 | FAIL in sample-test/stdout-fail-test (sample_test.clj:10) 33 | Expected: 34 | :same 35 | Actual: 36 | -:same +:not-same 37 | ╭───── Test output ─────────────────────────────────────────────────────── 38 | │ Bu yi le hu? 39 | ╰───────────────────────────────────────────────────────────────────────── 40 | 2 tests, 2 assertions, 1 failures. 41 | ``` 42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /test/features/command_line/print_config.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: Print the Kaocha configuration 2 | 3 | A Kaocha test run starts with building up a Kaocha configuration map, based on 4 | default values, the contents of `tests.edn`, command line flags, and active 5 | plugins. 6 | 7 | Debugging issues with Kaocha often starts with inspecting the configuration, 8 | which is why a `--print-config` flag is provided. This builds up the 9 | configuration from any available sources, runs it through any active plugins, 10 | and then pretty prints the result, an EDN map. 11 | 12 | Scenario: Using `--print-config` 13 | When I run `bin/kaocha --print-config` 14 | Then the output should contain: 15 | """ clojure 16 | {:kaocha.plugin.randomize/randomize? false, 17 | :kaocha/reporter [kaocha.report/dots], 18 | :kaocha/color? false, 19 | :kaocha/fail-fast? false, 20 | """ 21 | And the output should contain: 22 | """ clojure 23 | :kaocha/tests 24 | [{:kaocha.testable/type :kaocha.type/clojure.test, 25 | :kaocha.testable/id :unit, 26 | :kaocha/ns-patterns ["-test$"], 27 | :kaocha/source-paths ["src"], 28 | :kaocha/test-paths ["test"], 29 | :kaocha.filter/skip-meta [:kaocha/skip]}], 30 | """ 31 | -------------------------------------------------------------------------------- /bin/release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | if [[ ! -z "$(git status --porcelain)" ]]; then 6 | echo "Repo not clean. 7 | 8 | Status: 9 | $(git status --short) 10 | 11 | Diff: 12 | $(git diff)" 13 | 14 | exit 1 15 | fi 16 | 17 | if [[ -f tests.edn ]]; then 18 | bin/kaocha 19 | fi 20 | 21 | if [[ -d features ]]; then 22 | bin/check_features 23 | fi 24 | 25 | VERSION="0.0-$(git rev-list --count HEAD)" 26 | 27 | bin/prep_release 28 | git push --tags 29 | 30 | echo "// 31 | s/.*<\/version>/$VERSION<\/version> 32 | 33 | // 34 | // 35 | s/.*<\/tag>/$(git rev-parse HEAD)<\/tag> 36 | 37 | // 38 | s/.*<\/Git-Revision>/$(git rev-parse HEAD)<\/Git-Revision> 39 | wq" | ed pom.xml 40 | 41 | mvn deploy 42 | 43 | echo "1i 44 | # Unreleased 45 | 46 | ## Added 47 | 48 | ## Fixed 49 | 50 | ## Changed 51 | 52 | . 53 | wq" | ed CHANGELOG.md 54 | 55 | git add CHANGELOG.md pom.xml 56 | git commit -m 'Add CHANGELOG placeholders + update versions in pom.xml' 57 | git push 58 | 59 | echo "Building cljdoc" 60 | 61 | PROJECT=$(git remote get-url origin | sed 's/.*lambdaisland\/\([\.a-z-]\+\)\.git/\1/') 62 | curl -v -XPOST https://cljdoc.org/api/request-build2 -d project=lambdaisland/$PROJECT -d version=$VERSION 2>&1 | grep Location 63 | -------------------------------------------------------------------------------- /src/kaocha/history.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.history 2 | (:require [clojure.test :as t] 3 | [kaocha.hierarchy :as hierarchy])) 4 | 5 | (def ^:dynamic *history* nil) 6 | 7 | (defmulti track :type :hierarchy #'hierarchy/hierarchy) 8 | 9 | (defmethod track :default [m] (swap! *history* conj m)) 10 | 11 | (defmethod track :kaocha/fail-type [m] 12 | (swap! *history* conj (assoc m 13 | :testing-contexts t/*testing-contexts* 14 | :testing-vars t/*testing-vars*)) ) 15 | 16 | (defmethod track :error [m] 17 | (swap! *history* conj (assoc m 18 | :testing-contexts t/*testing-contexts* 19 | :testing-vars t/*testing-vars*))) 20 | 21 | (defn clojure-test-summary 22 | ([] 23 | (clojure-test-summary @*history*)) 24 | ([history] 25 | (reduce 26 | (fn [m {type :type :as event}] 27 | (cond 28 | (some #{type} [:pass :error :kaocha/pending]) (update m type inc) 29 | (hierarchy/isa? type :kaocha/begin-test) (update m :test inc) 30 | (hierarchy/fail-type? event) (update m :fail inc) 31 | :else m)) 32 | {:type :summary 33 | :test 0 34 | :pass 0 35 | :fail 0 36 | :error 0} 37 | history))) 38 | -------------------------------------------------------------------------------- /test/features/plugins/capture_output.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Capture output 2 | 3 | Kaocha has a plugin which will capture all output written to stdout or stderr 4 | during the test run. When tests pass this output is hidden, when they fail the 5 | output is made visible to help understand the problem. 6 | 7 | This plugin is loaded by default, but can be disabled with `--no-capture-output` 8 | 9 | Scenario: Show output of failing test 10 | Given a file named "test/sample_test.clj" with: 11 | """ clojure 12 | (ns sample-test 13 | (:require [clojure.test :refer :all])) 14 | 15 | (deftest stdout-pass-test 16 | (println "You peng zi yuan fang lai") 17 | (is (= :same :same))) 18 | 19 | (deftest stdout-fail-test 20 | (println "Bu yi le hu?") 21 | (is (= :same :not-same))) 22 | """ 23 | When I run `bin/kaocha` 24 | Then the output should contain: 25 | """ 26 | FAIL in sample-test/stdout-fail-test (sample_test.clj:10) 27 | Expected: 28 | :same 29 | Actual: 30 | -:same +:not-same 31 | ╭───── Test output ─────────────────────────────────────────────────────── 32 | │ Bu yi le hu? 33 | ╰───────────────────────────────────────────────────────────────────────── 34 | 2 tests, 2 assertions, 1 failures. 35 | """ 36 | -------------------------------------------------------------------------------- /src/kaocha/output.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.output 2 | (:require [kaocha.jit :refer [jit]])) 3 | 4 | (def ^:dynamic *colored-output* true) 5 | 6 | (def ESC \u001b) 7 | 8 | (def colors 9 | {:black (str ESC "[30m") 10 | :red-bg (str ESC "[41m") 11 | :red (str ESC "[31m") 12 | :green (str ESC "[32m") 13 | :yellow (str ESC "[33m") 14 | :blue (str ESC "[34m") 15 | :magenta (str ESC "[35m") 16 | :cyan (str ESC "[36m") 17 | :white (str ESC "[37m") 18 | :underline (str ESC "[4m") 19 | :reset (str ESC "[m")}) 20 | 21 | (defn colored [color string] 22 | (if *colored-output* 23 | (str (get colors color) string (:reset colors)) 24 | string)) 25 | 26 | (defn warn [& args] 27 | (binding [*out* *err*] 28 | (println (apply str (colored :yellow "WARNING: ") args)))) 29 | 30 | (defn error [& args] 31 | (binding [*out* *err*] 32 | (println (apply str (colored :red "ERROR: ") args)))) 33 | 34 | (defn printer [& [opts]] 35 | ((jit lambdaisland.deep-diff/printer) (merge {:print-color *colored-output*} opts))) 36 | 37 | (defn print-doc 38 | ([doc] 39 | (print-doc doc (printer))) 40 | ([doc printer] 41 | ((jit fipp.engine/pprint-document) doc {:width (:width printer)}))) 42 | 43 | (defn format-doc 44 | ([doc] 45 | (format-doc doc (printer))) 46 | ([doc printer] 47 | ((jit puget.printer/format-doc) printer doc))) 48 | -------------------------------------------------------------------------------- /test/unit/kaocha/random_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.random-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.random :refer :all])) 4 | 5 | (deftest rand-ints-test 6 | (is (= (take 10 (repeatedly (rng 321098))) 7 | [-141996321 1985580023 -305308934 1158906095 -1212597759 -42859192 -98240991 1350981438 31847656 -294128715])) 8 | 9 | (is (= (take 10 (repeatedly (rng 1))) 10 | [-1155869325 431529176 1761283695 1749940626 892128508 155629808 1429008869 -1465154083 -138487339 -1242363800]))) 11 | 12 | (deftest randomize-tests-test 13 | (is (= (randomize-tests 1 '{first.ns [first.var1 first.var2 first.var3] 14 | other.ns [other.var1 other.var2 other.var3] 15 | third.ns [third.var1 third.var2 third.var3]}) 16 | '([first.ns (first.var2 first.var1 first.var3)] 17 | [other.ns (other.var1 other.var2 other.var3)] 18 | [third.ns (third.var3 third.var2 third.var1)]))) 19 | 20 | (is (= (randomize-tests 431 '{first.ns [first.var1 first.var2 first.var3] 21 | other.ns [other.var1 other.var2 other.var3] 22 | third.ns [third.var1 third.var2 third.var3]}) 23 | '([other.ns (other.var3 other.var2 other.var1)] 24 | [third.ns (third.var1 third.var3 third.var2)] 25 | [first.ns (first.var3 first.var2 first.var1)])))) 26 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/hooks_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.hooks-test 2 | (:require [kaocha.test :refer :all] 3 | [kaocha.plugin.hooks :as hooks] 4 | [kaocha.testable :as testable] 5 | [kaocha.test-util :as util] 6 | [kaocha.plugin :as plugin] 7 | [clojure.test :as t])) 8 | 9 | (def ^:dynamic *inside-wrap-run?* false) 10 | 11 | (deftest wrap-run-test 12 | (let [run (fn [] 13 | (print "inside-wrap-run?:" *inside-wrap-run?*)) 14 | wrap-run (fn [run] 15 | (fn [& args] 16 | (binding [*inside-wrap-run?* true] 17 | (apply run args)))) 18 | test-plan {:kaocha.hooks/wrap-run [wrap-run]}] 19 | (binding [plugin/*current-chain* [hooks/hooks-hooks]] 20 | (let [run' (plugin/run-hook :kaocha.hooks/wrap-run run test-plan)] 21 | (is (= "inside-wrap-run?: true" 22 | (with-out-str (run')))))))) 23 | 24 | (deftest pre-report-test 25 | (is (match? {:report [{:type :fail 26 | :went-through-hook? true}]} 27 | (let [test-plan {:kaocha.hooks/pre-report 28 | [(fn [m] (assoc m :went-through-hook? true))]}] 29 | (util/with-test-ctx {} 30 | (binding [plugin/*current-chain* [hooks/hooks-hooks] 31 | testable/*test-plan* test-plan] 32 | (t/do-report {:type :fail}))))))) 33 | -------------------------------------------------------------------------------- /test/features/pending.feature: -------------------------------------------------------------------------------- 1 | Feature: Marking tests as pending 2 | 3 | Pending tests are tests that are not yet implemented, or that need fixing, and 4 | that you don't want to forget about. Pending tests are similar to skipped 5 | tests (see the section on "Filtering"), in that the runner will skip over them 6 | without trying to run them. 7 | 8 | The difference is that pending tests are explicitly reported in the test 9 | result. At the end of each test run you get to see the number of pending 10 | tests, followed by a list of their test ids and file/line information. This 11 | constant reminder is there to make sure pending tests are not left 12 | unaddressed. 13 | 14 | Add the `^:kaocha/pending` metadata to a test to mark it as pending. The 15 | metadata check is done inside the Kaocha runner itself, not in the specific 16 | test type implementation, so this metadata is supported on any test type that 17 | allows setting metadata tags, including ClojureScript and Cucumber tests. 18 | 19 | Scenario: Marking a test as pending 20 | Given a file named "test/sample/sample_test.clj" with: 21 | """clojure 22 | (ns sample.sample-test 23 | (:require [clojure.test :refer :all])) 24 | 25 | (deftest ^:kaocha/pending my-test) 26 | """ 27 | When I run `bin/kaocha` 28 | Then the output should contain: 29 | """ 30 | [(P)] 31 | 1 tests, 0 assertions, 1 pending, 0 failures. 32 | 33 | PENDING sample.sample-test/my-test (sample/sample_test.clj:4) 34 | """ 35 | -------------------------------------------------------------------------------- /doc/pending.md: -------------------------------------------------------------------------------- 1 | # Marking tests as pending 2 | 3 | Pending tests are tests that are not yet implemented, or that need fixing, and 4 | that you don't want to forget about. Pending tests are similar to skipped 5 | tests (see the section on "Filtering"), in that the runner will skip over them 6 | without trying to run them. 7 | 8 | The difference is that pending tests are explicitly reported in the test 9 | result. At the end of each test run you get to see the number of pending 10 | tests, followed by a list of their test ids and file/line information. This 11 | constant reminder is there to make sure pending tests are not left 12 | unaddressed. 13 | 14 | Add the `^:kaocha/pending` metadata to a test to mark it as pending. The 15 | metadata check is done inside the Kaocha runner itself, not in the specific 16 | test type implementation, so this metadata is supported on any test type that 17 | allows setting metadata tags, including ClojureScript and Cucumber tests. 18 | 19 | ## Marking a test as pending 20 | 21 | - Given a file named "test/sample/sample_test.clj" with: 22 | 23 | ``` clojure 24 | (ns sample.sample-test 25 | (:require [clojure.test :refer :all])) 26 | 27 | (deftest ^:kaocha/pending my-test) 28 | ``` 29 | 30 | 31 | - When I run `bin/kaocha` 32 | 33 | - Then the output should contain: 34 | 35 | ``` nil 36 | [(P)] 37 | 1 tests, 0 assertions, 1 pending, 0 failures. 38 | 39 | PENDING sample.sample-test/my-test (sample/sample_test.clj:4) 40 | ``` 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_plugins.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-plugins 2 | (:require [kaocha.plugin :as plugin] 3 | [kaocha.output :as output] 4 | [clojure.test :as t] 5 | [clojure.string :as str] 6 | [kaocha.hierarchy :as hierarchy])) 7 | 8 | (hierarchy/derive! ::unexpected-error :kaocha/known-key) 9 | (hierarchy/derive! ::unexpected-error :kaocha/fail-type) 10 | 11 | (hierarchy/derive! ::unexpected-warning :kaocha/known-key) 12 | (hierarchy/derive! ::unexpected-warning :kaocha/fail-type) 13 | 14 | (plugin/defplugin ::capture-warn+error 15 | "Turn errors and warning messages into test failures." 16 | (wrap-run [run test-plan] 17 | (fn [& args] 18 | (let [warnings (atom []) 19 | errors (atom [])] 20 | (with-redefs [output/warn (fn [& xs] (swap! warnings conj xs)) 21 | output/error (fn [& xs] (swap! errors conj xs))] 22 | (let [result (apply run args)] 23 | (when (seq @errors) 24 | (#'t/do-report {:type ::unexpected-error 25 | :message (str "Unexpected Error:\n" 26 | (str/join "\n" (map #(apply str " ERROR: " %) @errors)))})) 27 | (when (seq @warnings) 28 | (#'t/do-report {:type ::unexpected-warning 29 | :message (str "Unexpected Warning:\n" 30 | (str/join "\n" (map #(apply str "WARNING: " %) @warnings)))})) 31 | result)))))) 32 | -------------------------------------------------------------------------------- /src/kaocha/plugin/print_invocations.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.print-invocations 2 | (:require [kaocha.plugin :as plugin :refer [defplugin]] 3 | [kaocha.testable :as testable] 4 | [kaocha.result :as result] 5 | [clojure.string :as str])) 6 | 7 | 8 | ;; This is an attempt to print out invocations that each re-run a single failing 9 | ;; test. The hard part is recreating a full command line invocation, which might 10 | ;; not be fully feasible. 11 | 12 | (defplugin kaocha.plugin/print-invocations 13 | (post-summary [results] 14 | (when (result/failed? results) 15 | (println) 16 | (doseq [test (testable/test-seq results)] 17 | (if (and (not (seq (::result/tests test))) (result/failed? test)) 18 | (let [id (str (::testable/id test))] 19 | (println "bin/kaocha" 20 | (str/join 21 | " " 22 | (mapcat (fn [[k v]] 23 | (if (vector? v) 24 | (mapcat (fn [v] [(str "--" (name k)) v]) v) 25 | [(str "--" (name k)) v])) 26 | (cond-> (dissoc (:kaocha/cli-options results) :focus) 27 | (= "tests.edn" (:config-file (:kaocha/cli-options results))) 28 | (dissoc :config-file)))) 29 | "--focus" 30 | (str 31 | "'" (cond-> id (= (first id) \:) (subs 1)) "'")))))) 32 | results)) 33 | -------------------------------------------------------------------------------- /doc/plugins/hooks_plugin.md: -------------------------------------------------------------------------------- 1 | # Plugin: Hooks 2 | 3 | The hooks plugin allows hooking into Kaocha's process with arbitrary 4 | functions. This is very similar to using writing a plugin, but requires less 5 | boilerplate. 6 | 7 | See the documentation for extending Kaocha for a description of the different 8 | hooks. The supported hooks are: pre-load, post-load, pre-run, post-run, 9 | pre-test, post-test, pre-report. 10 | 11 | ## Implementing a hook 12 | 13 | - Given a file named "tests.edn" with: 14 | 15 | ``` clojure 16 | #kaocha/v1 17 | {:plugins [:kaocha.plugin/hooks] 18 | :kaocha.hooks/pre-test [my.kaocha.hooks/sample-hook]} 19 | ``` 20 | 21 | 22 | - And a file named "src/my/kaocha/hooks.clj" with: 23 | 24 | ``` clojure 25 | (ns my.kaocha.hooks) 26 | 27 | (println "ok") 28 | 29 | (defn sample-hook [test test-plan] 30 | (if (re-find #"fail" (str (:kaocha.testable/id test))) 31 | (assoc test :kaocha.testable/pending true) 32 | test)) 33 | ``` 34 | 35 | 36 | - And a file named "test/sample_test.clj" with: 37 | 38 | ``` clojure 39 | (ns sample-test 40 | (:require [clojure.test :refer :all])) 41 | 42 | (deftest stdout-pass-test 43 | (println "You peng zi yuan fang lai") 44 | (is (= :same :same))) 45 | 46 | (deftest stdout-fail-test 47 | (println "Bu yi le hu?") 48 | (is (= :same :not-same))) 49 | ``` 50 | 51 | 52 | - When I run `bin/kaocha` 53 | 54 | - Then the output should contain: 55 | 56 | ``` nil 57 | PENDING sample-test/stdout-fail-test (sample_test.clj:8) 58 | ``` 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /test/features/plugins/hooks_plugin.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Hooks 2 | 3 | The hooks plugin allows hooking into Kaocha's process with arbitrary 4 | functions. This is very similar to using writing a plugin, but requires less 5 | boilerplate. 6 | 7 | See the documentation for extending Kaocha for a description of the different 8 | hooks. The supported hooks are: pre-load, post-load, pre-run, post-run, 9 | pre-test, post-test, pre-report. 10 | 11 | Scenario: Implementing a hook 12 | Given a file named "tests.edn" with: 13 | """ clojure 14 | #kaocha/v1 15 | {:plugins [:kaocha.plugin/hooks] 16 | :kaocha.hooks/pre-test [my.kaocha.hooks/sample-hook]} 17 | """ 18 | And a file named "src/my/kaocha/hooks.clj" with: 19 | """ clojure 20 | (ns my.kaocha.hooks) 21 | 22 | (println "ok") 23 | 24 | (defn sample-hook [test test-plan] 25 | (if (re-find #"fail" (str (:kaocha.testable/id test))) 26 | (assoc test :kaocha.testable/pending true) 27 | test)) 28 | """ 29 | And a file named "test/sample_test.clj" with: 30 | """ clojure 31 | (ns sample-test 32 | (:require [clojure.test :refer :all])) 33 | 34 | (deftest stdout-pass-test 35 | (println "You peng zi yuan fang lai") 36 | (is (= :same :same))) 37 | 38 | (deftest stdout-fail-test 39 | (println "Bu yi le hu?") 40 | (is (= :same :not-same))) 41 | """ 42 | When I run `bin/kaocha` 43 | Then the output should contain: 44 | """ 45 | PENDING sample-test/stdout-fail-test (sample_test.clj:8) 46 | """ 47 | -------------------------------------------------------------------------------- /src/kaocha/type/clojure/test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.clojure.test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [kaocha.core-ext :refer :all] 4 | [clojure.spec.alpha :as s] 5 | [kaocha.type.ns :as type.ns] 6 | [kaocha.testable :as testable] 7 | [kaocha.classpath :as classpath] 8 | [kaocha.hierarchy :as hierarchy] 9 | [kaocha.load :as load] 10 | [clojure.java.io :as io] 11 | [clojure.test :as t])) 12 | 13 | (defmethod testable/-load :kaocha.type/clojure.test [testable] 14 | (assoc (load/load-test-namespaces testable type.ns/->testable) 15 | ::testable/desc (str (name (::testable/id testable)) " (clojure.test)"))) 16 | 17 | (defmethod testable/-run :kaocha.type/clojure.test [testable test-plan] 18 | (t/do-report {:type :begin-test-suite}) 19 | (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) 20 | testable (-> testable 21 | (dissoc :kaocha.test-plan/tests) 22 | (assoc :kaocha.result/tests results))] 23 | (t/do-report {:type :end-test-suite 24 | :kaocha/testable testable}) 25 | testable)) 26 | 27 | (s/def :kaocha.type/clojure.test (s/keys :req [:kaocha/source-paths 28 | :kaocha/test-paths 29 | :kaocha/ns-patterns])) 30 | 31 | (s/def :kaocha/source-paths (s/coll-of string?)) 32 | (s/def :kaocha/test-paths (s/coll-of string?)) 33 | (s/def :kaocha/ns-patterns (s/coll-of string?)) 34 | 35 | (hierarchy/derive! :kaocha.type/clojure.test :kaocha.testable.type/suite) 36 | -------------------------------------------------------------------------------- /test/unit/kaocha/api_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.api-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.api :refer :all] 4 | [kaocha.test-util :refer [with-out-err]])) 5 | 6 | (deftest run-test 7 | (testing "allows API usage" 8 | (let [config {:kaocha/tests [{:kaocha.testable/id :unit 9 | :kaocha.testable/desc "unit (clojure.test)" 10 | :kaocha.testable/type :kaocha.type/clojure.test 11 | :kaocha/test-paths ["fixtures/a-tests"] 12 | :kaocha/source-paths ["src"] 13 | :kaocha/ns-patterns ["-test$"]}]}] 14 | (is (match? 15 | {:kaocha.result/tests 16 | [{:kaocha.testable/id :unit 17 | :kaocha.testable/type :kaocha.type/clojure.test 18 | :kaocha/test-paths ["fixtures/a-tests"] 19 | :kaocha/source-paths ["src"] 20 | :kaocha/ns-patterns ["-test$"] 21 | :kaocha.result/tests 22 | [{:kaocha.testable/type :kaocha.type/ns 23 | :kaocha.testable/id :foo.bar-test 24 | :kaocha.result/tests 25 | [{:kaocha.testable/type :kaocha.type/var 26 | :kaocha.testable/id :foo.bar-test/a-test 27 | :kaocha.testable/desc "a-test" 28 | :kaocha.var/name 'foo.bar-test/a-test 29 | :kaocha.result/count 1 30 | :kaocha.result/pass 1 31 | :kaocha.result/error 0 32 | :kaocha.result/fail 0}]}]}]} 33 | (:result (with-out-err (run config)))))))) 34 | -------------------------------------------------------------------------------- /bin/prep_release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function echo_info() { 4 | echo -en "[\033[0;32mINFO\033[0m] " 5 | echo "$@" 6 | } 7 | 8 | function echo_error() { 9 | echo -en "[\033[0;31mERROR\033[0m] " 10 | echo "$@" 11 | } 12 | 13 | PROJECT=$(git remote get-url origin | sed 's/.*lambdaisland\/\([\.a-z-]\+\)\.git/\1/') 14 | BRANCH=$(git rev-parse --abbrev-ref HEAD) 15 | 16 | if [[ "$BRANCH" != "master" ]]; then 17 | echo_error "On branch $BRANCH, release must be on master." 18 | exit 1; 19 | fi 20 | 21 | git clean -xfd -e bin 22 | 23 | VERSION="0.0-$(git rev-list --count HEAD)" 24 | 25 | echo_info "Bumping version to $VERSION" 26 | 27 | sed 's/lambdaisland\/'"${PROJECT}"' {:mvn\/version "[^"]*"}/lambdaisland\/'"${PROJECT}"' {:mvn\/version "'${VERSION}'"}/' -i README.md 28 | sed 's/\[lambdaisland\/'"${PROJECT}"' "[^"]*"\]/\[lambdaisland\/'"${PROJECT}"' "'${VERSION}'"\]/' -i README.md 29 | 30 | if [[ -d doc ]]; then 31 | sed 's/lambdaisland\/'"${PROJECT}"' {:mvn\/version "[^"]*"}/lambdaisland\/'"${PROJECT}"' {:mvn\/version "'${VERSION}'"}/' -i doc/*.md 32 | sed 's/\[lambdaisland\/'"${PROJECT}"' "[^"]*"\]/\[lambdaisland\/'"${PROJECT}"' "'${VERSION}'"\]/' -i doc/*.md 33 | ed < bin/update_toc.ed 34 | fi 35 | 36 | echo_info "Updating pom.xml dependendencies" 37 | 38 | clojure -Spom 39 | sed '/^ *$/d' -i pom.xml # https://dev.clojure.org/jira/browse/TDEPS-29 40 | 41 | echo_info "Updating CHANGELOG" 42 | 43 | { 44 | echo "1d" 45 | echo "1i" 46 | echo "# ${VERSION} ($(date --rfc-3339=date) / $(git rev-parse --short HEAD))" 47 | echo "." 48 | echo "wq" 49 | } | ed CHANGELOG.md 50 | 51 | git add -A 52 | git commit -m "Release v$VERSION" 53 | git tag v$VERSION 54 | 55 | git show v$VERSION 56 | -------------------------------------------------------------------------------- /doc/command_line/reporter.md: -------------------------------------------------------------------------------- 1 | # CLI: `--reporter` option 2 | 3 | The progress and summary printed by Kaocha are done by one or more "reporter" 4 | functions. A reporter can be specified with the `--reporter` option followed 5 | by a fully qualified function name. 6 | 7 | Reporters in the `kaocha.report` namespace can be specified without a 8 | namespace prefix. 9 | 10 | ## Background: An example test 11 | 12 | - Given a file named "test/my/project/reporter_test.clj" with: 13 | 14 | ``` clojure 15 | (ns my.project.reporter-test 16 | (:require [clojure.test :refer :all])) 17 | 18 | (deftest test-1 19 | (is (= 1 0))) 20 | 21 | (deftest test-2 22 | (is true) 23 | (is (throw (Exception. ""))) 24 | (is true)) 25 | 26 | (deftest test-3 27 | (is true)) 28 | ``` 29 | 30 | 31 | 32 | ## Using a fully qualified function as a reporter 33 | 34 | - When I run `bin/kaocha --reporter kaocha.report/documentation` 35 | 36 | - And the output should contain: 37 | 38 | ``` nil 39 | my.project.reporter-test 40 | test-1 FAIL 41 | test-2 ERROR 42 | test-3 43 | ``` 44 | 45 | 46 | 47 | ## Specifying a reporter via shorthand 48 | 49 | - When I run `bin/kaocha --reporter documentation` 50 | 51 | - Then the exit-code should be 2 52 | 53 | - And the output should contain: 54 | 55 | ``` nil 56 | my.project.reporter-test 57 | test-1 FAIL 58 | test-2 ERROR 59 | test-3 60 | ``` 61 | 62 | 63 | 64 | ## Using a reporter which does not exist 65 | 66 | - When I run `bin/kaocha --reporter does/not-exist` 67 | 68 | - Then stderr should contain 69 | 70 | ``` nil 71 | ERROR: Failed to resolve reporter var: does/not-exist 72 | ``` 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /test/features/command_line/reporter.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: `--reporter` option 2 | 3 | The progress and summary printed by Kaocha are done by one or more "reporter" 4 | functions. A reporter can be specified with the `--reporter` option followed 5 | by a fully qualified function name. 6 | 7 | Reporters in the `kaocha.report` namespace can be specified without a 8 | namespace prefix. 9 | 10 | Background: An example test 11 | Given a file named "test/my/project/reporter_test.clj" with: 12 | """clojure 13 | (ns my.project.reporter-test 14 | (:require [clojure.test :refer :all])) 15 | 16 | (deftest test-1 17 | (is (= 1 0))) 18 | 19 | (deftest test-2 20 | (is true) 21 | (is (throw (Exception. ""))) 22 | (is true)) 23 | 24 | (deftest test-3 25 | (is true)) 26 | """ 27 | 28 | Scenario: Using a fully qualified function as a reporter 29 | When I run `bin/kaocha --reporter kaocha.report/documentation` 30 | And the output should contain: 31 | """ 32 | my.project.reporter-test 33 | test-1 FAIL 34 | test-2 ERROR 35 | test-3 36 | """ 37 | 38 | Scenario: Specifying a reporter via shorthand 39 | When I run `bin/kaocha --reporter documentation` 40 | Then the exit-code should be 2 41 | And the output should contain: 42 | """ 43 | my.project.reporter-test 44 | test-1 FAIL 45 | test-2 ERROR 46 | test-3 47 | """ 48 | 49 | Scenario: Using a reporter which does not exist 50 | When I run `bin/kaocha --reporter does/not-exist` 51 | Then stderr should contain 52 | """ 53 | ERROR: Failed to resolve reporter var: does/not-exist 54 | """ 55 | -------------------------------------------------------------------------------- /test/features/command_line/suite_names.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: Selecting test suites 2 | 3 | Each test suite has a unique id, given as a keyword in the test configuration. 4 | You can supply one or more of these ids on the command line to run only those 5 | test suites. 6 | 7 | Background: Given two test suites, `:aaa` and `:bbb` 8 | Given a file named "tests.edn" with: 9 | """clojure 10 | #kaocha/v1 11 | {:tests [{:id :aaa 12 | :test-paths ["tests/aaa"]} 13 | {:id :bbb 14 | :test-paths ["tests/bbb"]}]} 15 | """ 16 | 17 | And a file named "tests/aaa/aaa_test.clj" with: 18 | """clojure 19 | (ns aaa-test (:require [clojure.test :refer :all])) 20 | (deftest foo-test (is true)) 21 | """ 22 | 23 | And a file named "tests/bbb/bbb_test.clj" with: 24 | """clojure 25 | (ns bbb-test (:require [clojure.test :refer :all])) 26 | (deftest bbb-test (is true)) 27 | """ 28 | 29 | Scenario: Specifying a test suite on the command line 30 | When I run `bin/kaocha aaa --reporter documentation` 31 | And the output should contain: 32 | """ 33 | aaa-test 34 | foo-test 35 | """ 36 | And the output should not contain 37 | """ 38 | bbb-test 39 | """ 40 | 41 | Scenario: Specifying a test suite using keyword syntax 42 | When I run `bin/kaocha :aaa --reporter documentation` 43 | And the output should contain: 44 | """ 45 | aaa-test 46 | foo-test 47 | """ 48 | And the output should not contain 49 | """ 50 | bbb-test 51 | """ 52 | 53 | Scenario: Specifying an unkown suite 54 | When I run `bin/kaocha suite-name` 55 | Then the output should contain: 56 | """ 57 | No such suite: :suite-name, valid options: :aaa, :bbb. 58 | """ 59 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | kaocha: lambdaisland/kaocha@dev:first 5 | clojure: lambdaisland/clojure@dev:first 6 | 7 | commands: 8 | checkout_and_run: 9 | parameters: 10 | clojure_version: 11 | type: string 12 | steps: 13 | - checkout 14 | - clojure/with_cache: 15 | cache_version: << parameters.clojure_version >> 16 | steps: 17 | - run: clojure -e '(println (System/getProperty "java.runtime.name") (System/getProperty "java.runtime.version") "\nClojure" (clojure-version))' 18 | - kaocha/execute: 19 | args: "unit --reporter documentation --plugin cloverage --codecov" 20 | clojure_version: << parameters.clojure_version >> 21 | - kaocha/upload_codecov: 22 | flags: unit 23 | - kaocha/execute: 24 | args: "integration --reporter documentation --plugin cloverage --codecov" 25 | clojure_version: << parameters.clojure_version >> 26 | - kaocha/upload_codecov: 27 | flags: integration 28 | file: target/coverage/integration*/codecov.json 29 | 30 | jobs: 31 | java-11-clojure-1_10: 32 | executor: clojure/openjdk11 33 | steps: [{checkout_and_run: {clojure_version: "1.10.0"}}] 34 | 35 | java-9-clojure-1_9: 36 | executor: clojure/openjdk9 37 | steps: [{checkout_and_run: {clojure_version: "1.9.0"}}] 38 | 39 | java-8-clojure-1_10: 40 | executor: clojure/openjdk8 41 | steps: [{checkout_and_run: {clojure_version: "1.10.0"}}] 42 | 43 | java-8-clojure-1_9: 44 | executor: clojure/openjdk8 45 | steps: [{checkout_and_run: {clojure_version: "1.9.0"}}] 46 | 47 | workflows: 48 | kaocha_test: 49 | jobs: 50 | - java-11-clojure-1_10 51 | - java-9-clojure-1_9 52 | - java-8-clojure-1_10 53 | - java-8-clojure-1_9 54 | -------------------------------------------------------------------------------- /doc/command_line/suite_names.md: -------------------------------------------------------------------------------- 1 | # CLI: Selecting test suites 2 | 3 | Each test suite has a unique id, given as a keyword in the test configuration. 4 | You can supply one or more of these ids on the command line to run only those 5 | test suites. 6 | 7 | ## Background: Given two test suites, `:aaa` and `:bbb` 8 | 9 | - Given a file named "tests.edn" with: 10 | 11 | ``` clojure 12 | #kaocha/v1 13 | {:tests [{:id :aaa 14 | :test-paths ["tests/aaa"]} 15 | {:id :bbb 16 | :test-paths ["tests/bbb"]}]} 17 | ``` 18 | 19 | 20 | - And a file named "tests/aaa/aaa_test.clj" with: 21 | 22 | ``` clojure 23 | (ns aaa-test (:require [clojure.test :refer :all])) 24 | (deftest foo-test (is true)) 25 | ``` 26 | 27 | 28 | - And a file named "tests/bbb/bbb_test.clj" with: 29 | 30 | ``` clojure 31 | (ns bbb-test (:require [clojure.test :refer :all])) 32 | (deftest bbb-test (is true)) 33 | ``` 34 | 35 | 36 | 37 | ## Specifying a test suite on the command line 38 | 39 | - When I run `bin/kaocha aaa --reporter documentation` 40 | 41 | - And the output should contain: 42 | 43 | ``` nil 44 | aaa-test 45 | foo-test 46 | ``` 47 | 48 | 49 | - And the output should not contain 50 | 51 | ``` nil 52 | bbb-test 53 | ``` 54 | 55 | 56 | 57 | ## Specifying a test suite using keyword syntax 58 | 59 | - When I run `bin/kaocha :aaa --reporter documentation` 60 | 61 | - And the output should contain: 62 | 63 | ``` nil 64 | aaa-test 65 | foo-test 66 | ``` 67 | 68 | 69 | - And the output should not contain 70 | 71 | ``` nil 72 | bbb-test 73 | ``` 74 | 75 | 76 | 77 | ## Specifying an unkown suite 78 | 79 | - When I run `bin/kaocha suite-name` 80 | 81 | - Then the output should contain: 82 | 83 | ``` nil 84 | No such suite: :suite-name, valid options: :aaa, :bbb. 85 | ``` 86 | 87 | 88 | 89 | -------------------------------------------------------------------------------- /src/kaocha/core_ext.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.core-ext 2 | "Core language extensions" 3 | (:refer-clojure :exclude [symbol]) 4 | (:import [java.util.regex Pattern])) 5 | 6 | (defn regex? [x] 7 | (instance? Pattern x)) 8 | 9 | (defn exception? [x] 10 | (instance? java.lang.Exception x)) 11 | 12 | (defn error? [x] 13 | (instance? java.lang.Error x)) 14 | 15 | (defn throwable? [x] 16 | (instance? java.lang.Throwable x)) 17 | 18 | (defn ns? [x] 19 | (instance? clojure.lang.Namespace x)) 20 | 21 | (defn file? [x] 22 | (and (instance? java.io.File x) (.isFile x))) 23 | 24 | (defn directory? [x] 25 | (and (instance? java.io.File x) (.isDirectory x))) 26 | 27 | (defn path? [x] 28 | (instance? java.nio.file.Path x)) 29 | 30 | (defn regex 31 | ([x & xs] 32 | (regex (apply str x xs))) 33 | ([x] 34 | (cond 35 | (regex? x) x 36 | (string? x) (Pattern/compile x) 37 | :else (throw (ex-info (str "Can't coerce " (class x) " to regex.") {}))))) 38 | 39 | (defn mapply 40 | "Applies a function f to the argument list formed by concatenating 41 | everything but the last element of args with the last element of 42 | args. This is useful for applying a function that accepts keyword 43 | arguments to a map." 44 | [f & args] 45 | (apply f (apply concat (butlast args) (last args)))) 46 | 47 | (defn symbol 48 | "Backport from Clojure 1.10, symbol function that's a bit more lenient on its 49 | inputs. 50 | 51 | Returns a Symbol with the given namespace and name. Arity-1 works on strings, 52 | keywords, and vars." 53 | ^clojure.lang.Symbol 54 | ([name] 55 | (cond 56 | (symbol? name) name 57 | (instance? String name) (clojure.lang.Symbol/intern name) 58 | (instance? clojure.lang.Var name) (.toSymbol ^clojure.lang.Var name) 59 | (instance? clojure.lang.Keyword name) (.sym ^clojure.lang.Keyword name) 60 | :else (throw (IllegalArgumentException. "no conversion to symbol")))) 61 | ([ns name] (clojure.lang.Symbol/intern ns name))) 62 | -------------------------------------------------------------------------------- /src/kaocha/stacktrace.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.stacktrace 2 | (:require [clojure.stacktrace :as st] 3 | [clojure.string :as str])) 4 | 5 | (def ^:dynamic *stacktrace-filters* ["java.lang." 6 | "clojure.test$" 7 | "clojure.lang." 8 | "clojure.core" 9 | "kaocha.monkey_patch"]) 10 | 11 | (defn elide-element? [e] 12 | (some #(str/starts-with? (.getClassName e) %) *stacktrace-filters*)) 13 | 14 | (defn print-stack-trace 15 | "Prints a Clojure-oriented stack trace of tr, a Throwable. 16 | Prints a maximum of n stack frames (default: unlimited). Does not print 17 | chained exceptions (causes)." 18 | ([tr] 19 | (print-stack-trace tr nil)) 20 | ([^Throwable tr n] 21 | (let [st (.getStackTrace tr)] 22 | (st/print-throwable tr) 23 | (newline) 24 | (print " at ") 25 | (if-let [e (first st)] 26 | (st/print-trace-element e) ;; always print the first element 27 | (print "[empty stack trace]")) 28 | (newline) 29 | (loop [[e & st] (next st) 30 | eliding? false 31 | n n] 32 | (when e 33 | (let [n (cond-> n n dec)] 34 | (if (= 0 n) 35 | (println " ... and " (count st) "more") 36 | (if (elide-element? e) 37 | (do 38 | (when (not eliding?) 39 | (println " ...")) 40 | (recur st true n)) 41 | (do 42 | (print " ") 43 | (st/print-trace-element e) 44 | (newline) 45 | (recur st false n)))))))))) 46 | 47 | (defn print-cause-trace 48 | "Like print-stack-trace but prints chained exceptions (causes)." 49 | ([tr] 50 | (print-cause-trace tr nil)) 51 | ([tr n] 52 | (print-stack-trace tr n) 53 | (when-let [cause (.getCause tr)] 54 | (print "Caused by: " ) 55 | (recur cause n)))) 56 | -------------------------------------------------------------------------------- /src/kaocha/plugin/hooks.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.hooks 2 | (:require [kaocha.plugin :refer [defplugin]] 3 | [kaocha.testable :as testable])) 4 | 5 | (defn update? [m k f] 6 | (if (contains? m k) 7 | (update m k f) 8 | m)) 9 | 10 | (defn load-hooks [hs] 11 | (mapv (fn [h] 12 | (cond 13 | (qualified-symbol? h) 14 | (do 15 | (require (symbol (namespace h))) 16 | (resolve h)) 17 | 18 | (list? h) 19 | (eval h) 20 | 21 | :else 22 | h)) 23 | hs)) 24 | 25 | (defplugin kaocha.plugin/hooks 26 | "Configure hooks directly in `tests.edn`." 27 | (config [config] 28 | (-> config 29 | (update? :kaocha.hooks/pre-load load-hooks) 30 | (update? :kaocha.hooks/post-load load-hooks) 31 | (update? :kaocha.hooks/pre-run load-hooks) 32 | (update? :kaocha.hooks/post-run load-hooks) 33 | (update? :kaocha.hooks/wrap-run load-hooks) 34 | (update? :kaocha.hooks/pre-test load-hooks) 35 | (update? :kaocha.hooks/post-test load-hooks) 36 | (update? :kaocha.hooks/pre-report load-hooks))) 37 | 38 | (pre-load [config] 39 | (reduce #(%2 %1) config (:kaocha.hooks/pre-load config))) 40 | 41 | (post-load [test-plan] 42 | (reduce #(%2 %1) test-plan (:kaocha.hooks/post-load test-plan))) 43 | 44 | (pre-run [test-plan] 45 | (reduce #(%2 %1) test-plan (:kaocha.hooks/pre-run test-plan))) 46 | 47 | (post-run [test-plan] 48 | (reduce #(%2 %1) test-plan (:kaocha.hooks/post-run test-plan))) 49 | 50 | (wrap-run [run test-plan] 51 | (reduce #(%2 %1) run (:kaocha.hooks/wrap-run test-plan))) 52 | 53 | (pre-test [testable test-plan] 54 | (reduce #(%2 %1 test-plan) testable (:kaocha.hooks/pre-test test-plan))) 55 | 56 | (post-test [testable test-plan] 57 | (reduce #(%2 %1 test-plan) testable (:kaocha.hooks/post-test test-plan))) 58 | 59 | (pre-report [event] 60 | (reduce #(%2 %1) event (:kaocha.hooks/pre-report testable/*test-plan*)))) 61 | -------------------------------------------------------------------------------- /src/kaocha/result.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.result 2 | (:require [clojure.spec.alpha :as s])) 3 | 4 | (defn diff-test-result 5 | "Subtract two clojure.test style summary maps." 6 | [before after] 7 | {::pass (apply - (map :pass [after before])) 8 | ::error (apply - (map :error [after before])) 9 | ::fail (apply - (map :fail [after before])) 10 | ::pending (apply - (map :pending [after before]))}) 11 | 12 | (defn sum 13 | "Sum up kaocha result maps." 14 | [& rs] 15 | {::count (apply + (map #(::count % 0) rs)) 16 | ::pass (apply + (map #(::pass % 0) rs)) 17 | ::error (apply + (map #(::error % 0) rs)) 18 | ::fail (apply + (map #(::fail % 0) rs)) 19 | ::pending (apply + (map #(::pending % 0) rs))}) 20 | 21 | (s/fdef sum 22 | :ret (s/keys :req [::count ::pass ::error ::fail ::pending])) 23 | 24 | (declare testable-totals) 25 | 26 | (defn totals 27 | "Return a map of summed up results for a collection of testables." 28 | [testables] 29 | (apply sum (map testable-totals testables))) 30 | 31 | (defn testable-totals 32 | "Return a map of summed up results for a testable, including descendants." 33 | [testable] 34 | (if-let [testables (::tests testable)] 35 | (merge testable (totals testables)) 36 | (merge (sum) testable))) 37 | 38 | (s/fdef testable-totals 39 | :ret (s/keys :req [::count ::pass ::error ::fail ::pending])) 40 | 41 | (defn failed? 42 | "Did this testable, or one of its children, fail or error?" 43 | [testable] 44 | (let [{::keys [error fail]} (testable-totals testable)] 45 | (or (> error 0) (> fail 0)))) 46 | 47 | (defn failed-one? 48 | "Did this testable fail or error, does not recurse." 49 | [{::keys [error fail] :or {error 0 fail 0}}] 50 | (or (> error 0) (> fail 0))) 51 | 52 | (defn totals->clojure-test-summary 53 | "Turn a kaocha-style result map into a clojure.test style summary map." 54 | [totals] 55 | {:type :summary 56 | :test (::count totals) 57 | :pass (::pass totals) 58 | :fail (::fail totals) 59 | :pending (::pending totals) 60 | :error (::error totals)}) 61 | -------------------------------------------------------------------------------- /doc/01_introduction.md: -------------------------------------------------------------------------------- 1 | ## 1. Introduction 2 | 3 | > Quality is not an act, it is a habit. — Aristotle 4 | 5 | Kaocha is an all-in-one testing tool, its core task is to load tests and execute 6 | them, reporting on their progress and final result. It does this in a way that 7 | encourages good habits, supports multiple workflow styles, and optimizes for 8 | ergonomics. 9 | 10 | Kaocha has a modular architecture. It understands different types of tests: 11 | `clojure.test`, ClojureScript, Cucumber, Fudje, Expectations, so that all of a 12 | project's tests can be handled in the same way, and so that more can be added 13 | without requiring changes to the core. 14 | 15 | It aims to deliver all the features a developer might expect from their test 16 | tooling. Different people have different workflows, different styles of writing 17 | and running tests. We want to make sure we got you covered. 18 | 19 | Much of this is achieved through plugins. This way the Kaocha core can remain 20 | focused, while making it easy to experiment with new features. 21 | 22 | To use Kaocha you create a `tests.edn` at the root of your project, and run 23 | tests from the command line or from the REPL. 24 | 25 | Features include 26 | 27 | - Filtering tests based on test names or metadata 28 | - Watch mode: watch the file system for changes and re-run tests 29 | - Pretty, pluggable reporting 30 | - Randomize test order 31 | - Detect when interrupted with ctrl-C and print report 32 | - Fail fast mode: stop at first failure and print report 33 | - Profiling (show slowest tests) 34 | - Dynamic classpath handling 35 | - Tests as data (get test config, test plan, or test results as EDN) 36 | - Extensible test types (clojure.test, Cucumber, ...) 37 | - Extensible through plugins 38 | - Tool agnostic (Clojure CLI, Leiningen, boot) 39 | 40 | Currently Kaocha's versioning scheme is `0.0-${commit count}`, and releases are 41 | made often. As long as the version is at `0.0` Kaocha will be considered alpha, 42 | in other words: subject to change. Keep an eye on the CHANGELOG. 43 | 44 | Kaocha requires Clojure 1.9. ClojureScript support requires Clojure and 45 | ClojureScript 1.10. 46 | -------------------------------------------------------------------------------- /doc/plugins/version_filter.md: -------------------------------------------------------------------------------- 1 | # Plugin: Clojure/Java Version filter 2 | 3 | The `version-filter` plugin will look for test metadata specifying the minimum 4 | or maximum version of Clojure or Java the test is designed to work with, and 5 | skip the test unless it falls within the range specified. 6 | 7 | The recognized metadata keys are `:min-clojure-version`, 8 | `:max-clojure-version`, `:min-java-version`, and `:max-java-version`. The 9 | associated value is a version string, such as `"1.10.0"`. 10 | 11 | You can set both a minimum and a maximum to limit to a certain range. The 12 | boundaries are always inclusive, so `^{:max-clojure-version "1.9"}` will run 13 | on Clojure `1.9.*` or earlier. 14 | 15 | Specificty matters, a test with a max version of `"1.10" will also run on 16 | version `"1.10.2"`, whereas if the max version is `"1.10.0"` it will not. 17 | 18 | Note that the Java version is based on the "java.runtime.version" system 19 | property. Before Java 9 this was the so called "developer version", which 20 | started with `1.`, e.g. `"1.8.0"`, so Java (JDK) versions effectivel jumped 21 | from `1.8` to `9`. 22 | [1](https://blogs.oracle.com/java-platform-group/a-new-jdk-9-version-string-scheme) 23 | [2](https://en.wikipedia.org/wiki/Java_version_history#Versioning_change) 24 | 25 | ## Enabling in `tests.edn` 26 | 27 | - Given a file named "tests.edn" with: 28 | 29 | ``` clojure 30 | #kaocha/v1 31 | {:plugins [:kaocha.plugin/version-filter] 32 | :color? false} 33 | ``` 34 | 35 | 36 | - And a file named "test/my/sample_test.clj" with: 37 | 38 | ``` clojure 39 | (ns my.sample-test 40 | (:require [clojure.test :refer :all])) 41 | 42 | (deftest ^{:max-java-version "1.7"} this-test-gets-skipped 43 | (is false)) 44 | 45 | (deftest ^{:min-clojure-version "1.6.0"} this-test-runs 46 | (is true)) 47 | ``` 48 | 49 | 50 | - When I run `bin/kaocha --reporter documentation` 51 | 52 | - Then the output should contain: 53 | 54 | ``` nil 55 | --- unit (clojure.test) --------------------------- 56 | my.sample-test 57 | this-test-runs 58 | 59 | 1 tests, 1 assertions, 0 failures. 60 | ``` 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /test/features/plugins/version_filter.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Clojure/Java Version filter 2 | 3 | The `version-filter` plugin will look for test metadata specifying the minimum 4 | or maximum version of Clojure or Java the test is designed to work with, and 5 | skip the test unless it falls within the range specified. 6 | 7 | The recognized metadata keys are `:min-clojure-version`, 8 | `:max-clojure-version`, `:min-java-version`, and `:max-java-version`. The 9 | associated value is a version string, such as `"1.10.0"`. 10 | 11 | You can set both a minimum and a maximum to limit to a certain range. The 12 | boundaries are always inclusive, so `^{:max-clojure-version "1.9"}` will run 13 | on Clojure `1.9.*` or earlier. 14 | 15 | Specificty matters, a test with a max version of `"1.10" will also run on 16 | version `"1.10.2"`, whereas if the max version is `"1.10.0"` it will not. 17 | 18 | Note that the Java version is based on the "java.runtime.version" system 19 | property. Before Java 9 this was the so called "developer version", which 20 | started with `1.`, e.g. `"1.8.0"`, so Java (JDK) versions effectivel jumped 21 | from `1.8` to `9`. 22 | [1](https://blogs.oracle.com/java-platform-group/a-new-jdk-9-version-string-scheme) 23 | [2](https://en.wikipedia.org/wiki/Java_version_history#Versioning_change) 24 | 25 | Scenario: Enabling in `tests.edn` 26 | Given a file named "tests.edn" with: 27 | """ clojure 28 | #kaocha/v1 29 | {:plugins [:kaocha.plugin/version-filter] 30 | :color? false} 31 | """ 32 | And a file named "test/my/sample_test.clj" with: 33 | """ clojure 34 | (ns my.sample-test 35 | (:require [clojure.test :refer :all])) 36 | 37 | (deftest ^{:max-java-version "1.7"} this-test-gets-skipped 38 | (is false)) 39 | 40 | (deftest ^{:min-clojure-version "1.6.0"} this-test-runs 41 | (is true)) 42 | """ 43 | When I run `bin/kaocha --reporter documentation` 44 | Then the output should contain: 45 | """ 46 | --- unit (clojure.test) --------------------------- 47 | my.sample-test 48 | this-test-runs 49 | 50 | 1 tests, 1 assertions, 0 failures. 51 | """ 52 | -------------------------------------------------------------------------------- /src/kaocha/plugin/randomize.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.randomize 2 | (:require [kaocha.plugin :as plugin :refer [defplugin]] 3 | [kaocha.result :as result]) 4 | (:import [java.util Random])) 5 | 6 | (defn rng [seed] 7 | (let [rng (java.util.Random. seed)] 8 | (fn [& _] (.nextInt rng)))) 9 | 10 | (defn straight-sort [test-plan] 11 | (if-let [tests (:kaocha.test-plan/tests test-plan)] 12 | (assoc test-plan 13 | :kaocha.test-plan/tests 14 | (->> tests 15 | (sort-by :kaocha.testable/id) 16 | (map straight-sort))) 17 | test-plan)) 18 | 19 | (defn rng-sort [rng test-plan] 20 | (if-let [tests (:kaocha.test-plan/tests test-plan)] 21 | (assoc test-plan 22 | :kaocha.test-plan/tests 23 | (->> tests 24 | (map #(assoc % ::sort-key (rng))) 25 | (sort-by ::sort-key) 26 | (map (partial rng-sort rng)))) 27 | test-plan)) 28 | 29 | (defplugin kaocha.plugin/randomize 30 | (cli-options [opts] 31 | (conj opts 32 | [nil "--[no-]randomize" "Run test namespaces and vars in random order."] 33 | [nil "--seed SEED" "Provide a seed to determine the random order of tests." 34 | :parse-fn #(Integer/parseInt %)])) 35 | 36 | (config [config] 37 | (let [randomize? (get-in config [:kaocha/cli-options :randomize]) 38 | seed (get-in config [:kaocha/cli-options :seed]) 39 | config (merge {::randomize? true} 40 | config 41 | (when (some? randomize?) 42 | {::randomize? randomize?}))] 43 | (if (::randomize? config) 44 | (merge {::seed (or seed (rand-int Integer/MAX_VALUE))} config) 45 | config))) 46 | 47 | (post-load [test-plan] 48 | (if (::randomize? test-plan) 49 | (let [rng (rng (::seed test-plan))] 50 | (->> test-plan 51 | straight-sort 52 | (rng-sort rng))) 53 | test-plan)) 54 | 55 | (post-run [test-plan] 56 | (if (and (::randomize? test-plan) (result/failed? test-plan)) 57 | (print "\nRandomized with --seed" (::seed test-plan))) 58 | test-plan)) 59 | -------------------------------------------------------------------------------- /doc/filtering/focusing.md: -------------------------------------------------------------------------------- 1 | # Focusing on specific tests 2 | 3 | You can limit the test run to only specific tests or test groups (e.g. 4 | namespaces) using the `--focus` command line flag, or `:kaocha.filter/focus` 5 | key in test suite configuration. 6 | 7 | ## Background: A simple test suite 8 | 9 | - Given a file named "test/my/project/sample_test.clj" with: 10 | 11 | ``` clojure 12 | (ns my.project.sample-test 13 | (:require [clojure.test :refer :all])) 14 | 15 | (deftest some-test 16 | (is (= 1 1))) 17 | 18 | (deftest other-test 19 | (is (= 2 2))) 20 | ``` 21 | 22 | 23 | - And a file named "test/my/project/other_sample_test.clj" with: 24 | 25 | ``` clojure 26 | (ns my.project.other-sample-test 27 | (:require [clojure.test :refer :all])) 28 | 29 | (deftest other-test 30 | (is (= 1 2))) 31 | ``` 32 | 33 | 34 | 35 | ## Focusing on test id from the command line 36 | 37 | - When I run `bin/kaocha --focus my.project.sample-test/some-test --reporter documentation` 38 | 39 | - Then the output should contain: 40 | 41 | ``` nil 42 | --- unit (clojure.test) --------------------------- 43 | my.project.sample-test 44 | some-test 45 | 46 | 1 tests, 1 assertions, 0 failures. 47 | ``` 48 | 49 | 50 | 51 | ## Focusing on test group id from the command line 52 | 53 | - When I run `bin/kaocha --focus my.project.sample-test --reporter documentation` 54 | 55 | - Then the output should contain: 56 | 57 | ``` nil 58 | --- unit (clojure.test) --------------------------- 59 | my.project.sample-test 60 | other-test 61 | some-test 62 | 63 | 2 tests, 2 assertions, 0 failures. 64 | ``` 65 | 66 | 67 | 68 | ## Focusing via configuration 69 | 70 | - Given a file named "tests.edn" with: 71 | 72 | ``` edn 73 | #kaocha/v1 74 | {:tests [{:kaocha.filter/focus [my.project.sample-test/other-test]}] 75 | :color? false 76 | :randomize? false} 77 | ``` 78 | 79 | 80 | - When I run `bin/kaocha --reporter documentation` 81 | 82 | - Then the output should contain: 83 | 84 | ``` nil 85 | --- unit (clojure.test) --------------------------- 86 | my.project.sample-test 87 | other-test 88 | 89 | 1 tests, 1 assertions, 0 failures. 90 | ``` 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /src/kaocha/report/progress.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.report.progress 2 | (:require [clojure.test :as t] 3 | [progrock.core :as pr] 4 | [kaocha.testable :as testable] 5 | [kaocha.output :as output] 6 | [kaocha.report :as report] 7 | [kaocha.hierarchy :as hierarchy])) 8 | 9 | (def bar (atom nil)) 10 | 11 | (defn color [{:keys [failed? pending?]}] 12 | (cond 13 | failed? :red 14 | pending? :yellow 15 | :else :green)) 16 | 17 | (defn format-bar [{:keys [label label-width] :as bar}] 18 | (str (format (str "%" label-width "s") label) 19 | ": :percent% [" 20 | (output/colored (color bar) ":bar") 21 | "] :progress/:total")) 22 | 23 | (defn print-bar [] 24 | (t/with-test-out 25 | (pr/print @bar {:format (format-bar @bar)}))) 26 | 27 | (defmulti progress :type :hierarchy #'hierarchy/hierarchy) 28 | (defmethod progress :default [_]) 29 | 30 | (defmethod progress :begin-test-suite [m] 31 | (let [testable (:kaocha/testable m) 32 | test-plan (:kaocha/test-plan m) 33 | leaf-tests (->> testable 34 | testable/test-seq 35 | (filter hierarchy/leaf?))] 36 | (reset! bar (assoc (pr/progress-bar (count leaf-tests)) 37 | :label (name (:kaocha.testable/id testable)) 38 | :label-width (->> test-plan 39 | :kaocha.test-plan/tests 40 | (remove :kaocha.testable/skip) 41 | (map (comp count name :kaocha.testable/id)) 42 | (apply max)))) 43 | (print-bar))) 44 | 45 | (defmethod progress :kaocha/end-test [m] 46 | (swap! bar pr/tick) 47 | (print-bar)) 48 | 49 | (defmethod progress :end-test-suite [m] 50 | (swap! bar assoc :done? true) 51 | (print-bar)) 52 | 53 | (defmethod progress :kaocha/fail-type [m] 54 | (swap! bar assoc :failed? true) 55 | (print-bar)) 56 | 57 | (defmethod progress :kaocha/pending [m] 58 | (swap! bar assoc :pending? true) 59 | (print-bar)) 60 | 61 | (defmethod progress :error [m] 62 | (swap! bar assoc :failed? true) 63 | (print-bar)) 64 | 65 | (def report [progress report/result]) 66 | -------------------------------------------------------------------------------- /test/features/filtering/focusing.feature: -------------------------------------------------------------------------------- 1 | Feature: Focusing on specific tests 2 | 3 | You can limit the test run to only specific tests or test groups (e.g. 4 | namespaces) using the `--focus` command line flag, or `:kaocha.filter/focus` 5 | key in test suite configuration. 6 | 7 | Background: A simple test suite 8 | Given a file named "test/my/project/sample_test.clj" with: 9 | """clojure 10 | (ns my.project.sample-test 11 | (:require [clojure.test :refer :all])) 12 | 13 | (deftest some-test 14 | (is (= 1 1))) 15 | 16 | (deftest other-test 17 | (is (= 2 2))) 18 | """ 19 | And a file named "test/my/project/other_sample_test.clj" with: 20 | """clojure 21 | (ns my.project.other-sample-test 22 | (:require [clojure.test :refer :all])) 23 | 24 | (deftest other-test 25 | (is (= 1 2))) 26 | """ 27 | 28 | Scenario: Focusing on test id from the command line 29 | When I run `bin/kaocha --focus my.project.sample-test/some-test --reporter documentation` 30 | Then the output should contain: 31 | """ 32 | --- unit (clojure.test) --------------------------- 33 | my.project.sample-test 34 | some-test 35 | 36 | 1 tests, 1 assertions, 0 failures. 37 | """ 38 | 39 | Scenario: Focusing on test group id from the command line 40 | When I run `bin/kaocha --focus my.project.sample-test --reporter documentation` 41 | Then the output should contain: 42 | """ 43 | --- unit (clojure.test) --------------------------- 44 | my.project.sample-test 45 | other-test 46 | some-test 47 | 48 | 2 tests, 2 assertions, 0 failures. 49 | """ 50 | 51 | Scenario: Focusing via configuration 52 | Given a file named "tests.edn" with: 53 | """ edn 54 | #kaocha/v1 55 | {:tests [{:kaocha.filter/focus [my.project.sample-test/other-test]}] 56 | :color? false 57 | :randomize? false} 58 | """ 59 | When I run `bin/kaocha --reporter documentation` 60 | Then the output should contain: 61 | """ 62 | --- unit (clojure.test) --------------------------- 63 | my.project.sample-test 64 | other-test 65 | 66 | 1 tests, 1 assertions, 0 failures. 67 | """ 68 | -------------------------------------------------------------------------------- /test/unit/kaocha/hierarchy_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.hierarchy-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.hierarchy :as h])) 4 | 5 | (require 'kaocha.type.clojure.test) 6 | 7 | (derive ::global-leaf :kaocha.testable.type/leaf) 8 | (derive ::global-group :kaocha.testable.type/group) 9 | (derive ::global-suite :kaocha.testable.type/suite) 10 | (h/derive! ::local-leaf :kaocha.testable.type/leaf) 11 | (h/derive! ::local-group :kaocha.testable.type/group) 12 | (h/derive! ::local-suite :kaocha.testable.type/suite) 13 | 14 | (deftest fail-type?-test 15 | (is (h/fail-type? {:type :fail})) 16 | (is (h/fail-type? {:type :error}))) 17 | 18 | (deftest error-type?-test 19 | (is (h/error-type? {:type :error}))) 20 | 21 | (deftest pass-type?-test 22 | (is (h/pass-type? {:type :pass}))) 23 | 24 | (deftest known-key?-test 25 | (is (h/known-key? {:type :pass})) 26 | (is (h/known-key? {:type :fail})) 27 | (is (h/known-key? {:type :error})) 28 | (is (h/known-key? {:type :kaocha/known-key})) 29 | (is (h/known-key? {:type :kaocha/deferred})) 30 | (is (not (h/known-key? {:type :kaocha/foo})))) 31 | 32 | (derive ::global-deferred :kaocha/deferred) 33 | (h/derive! ::local-deferred :kaocha/deferred) 34 | 35 | (deftest deferred?-test 36 | (is (h/deferred? {:type ::global-deferred})) 37 | (is (h/deferred? {:type ::local-deferred}))) 38 | 39 | (deftest pending?-test 40 | (is (h/pending? {:type :kaocha/pending}))) 41 | 42 | (deftest suite-test 43 | (is (h/suite? {:kaocha.testable/type :kaocha.testable.type/suite})) 44 | (is (h/suite? {:kaocha.testable/type :kaocha.type/clojure.test})) 45 | (is (h/suite? {:kaocha.testable/type ::global-suite})) 46 | (is (h/suite? {:kaocha.testable/type ::local-suite}))) 47 | 48 | (deftest group-test 49 | (is (h/group? {:kaocha.testable/type :kaocha.testable.type/group})) 50 | (is (h/group? {:kaocha.testable/type :kaocha.type/ns})) 51 | (is (h/group? {:kaocha.testable/type ::global-group})) 52 | (is (h/group? {:kaocha.testable/type ::local-group}))) 53 | 54 | (deftest leaf-test 55 | (is (h/leaf? {:kaocha.testable/type :kaocha.testable.type/leaf})) 56 | (is (h/leaf? {:kaocha.testable/type :kaocha.type/var})) 57 | (is (h/leaf? {:kaocha.testable/type ::global-leaf})) 58 | (is (h/leaf? {:kaocha.testable/type ::local-leaf}))) 59 | -------------------------------------------------------------------------------- /src/kaocha/plugin/version_filter.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.version-filter 2 | "Filter tests based on the Clojure or Java version. 3 | 4 | This plugin will look for test metadata specifying the minimum or maximum 5 | version of Clojure or Java this test is designed to work with. 6 | 7 | The recognized metadata keys are `:min-clojure-version`, 8 | `:max-clojure-version`, `:min-java-version`, and `:max-java-version`. The 9 | associated value is a version string, such as `\"1.10.0\"`. 10 | 11 | You can set both a minimum and a maximum to limit to a certain range. The 12 | boundaries are always inclusive, so `^{:max-clojure-version \"1.9\"}` will run 13 | on Clojure `1.9.*` or earlier. 14 | 15 | Specificty matters, a test with a max version of `\"1.10\" will also run on 16 | version `\"1.10.2\"`, whereas if the max version is `\"1.10.0\"` it will not." 17 | (:require [clojure.string :as str] 18 | [kaocha.plugin :refer [defplugin]] 19 | [kaocha.testable :as testable])) 20 | 21 | (defn version-vector [v] 22 | (let [v (first (str/split v #"\+"))] 23 | (mapv #(Integer/parseInt (re-find #"^\d+" %)) 24 | (clojure.string/split v #"\.")))) 25 | 26 | (defn java-version [] 27 | (System/getProperty "java.runtime.version")) 28 | 29 | (defn compare-versions [v1 v2] 30 | (let [v1 (version-vector v1) 31 | v2 (version-vector v2) 32 | significance (min (count v1) (count v2))] 33 | (compare (vec (take significance v1)) 34 | (vec (take significance v2))))) 35 | 36 | (defn version>=? [v1 v2] 37 | (if (and v1 v2) 38 | (>= (compare-versions v1 v2) 0) 39 | true)) 40 | 41 | (defn skip? [testable] 42 | (let [{:keys [min-clojure-version 43 | max-clojure-version 44 | min-java-version 45 | max-java-version]} 46 | (::testable/meta testable)] 47 | (not 48 | (and 49 | (version>=? (clojure-version) min-clojure-version) 50 | (version>=? max-clojure-version (clojure-version)) 51 | (version>=? (java-version) min-java-version) 52 | (version>=? max-java-version (java-version)))))) 53 | 54 | (defplugin kaocha.plugin/version-filter 55 | (pre-test [testable test-plan] 56 | (if (skip? testable) 57 | (assoc testable ::testable/skip true) 58 | testable))) 59 | -------------------------------------------------------------------------------- /doc/filtering/skipping.md: -------------------------------------------------------------------------------- 1 | # Skipping test based on ids 2 | 3 | You can tell Kaocha to completely ignore certain tests or test groups, either 4 | with the `--skip` command line flag, or the `:kaocha.filter/skip` test suite 5 | configuration key. 6 | 7 | Both of these take test ids or test group ids (e.g. the fully qualified name 8 | of a test var, or the name of a test namespace). 9 | 10 | ## Background: A simple test suite 11 | 12 | - Given a file named "test/my/project/sample_test.clj" with: 13 | 14 | ``` clojure 15 | (ns my.project.sample-test 16 | (:require [clojure.test :refer :all])) 17 | 18 | (deftest some-test 19 | (is (= 1 1))) 20 | 21 | (deftest other-test 22 | (is (= 2 2))) 23 | ``` 24 | 25 | 26 | - And a file named "test/my/project/other_sample_test.clj" with: 27 | 28 | ``` clojure 29 | (ns my.project.other-sample-test 30 | (:require [clojure.test :refer :all])) 31 | 32 | (deftest other-test 33 | (is (= 3 3))) 34 | ``` 35 | 36 | 37 | 38 | ## Skipping test id from the command line 39 | 40 | - When I run `bin/kaocha --skip my.project.sample-test/some-test --reporter documentation` 41 | 42 | - Then the output should contain: 43 | 44 | ``` nil 45 | --- unit (clojure.test) --------------------------- 46 | my.project.other-sample-test 47 | other-test 48 | 49 | my.project.sample-test 50 | other-test 51 | 52 | 2 tests, 2 assertions, 0 failures. 53 | ``` 54 | 55 | 56 | 57 | ## Skipping a test group id from the command line 58 | 59 | - When I run `bin/kaocha --skip my.project.sample-test --reporter documentation` 60 | 61 | - Then the output should contain: 62 | 63 | ``` nil 64 | --- unit (clojure.test) --------------------------- 65 | my.project.other-sample-test 66 | other-test 67 | 68 | 1 tests, 1 assertions, 0 failures. 69 | ``` 70 | 71 | 72 | 73 | ## Skipping via configuration 74 | 75 | - Given a file named "tests.edn" with: 76 | 77 | ``` edn 78 | #kaocha/v1 79 | {:tests [{:kaocha.filter/skip [my.project.sample-test]}] 80 | :color? false 81 | :randomize? false} 82 | ``` 83 | 84 | 85 | - When I run `bin/kaocha --reporter documentation` 86 | 87 | - Then the output should contain: 88 | 89 | ``` nil 90 | --- unit (clojure.test) --------------------------- 91 | my.project.other-sample-test 92 | other-test 93 | 94 | 1 tests, 1 assertions, 0 failures. 95 | ``` 96 | 97 | 98 | 99 | -------------------------------------------------------------------------------- /test/features/filtering/skipping.feature: -------------------------------------------------------------------------------- 1 | Feature: Skipping test based on ids 2 | 3 | You can tell Kaocha to completely ignore certain tests or test groups, either 4 | with the `--skip` command line flag, or the `:kaocha.filter/skip` test suite 5 | configuration key. 6 | 7 | Both of these take test ids or test group ids (e.g. the fully qualified name 8 | of a test var, or the name of a test namespace). 9 | 10 | Background: A simple test suite 11 | Given a file named "test/my/project/sample_test.clj" with: 12 | """clojure 13 | (ns my.project.sample-test 14 | (:require [clojure.test :refer :all])) 15 | 16 | (deftest some-test 17 | (is (= 1 1))) 18 | 19 | (deftest other-test 20 | (is (= 2 2))) 21 | """ 22 | And a file named "test/my/project/other_sample_test.clj" with: 23 | """clojure 24 | (ns my.project.other-sample-test 25 | (:require [clojure.test :refer :all])) 26 | 27 | (deftest other-test 28 | (is (= 3 3))) 29 | """ 30 | 31 | Scenario: Skipping test id from the command line 32 | When I run `bin/kaocha --skip my.project.sample-test/some-test --reporter documentation` 33 | Then the output should contain: 34 | """ 35 | --- unit (clojure.test) --------------------------- 36 | my.project.other-sample-test 37 | other-test 38 | 39 | my.project.sample-test 40 | other-test 41 | 42 | 2 tests, 2 assertions, 0 failures. 43 | """ 44 | 45 | Scenario: Skipping a test group id from the command line 46 | When I run `bin/kaocha --skip my.project.sample-test --reporter documentation` 47 | Then the output should contain: 48 | """ 49 | --- unit (clojure.test) --------------------------- 50 | my.project.other-sample-test 51 | other-test 52 | 53 | 1 tests, 1 assertions, 0 failures. 54 | """ 55 | 56 | Scenario: Skipping via configuration 57 | Given a file named "tests.edn" with: 58 | """ edn 59 | #kaocha/v1 60 | {:tests [{:kaocha.filter/skip [my.project.sample-test]}] 61 | :color? false 62 | :randomize? false} 63 | """ 64 | When I run `bin/kaocha --reporter documentation` 65 | Then the output should contain: 66 | """ 67 | --- unit (clojure.test) --------------------------- 68 | my.project.other-sample-test 69 | other-test 70 | 71 | 1 tests, 1 assertions, 0 failures. 72 | """ 73 | -------------------------------------------------------------------------------- /src/kaocha/classpath.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.classpath 2 | "This is the add-classpath function from Pomegranate 1.0.0, extracted so we 3 | don't need to pull in Aether." 4 | (:refer-clojure :exclude [add-classpath]) 5 | (:require [dynapath.util :as dp] 6 | [clojure.java.io :as io])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Pomegranate 10 | 11 | (defn ensure-compiler-loader 12 | "Ensures the clojure.lang.Compiler/LOADER var is bound to a DynamicClassLoader, 13 | so that we can add to Clojure's classpath dynamically." 14 | [] 15 | (when-not (bound? Compiler/LOADER) 16 | (.bindRoot Compiler/LOADER (clojure.lang.DynamicClassLoader. (clojure.lang.RT/baseLoader))))) 17 | 18 | (defn- classloader-hierarchy 19 | "Returns a seq of classloaders, with the tip of the hierarchy first. 20 | Uses the current thread context ClassLoader as the tip ClassLoader 21 | if one is not provided." 22 | ([] 23 | (ensure-compiler-loader) 24 | (classloader-hierarchy (deref clojure.lang.Compiler/LOADER))) 25 | ([tip] 26 | (->> tip 27 | (iterate #(.getParent %)) 28 | (take-while boolean)))) 29 | 30 | (defn- modifiable-classloader? 31 | "Returns true iff the given ClassLoader is of a type that satisfies 32 | the dynapath.dynamic-classpath/DynamicClasspath protocol, and it can 33 | be modified." 34 | [cl] 35 | (dp/addable-classpath? cl)) 36 | 37 | (defn add-classpath 38 | "A corollary to the (deprecated) `add-classpath` in clojure.core. This implementation 39 | requires a java.io.File or String path to a jar file or directory, and will attempt 40 | to add that path to the right classloader (with the search rooted at the current 41 | thread's context classloader)." 42 | ([jar-or-dir classloader] 43 | (if-not (dp/add-classpath-url classloader (.toURL (.toURI (io/file jar-or-dir)))) 44 | (throw (IllegalStateException. (str classloader " is not a modifiable classloader"))))) 45 | ([jar-or-dir] 46 | (let [classloaders (classloader-hierarchy)] 47 | (if-let [cl (filter modifiable-classloader? classloaders)] 48 | ;; Add to all classloaders that allow it. Brute force but doesn't hurt. 49 | (run! #(add-classpath jar-or-dir %) cl) 50 | (throw (IllegalStateException. (str "Could not find a suitable classloader to modify from " 51 | classloaders))))))) 52 | 53 | ;; /Pomegranate 54 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 55 | -------------------------------------------------------------------------------- /doc/filtering/focus_meta.md: -------------------------------------------------------------------------------- 1 | # Focusing based on metadata 2 | 3 | You can limit the test run based on test's metadata. How to associate metadata 4 | with a test depends on the test type, for `clojure.test` type tests metadata 5 | can be associated with a test var or test namespace. 6 | 7 | Using the `--focus-meta` command line flag, or `:kaocha.filter/focus-meta` key 8 | in test suite configuration, you can limit the tests being run to only those 9 | where the given metadata key has a truthy value associated with it. 10 | 11 | ## Background: Some tests with metadata 12 | 13 | - Given a file named "test/my/project/sample_test.clj" with: 14 | 15 | ``` clojure 16 | (ns ^:xxx my.project.sample-test 17 | (:require [clojure.test :refer :all])) 18 | 19 | (deftest some-test 20 | (is (= 1 1))) 21 | 22 | (deftest other-test 23 | (is (= 2 2))) 24 | ``` 25 | 26 | 27 | - And a file named "test/my/project/other_sample_test.clj" with: 28 | 29 | ``` clojure 30 | (ns my.project.other-sample-test 31 | (:require [clojure.test :refer :all])) 32 | 33 | (deftest ^:yyy other-test 34 | (is (= 3 3))) 35 | ``` 36 | 37 | 38 | 39 | ## Focusing by metadata from the command line 40 | 41 | - When I run `bin/kaocha --focus-meta :xxx --reporter documentation` 42 | 43 | - Then the output should contain: 44 | 45 | ``` nil 46 | --- unit (clojure.test) --------------------------- 47 | my.project.sample-test 48 | other-test 49 | some-test 50 | 51 | 2 tests, 2 assertions, 0 failures. 52 | ``` 53 | 54 | 55 | 56 | ## Focusing on a test group by metadata from the command line 57 | 58 | - When I run `bin/kaocha --focus-meta :yyy --reporter documentation` 59 | 60 | - Then the output should contain: 61 | 62 | ``` nil 63 | --- unit (clojure.test) --------------------------- 64 | my.project.other-sample-test 65 | other-test 66 | 67 | 1 tests, 1 assertions, 0 failures. 68 | ``` 69 | 70 | 71 | 72 | ## Focusing based on metadata via configuration 73 | 74 | - Given a file named "tests.edn" with: 75 | 76 | ``` edn 77 | #kaocha/v1 78 | {:tests [{:kaocha.filter/focus-meta [:yyy]}] 79 | :color? false 80 | :randomize? false} 81 | ``` 82 | 83 | 84 | - When I run `bin/kaocha --reporter documentation` 85 | 86 | - Then the output should contain: 87 | 88 | ``` nil 89 | --- unit (clojure.test) --------------------------- 90 | my.project.other-sample-test 91 | other-test 92 | 93 | 1 tests, 1 assertions, 0 failures. 94 | ``` 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/version_filter_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.version-filter-test 2 | (:require [clojure.test :refer [is]] 3 | [kaocha.test :refer [deftest]] 4 | [kaocha.plugin.version-filter :as v])) 5 | 6 | (defmacro with-java-version 7 | {:style/indent [1]} 8 | [version & body] 9 | `(let [original# (System/getProperty "java.runtime.version")] 10 | (try 11 | (System/setProperty "java.runtime.version" ~version) 12 | ~@body 13 | (finally 14 | (System/setProperty "java.runtime.version" original#))))) 15 | 16 | (defmacro with-clojure-version 17 | {:style/indent [1]} 18 | [version & body] 19 | `(binding [*clojure-version* (zipmap [:major :minor :incremental] (v/version-vector ~version))] 20 | ~@body)) 21 | 22 | (deftest version-vector-test 23 | (is (= (v/version-vector "1.10.0") [1 10 0])) 24 | (is (= (v/version-vector "1.10.0-beta5") [1 10 0])) 25 | (is (= (v/version-vector "10.0.2+13-Ubuntu-1ubuntu0.18.04.4") [10 0 2]))) 26 | 27 | (deftest compare-versions-test 28 | (is (= 0 (v/compare-versions "1.10.0" "1.10.0"))) 29 | (is (= 0 (v/compare-versions "1.10.0" "1.10"))) 30 | (is (= 0 (v/compare-versions "1.10" "1.10.0"))) 31 | (is (= -1 (v/compare-versions "1.9" "1.10"))) 32 | (is (= 1 (v/compare-versions "1.10" "1.9"))) 33 | (is (= 0 (v/compare-versions "1.10" "1.10.0"))) 34 | (is (= 1 (v/compare-versions "1.10.1" "1.10.0")))) 35 | 36 | (deftest version>=?-test 37 | (is (v/version>=? "1.10.0" "1.9")) 38 | (is (v/version>=? "1.10.0" "1.10")) 39 | (is (not (v/version>=? "1.10.0" "1.10.1"))) 40 | (is (not (v/version>=? "1.9" "1.10.0"))) 41 | (is (v/version>=? "1.10" "1.10.0")) 42 | (is (v/version>=? "1.10.1" "1.10.0")) 43 | (is (v/version>=? "1" nil)) 44 | (is (v/version>=? nil "1")) 45 | (is (v/version>=? nil nil))) 46 | 47 | (deftest skip?-test 48 | (with-clojure-version "1.10.0" 49 | (with-java-version "9" 50 | (is (not (v/skip? {:kaocha.testable/meta {:min-clojure-version "1.10" 51 | :max-java-version "10"}}))))) 52 | 53 | (with-clojure-version "1.9.0" 54 | (with-java-version "9" 55 | (is (v/skip? {:kaocha.testable/meta {:min-clojure-version "1.10" 56 | :max-java-version "10"}})))) 57 | 58 | (with-clojure-version "1.10.0" 59 | (with-java-version "11" 60 | (is (v/skip? {:kaocha.testable/meta {:min-clojure-version "1.10" 61 | :max-java-version "10"}}))))) 62 | -------------------------------------------------------------------------------- /test/features/filtering/focus_meta.feature: -------------------------------------------------------------------------------- 1 | Feature: Focusing based on metadata 2 | 3 | You can limit the test run based on test's metadata. How to associate metadata 4 | with a test depends on the test type, for `clojure.test` type tests metadata 5 | can be associated with a test var or test namespace. 6 | 7 | Using the `--focus-meta` command line flag, or `:kaocha.filter/focus-meta` key 8 | in test suite configuration, you can limit the tests being run to only those 9 | where the given metadata key has a truthy value associated with it. 10 | 11 | Background: Some tests with metadata 12 | Given a file named "test/my/project/sample_test.clj" with: 13 | """clojure 14 | (ns ^:xxx my.project.sample-test 15 | (:require [clojure.test :refer :all])) 16 | 17 | (deftest some-test 18 | (is (= 1 1))) 19 | 20 | (deftest other-test 21 | (is (= 2 2))) 22 | """ 23 | And a file named "test/my/project/other_sample_test.clj" with: 24 | """clojure 25 | (ns my.project.other-sample-test 26 | (:require [clojure.test :refer :all])) 27 | 28 | (deftest ^:yyy other-test 29 | (is (= 3 3))) 30 | """ 31 | 32 | Scenario: Focusing by metadata from the command line 33 | When I run `bin/kaocha --focus-meta :xxx --reporter documentation` 34 | Then the output should contain: 35 | """ 36 | --- unit (clojure.test) --------------------------- 37 | my.project.sample-test 38 | other-test 39 | some-test 40 | 41 | 2 tests, 2 assertions, 0 failures. 42 | """ 43 | 44 | Scenario: Focusing on a test group by metadata from the command line 45 | When I run `bin/kaocha --focus-meta :yyy --reporter documentation` 46 | Then the output should contain: 47 | """ 48 | --- unit (clojure.test) --------------------------- 49 | my.project.other-sample-test 50 | other-test 51 | 52 | 1 tests, 1 assertions, 0 failures. 53 | """ 54 | 55 | Scenario: Focusing based on metadata via configuration 56 | Given a file named "tests.edn" with: 57 | """ edn 58 | #kaocha/v1 59 | {:tests [{:kaocha.filter/focus-meta [:yyy]}] 60 | :color? false 61 | :randomize? false} 62 | """ 63 | When I run `bin/kaocha --reporter documentation` 64 | Then the output should contain: 65 | """ 66 | --- unit (clojure.test) --------------------------- 67 | my.project.other-sample-test 68 | other-test 69 | 70 | 1 tests, 1 assertions, 0 failures. 71 | """ 72 | -------------------------------------------------------------------------------- /test/unit/kaocha/testable_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.testable-test 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.test :as t :refer :all] 4 | [kaocha.report :as report] 5 | [kaocha.testable :as testable] 6 | [kaocha.test-helper] 7 | [kaocha.test-factories :as f])) 8 | 9 | (s/def :kaocha.type/unknown map?) 10 | 11 | (deftest load--default 12 | (is (thrown-ex-data? "No implementation of kaocha.testable/load for :kaocha.type/unknown" 13 | {:kaocha.error/reason :kaocha.error/missing-method, 14 | :kaocha.error/missing-method 'kaocha.testable/load, 15 | :kaocha/testable {:kaocha.testable/type :kaocha.type/unknown 16 | :kaocha.testable/id :foo 17 | :kaocha.testable/desc "foo"}} 18 | (testable/load {:kaocha.testable/type :kaocha.type/unknown 19 | :kaocha.testable/id :foo 20 | :kaocha.testable/desc "foo"})))) 21 | 22 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 23 | 24 | (deftest run--default 25 | (is (thrown-ex-data? "No implementation of kaocha.testable/run for :kaocha.type/unknown" 26 | {:kaocha.error/reason :kaocha.error/missing-method, 27 | :kaocha.error/missing-method 'kaocha.testable/run, 28 | :kaocha/testable #:kaocha.testable{:type :kaocha.type/unknown 29 | :id :foo 30 | :desc "foo"}} 31 | 32 | (testable/run {:kaocha.testable/type :kaocha.type/unknown 33 | :kaocha.testable/id :foo 34 | :kaocha.testable/desc "foo"} 35 | (f/test-plan {}))))) 36 | 37 | 38 | (deftest test-seq-test 39 | (is (= (testable/test-seq 40 | {:kaocha.testable/id :x/_1 41 | :kaocha/tests [{:kaocha.testable/id :y/_1} 42 | {:kaocha.testable/id :z/_1}]}) 43 | [{:kaocha.testable/id :x/_1, 44 | :kaocha/tests [#:kaocha.testable{:id :y/_1} 45 | #:kaocha.testable{:id :z/_1}]} 46 | #:kaocha.testable{:id :y/_1} 47 | #:kaocha.testable{:id :z/_1}]))) 48 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_util.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-util 2 | (:require [clojure.test :as t] 3 | [clojure.spec.alpha :as s] 4 | [expound.alpha :as expound] 5 | [kaocha.report :as report] 6 | [kaocha.testable :as testable] 7 | [kaocha.output :as output] 8 | [kaocha.type :as type])) 9 | 10 | (def ^:dynamic *report-history* nil) 11 | 12 | (defmacro with-test-ctx 13 | "When testing lower level functions, make sure the necessary shared state is set 14 | up. This also helps to isolate us from the outer test runner state." 15 | [opts & body] 16 | `(binding [t/*report-counters* (ref type/initial-counters) 17 | t/*testing-vars* (list) 18 | *report-history* (atom []) 19 | testable/*fail-fast?* (:fail-fast? ~opts)] 20 | (with-redefs [t/report (fn [m#] 21 | (swap! *report-history* conj m#) 22 | (report/report-counters m#) 23 | (when (:fail-fast? ~opts) (report/fail-fast m#)))] 24 | (let [result# (do ~@body)] 25 | {:result result# 26 | :report @*report-history*})))) 27 | 28 | (defmacro with-out-err 29 | "Captures the return value of the expression, as well as anything written on 30 | stdout or stderr." 31 | [& body] 32 | `(let [o# (java.io.StringWriter.) 33 | e# (java.io.StringWriter.)] 34 | (binding [*out* o# 35 | *err* e#] 36 | (let [r# (do ~@body)] 37 | {:out (str o#) 38 | :err (str e#) 39 | :result r#})))) 40 | 41 | (defmacro expect-warning 42 | {:style/indent [1]} 43 | [pattern & body] 44 | `(let [warnings# (atom [])] 45 | (with-redefs [output/warn (fn [& xs#] (swap! warnings# conj xs#))] 46 | (let [result# (do ~@body)] 47 | (when-not (seq (filter #(re-find ~pattern (apply str %)) @warnings#)) 48 | (#'t/do-report {:type :fail 49 | :message (str "Expected test to generate a warning (" 50 | ~pattern 51 | ") but no warning occured.")})) 52 | result#)))) 53 | 54 | (defmacro with-test-out-str 55 | "Evaluates exprs in a context in which [[clojure.test/*test-out*]] is bound to a 56 | fresh StringWriter. Returns the string created by any nested printing calls 57 | that use [[clojure.test/with-test-out]]." 58 | [& body] 59 | `(let [s# (new java.io.StringWriter)] 60 | (binding [clojure.test/*test-out* s#] 61 | ~@body 62 | (str s#)))) 63 | -------------------------------------------------------------------------------- /doc/clojure_test/assertions.md: -------------------------------------------------------------------------------- 1 | # `clojure.test` assertion extensions 2 | 3 | \When running `clojure.test` based tests through Kaocha, some of the behavior 4 | is a little different. Kaocha tries to detect certain scenarios that are 5 | likely mistakes which make a test pass trivially, and turns them into errors 6 | so you can investigate and see what's up. 7 | 8 | Kaocha will also render failures differently, and provides extra multimethods 9 | to influence how certain failures are presented. 10 | 11 | ## Detecting missing assertions 12 | 13 | - Given a file named "test/sample_test.clj" with: 14 | 15 | ``` clojure 16 | (ns sample-test 17 | (:require [clojure.test :refer :all])) 18 | 19 | (deftest my-test 20 | (= 4 5)) 21 | ``` 22 | 23 | 24 | - When I run `bin/kaocha` 25 | 26 | - Then the output should contain: 27 | 28 | ``` text 29 | FAIL in sample-test/my-test (sample_test.clj:4) 30 | Test ran without assertions. Did you forget an (is ...)? 31 | ``` 32 | 33 | 34 | 35 | ## Detecting single argument `=` 36 | 37 | - Given a file named "test/sample_test.clj" with: 38 | 39 | ``` clojure 40 | (ns sample-test 41 | (:require [clojure.test :refer :all])) 42 | 43 | (deftest my-test 44 | (is (= 4) 5)) 45 | ``` 46 | 47 | 48 | - When I run `bin/kaocha` 49 | 50 | - Then the output should contain: 51 | 52 | ``` text 53 | FAIL in sample-test/my-test (sample_test.clj:5) 54 | Equality assertion expects 2 or more values to compare, but only 1 arguments given. 55 | Expected: 56 | (= 4 arg2) 57 | Actual: 58 | (= 4) 59 | 1 tests, 1 assertions, 1 failures. 60 | ``` 61 | 62 | 63 | 64 | ## Pretty printed diffs 65 | 66 | - Given a file named "test/sample_test.clj" with: 67 | 68 | ``` clojure 69 | (ns sample-test 70 | (:require [clojure.test :refer :all])) 71 | 72 | (defn my-fn [] 73 | {:xxx [1 2 3] 74 | :blue :red 75 | "hello" {:world :!}}) 76 | 77 | (deftest my-test 78 | (is (= {:xxx [1 3 4] 79 | "hello" {:world :?}} 80 | {:xxx [1 2 3] 81 | :blue :red 82 | "hello" {:world :!}}))) 83 | ``` 84 | 85 | 86 | - When I run `bin/kaocha` 87 | 88 | - Then the output should contain: 89 | 90 | ``` text 91 | FAIL in sample-test/my-test (sample_test.clj:10) 92 | Expected: 93 | {"hello" {:world :?}, :xxx [1 3 4]} 94 | Actual: 95 | {"hello" {:world -:? +:!}, :xxx [1 +2 3 -4], +:blue :red} 96 | 1 tests, 1 assertions, 1 failures. 97 | ``` 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /src/kaocha/assertions.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.assertions 2 | (:require [kaocha.output :as output] 3 | [clojure.test :as t] 4 | [kaocha.report :as report] 5 | [puget.color :as color] 6 | [clojure.string :as str] 7 | [slingshot.slingshot :refer [try+ throw+]])) 8 | 9 | (defmethod t/assert-expr 'substring? [msg form] 10 | (let [[_ s1 s2] form] 11 | `(if (.contains ~s2 ~s1) 12 | (t/do-report {:type :pass :message ~msg}) 13 | (t/do-report {:type :fail 14 | :expected (list '~'substring? ~s1 ~s2) 15 | :actual (list '~'not '~form) 16 | :message ~msg})))) 17 | 18 | (def x-last 19 | "Reducing version of [[clojure.core/last]]" 20 | (completing (fn [_ x] x))) 21 | 22 | (defn longest-substring [s1 s2] 23 | (transduce (comp (map #(subs s1 0 (inc %))) 24 | (take-while #(.contains s2 %))) 25 | x-last 26 | nil 27 | (range (count s1)))) 28 | 29 | (defn show-trailing-whitespace [s] 30 | (str/replace s 31 | #"(?m)[ \h\x0B\f\r\x85\u2028\u2029]+$" 32 | (fn [s] 33 | (output/colored :red-bg s)))) 34 | 35 | (defmethod report/print-expr 'substring? [{:keys [expected] :as m}] 36 | (let [[_ s1 s2] expected 37 | long-sub (longest-substring s1 s2) 38 | remainder (subs s1 (count long-sub)) 39 | printer (output/printer {:color-scheme {::long-sub [:green] 40 | ::header [:blue]}})] 41 | (output/print-doc 42 | [:span 43 | "Expected: (substring? needle haystack)" 44 | :break 45 | (color/document printer ::header (output/colored :underline "Haystack:")) 46 | :break 47 | (show-trailing-whitespace s2) 48 | :break 49 | (color/document printer ::header (output/colored :underline "Needle:")) 50 | :break 51 | (color/document printer ::long-sub long-sub) 52 | (show-trailing-whitespace remainder)]))) 53 | 54 | #_ 55 | (defmethod t/assert-expr 'thrown+? [msg form] 56 | (let [expr (second form) 57 | body (nthnext form 2)] 58 | `(try+ 59 | ~@body 60 | (t/do-report {:type :fail, :message ~msg, 61 | :expected '~form, :actual nil}) 62 | (catch ~expr e# 63 | (t/do-report {:type :pass, :message ~msg, 64 | :expected '~form, :actual e#}) 65 | e#)))) 66 | 67 | ;; Configured as a pre-load hook 68 | (defn load-assertions [config] 69 | (require 'matcher-combinators.test) 70 | config) 71 | -------------------------------------------------------------------------------- /test/features/clojure_test/assertions.feature: -------------------------------------------------------------------------------- 1 | Feature: `clojure.test` assertion extensions 2 | 3 | \When running `clojure.test` based tests through Kaocha, some of the behavior 4 | is a little different. Kaocha tries to detect certain scenarios that are 5 | likely mistakes which make a test pass trivially, and turns them into errors 6 | so you can investigate and see what's up. 7 | 8 | Kaocha will also render failures differently, and provides extra multimethods 9 | to influence how certain failures are presented. 10 | 11 | Scenario: Detecting missing assertions 12 | Given a file named "test/sample_test.clj" with: 13 | """ clojure 14 | (ns sample-test 15 | (:require [clojure.test :refer :all])) 16 | 17 | (deftest my-test 18 | (= 4 5)) 19 | """ 20 | When I run `bin/kaocha` 21 | Then the output should contain: 22 | """ text 23 | FAIL in sample-test/my-test (sample_test.clj:4) 24 | Test ran without assertions. Did you forget an (is ...)? 25 | """ 26 | 27 | Scenario: Detecting single argument `=` 28 | Given a file named "test/sample_test.clj" with: 29 | """ clojure 30 | (ns sample-test 31 | (:require [clojure.test :refer :all])) 32 | 33 | (deftest my-test 34 | (is (= 4) 5)) 35 | """ 36 | When I run `bin/kaocha` 37 | Then the output should contain: 38 | """ text 39 | FAIL in sample-test/my-test (sample_test.clj:5) 40 | Equality assertion expects 2 or more values to compare, but only 1 arguments given. 41 | Expected: 42 | (= 4 arg2) 43 | Actual: 44 | (= 4) 45 | 1 tests, 1 assertions, 1 failures. 46 | """ 47 | 48 | Scenario: Pretty printed diffs 49 | Given a file named "test/sample_test.clj" with: 50 | """ clojure 51 | (ns sample-test 52 | (:require [clojure.test :refer :all])) 53 | 54 | (defn my-fn [] 55 | {:xxx [1 2 3] 56 | :blue :red 57 | "hello" {:world :!}}) 58 | 59 | (deftest my-test 60 | (is (= {:xxx [1 3 4] 61 | "hello" {:world :?}} 62 | {:xxx [1 2 3] 63 | :blue :red 64 | "hello" {:world :!}}))) 65 | """ 66 | When I run `bin/kaocha` 67 | Then the output should contain: 68 | """ text 69 | FAIL in sample-test/my-test (sample_test.clj:10) 70 | Expected: 71 | {"hello" {:world :?}, :xxx [1 3 4]} 72 | Actual: 73 | {"hello" {:world -:? +:!}, :xxx [1 +2 3 -4], +:blue :red} 74 | 1 tests, 1 assertions, 1 failures. 75 | """ 76 | -------------------------------------------------------------------------------- /test/step_definitions/kaocha_integration.clj: -------------------------------------------------------------------------------- 1 | (ns features.steps.kaocha-integration 2 | (:require [clojure.java.io :as io] 3 | [clojure.java.shell :as shell] 4 | [clojure.string :as str] 5 | [clojure.test :as t :refer :all] 6 | [kaocha.integration-helpers :refer :all] 7 | [kaocha.output :as output] 8 | [kaocha.shellwords :refer [shellwords]] 9 | [lambdaisland.cucumber.dsl :refer :all] 10 | [me.raynes.fs :as fs])) 11 | 12 | (require 'kaocha.assertions) 13 | 14 | (Given "a file named {string} with:" [m path contents] 15 | (spit-file m path contents)) 16 | 17 | (def last-cpcache-dir (atom nil)) 18 | 19 | (When "I run `(.*)`" [m args] 20 | (let [{:keys [config-file dir] :as m} (test-dir-setup m)] 21 | 22 | (when-let [cache @last-cpcache-dir] 23 | (let [target (join dir ".cpcache")] 24 | (when-not (.isDirectory (io/file target)) 25 | (mkdir target) 26 | (run! #(fs/copy % (io/file (join target (.getName %)))) (fs/glob cache "*"))))) 27 | 28 | (let [result (apply shell/sh (conj (shellwords args) 29 | :dir dir))] 30 | ;; By default these are hidden unless the test fails 31 | (when (seq (:out result)) 32 | (println (str dir) "$" args) 33 | (println (str (output/colored :underline "stdout") ":\n" (:out result)))) 34 | (when (seq (:err result)) 35 | (println (str (output/colored :underline "stderr") ":\n" (:err result)))) 36 | (let [cpcache (io/file (join dir ".cpcache"))] 37 | (when (.exists cpcache) 38 | (reset! last-cpcache-dir cpcache))) 39 | (merge m result)))) 40 | 41 | (Then "the exit-code is non-zero" [{:keys [exit] :as m}] 42 | (is (not= "0" exit)) 43 | m) 44 | 45 | (Then "the exit-code should be {int}" [{:keys [exit] :as m} code] 46 | (is (= code (Integer. exit))) 47 | m) 48 | 49 | (Then "the output should contain:" [m output] 50 | (is (substring? output (:out m))) 51 | m) 52 | 53 | (Then "the output should be" [m output] 54 | (is (= (str/trim output) (str/trim (:out m)))) 55 | m) 56 | 57 | (Then "the output should not contain" [m output] 58 | (is (not (str/includes? (:out m) output))) 59 | m) 60 | 61 | (Then "stderr should contain" [m output] 62 | (is (substring? output (:err m))) 63 | m) 64 | 65 | (Then "print output" [m] 66 | (t/with-test-out 67 | (println "----out---------------------------------------") 68 | (println (:out m)) 69 | (println "----err---------------------------------------") 70 | (println (:err m))) 71 | m) 72 | 73 | #_ 74 | (do 75 | (require 'kaocha.repl) 76 | (kaocha.repl/run :plugins.version-filter {:kaocha.plugin.capture-output/capture-output? false 77 | } 78 | )) 79 | -------------------------------------------------------------------------------- /doc/06_focusing_and_skipping.md: -------------------------------------------------------------------------------- 1 | # 6. Focusing and Skipping 2 | 3 | Often you will want to *skip* certain tests, so they don't get run, or you want 4 | to *focus* on specific tests, so only those get run. 5 | 6 | For example: 7 | 8 | - Skip tests that aren't finished yet 9 | - Skip tests marked as slow 10 | - Focus on a test that previously failed 11 | 12 | You can skip tests, or focus on tests, either based on the test ID, or on test 13 | or namespace metadata, based on four command line flags and configuration keys. 14 | 15 | ``` shell 16 | --skip SYM Skip tests with this ID and their children. 17 | --focus SYM Only run this test, skip others. 18 | --skip-meta SYM Skip tests where this metadata key is truthy. 19 | --focus-meta SYM Only run tests where this metadata key is truthy. 20 | ``` 21 | 22 | ``` clojure 23 | #kaocha/v1 24 | {:tests [{:id :unit 25 | :skip [...] 26 | :focus [...] 27 | :skip-meta [...] 28 | :focus-meta [...]}]} 29 | ``` 30 | 31 | ## Matching 32 | 33 | ### On id 34 | 35 | A test id is a namespaced symbol, for clojure.test tests this is the 36 | fully qualified name of the test var. You can skip or focus on such a test 37 | either by providing its full name, or just the namespace part. 38 | 39 | So you can run a single test with 40 | 41 | ``` shell 42 | bin/kaocha --focus com.my.project-test/foo-test 43 | ``` 44 | 45 | To run all tests in that namespace, use 46 | 47 | ``` shell 48 | bin/kaocha --focus com.my.project-test 49 | ``` 50 | 51 | ### On metadata 52 | 53 | Suppose you have test that are checked into source code, but that still need 54 | work. You can mark these with a metadata tag: 55 | 56 | ``` clojure 57 | (deftest ^:pending my-test 58 | ,,,) 59 | ``` 60 | 61 | To ignore such tests, add a `:skip-meta` key to the test suite config: 62 | 63 | ``` clojure 64 | #kaocha/v1 65 | {:tests [{:id :unit 66 | :skip-meta [:pending]}]} 67 | ``` 68 | 69 | This also works for metadata placed on the test's namespace, or any other 70 | metadata that a given test type implementation exposes. For example 71 | kaocha-cucumber converts scenario tags into metadata. 72 | 73 | ### Focusing on metadata: special case 74 | 75 | `--focus-meta` will only work if at least one test has this metadata tag. If not 76 | a single test matches then this metadata is ignored. Assuming no other filters 77 | are in effect this will result in running all tests. 78 | 79 | This way you can configure a certain key in `tests.edn` that you can use when 80 | you want to zone in on a specific test. Add the metadata to the test and only 81 | this test runs, remove it and the whole suite runs. 82 | 83 | ``` clojure 84 | #kaocha/v1 85 | {:tests [{:focus-meta [:xxx]}]} 86 | ``` 87 | 88 | ```clojure 89 | (deftest ^:xxx my-test 90 | ,,,) 91 | ``` 92 | -------------------------------------------------------------------------------- /doc/plugins/notifier_plugin.md: -------------------------------------------------------------------------------- 1 | # Plugin: Notifier (desktop notifications) 2 | 3 | Desktop notifications can be enabled with the `:kaocha.plugin/notifier` 4 | plugin. This will pop up a fail/pass notification bubble including a summary 5 | of tests passed/errored/failed at the end of each test run. It's particularly 6 | useful in combination with `--watch`, e.g. `bin/kaocha --plugin notifier 7 | --watch`. 8 | 9 | It does this by invoking a shell command which can be configured, so it can be 10 | used to invoke an arbitrary command or script. By default it will try to 11 | detect which command to use, using either `notify-send` (Linux) or 12 | `terminal-notifier` (Mac OS X), either of which may need to be installed 13 | first. 14 | 15 | Several replacement patterns are available: 16 | 17 | - `%{title}` : The notification title, either `⛔️ Failing` or `✅ Passing` 18 | - `%{message}` : Test result summary, e.g. `5 tests, 12 assertions, 0 failures` 19 | - `%{icon}` : Full local path to an icon to use (currently uses the Clojure icon) 20 | - `%{failed?}` : `true` if any tests failed or errored, `false` otherwise 21 | - `%{count}` : the number of tests 22 | - `%{pass}` : the number of passing assertions 23 | - `%{fail}` : the number of failing assertions 24 | - `%{error}` : the number of errors 25 | - `%{pending}` : the number of pending tests 26 | - `%{urgency}` : `normal` if the tests pass, `critical` otherwise, meant for use with `notify-send` 27 | 28 | If no command is configured, and neither notification command is found, then 29 | the plugin will silently do nothing. You can explicitly inhibit its behaviour 30 | with `--no-notifications`. 31 | 32 | ## Enabling Desktop Notifications 33 | 34 | - Given a file named "tests.edn" with: 35 | 36 | ``` clojure 37 | #kaocha/v1 38 | {:plugins [:kaocha.plugin/notifier] 39 | 40 | ;; Configuring a command is optional. Since CI does not have desktop 41 | ;; notifications we pipe to a file instead. 42 | :kaocha.plugin.notifier/command 43 | "sh -c 'echo \"%{title}\n%{message}\n%{failed?}\n%{count}\n%{urgency}\" > /tmp/kaocha.txt'" 44 | 45 | ;; Fallbacks: 46 | 47 | ;; :kaocha.plugin.notifier/command 48 | ;; "notify-send -a Kaocha %{title} %{message} -i %{icon} -u %{urgency}" 49 | 50 | ;; :kaocha.plugin.notifier/command 51 | ;; "terminal-notifier -message %{message} -title %{title} -appIcon %{icon}" 52 | } 53 | ``` 54 | 55 | 56 | - And a file named "test/sample_test.clj" with: 57 | 58 | ``` clojure 59 | (ns sample-test 60 | (:require [clojure.test :refer :all])) 61 | 62 | (deftest simple-fail-test 63 | (is (= :same :not-same))) 64 | ``` 65 | 66 | 67 | - When I run `bin/kaocha` 68 | 69 | - And I run `cat /tmp/kaocha.txt` 70 | 71 | - Then the output should contain: 72 | 73 | ``` nil 74 | ⛔️ Failing 75 | 1 tests, 1 failures. 76 | true 77 | 1 78 | critical 79 | ``` 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /test/features/plugins/notifier_plugin.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Notifier (desktop notifications) 2 | 3 | Desktop notifications can be enabled with the `:kaocha.plugin/notifier` 4 | plugin. This will pop up a fail/pass notification bubble including a summary 5 | of tests passed/errored/failed at the end of each test run. It's particularly 6 | useful in combination with `--watch`, e.g. `bin/kaocha --plugin notifier 7 | --watch`. 8 | 9 | It does this by invoking a shell command which can be configured, so it can be 10 | used to invoke an arbitrary command or script. By default it will try to 11 | detect which command to use, using either `notify-send` (Linux) or 12 | `terminal-notifier` (Mac OS X), either of which may need to be installed 13 | first. 14 | 15 | Several replacement patterns are available: 16 | 17 | - `%{title}` : The notification title, either `⛔️ Failing` or `✅ Passing` 18 | - `%{message}` : Test result summary, e.g. `5 tests, 12 assertions, 0 failures` 19 | - `%{icon}` : Full local path to an icon to use (currently uses the Clojure icon) 20 | - `%{failed?}` : `true` if any tests failed or errored, `false` otherwise 21 | - `%{count}` : the number of tests 22 | - `%{pass}` : the number of passing assertions 23 | - `%{fail}` : the number of failing assertions 24 | - `%{error}` : the number of errors 25 | - `%{pending}` : the number of pending tests 26 | - `%{urgency}` : `normal` if the tests pass, `critical` otherwise, meant for use with `notify-send` 27 | 28 | If no command is configured, and neither notification command is found, then 29 | the plugin will silently do nothing. You can explicitly inhibit its behaviour 30 | with `--no-notifications`. 31 | 32 | Scenario: Enabling Desktop Notifications 33 | Given a file named "tests.edn" with: 34 | """ clojure 35 | #kaocha/v1 36 | {:plugins [:kaocha.plugin/notifier] 37 | 38 | ;; Configuring a command is optional. Since CI does not have desktop 39 | ;; notifications we pipe to a file instead. 40 | :kaocha.plugin.notifier/command 41 | "sh -c 'echo \"%{title}\n%{message}\n%{failed?}\n%{count}\n%{urgency}\" > /tmp/kaocha.txt'" 42 | 43 | ;; Fallbacks: 44 | 45 | ;; :kaocha.plugin.notifier/command 46 | ;; "notify-send -a Kaocha %{title} %{message} -i %{icon} -u %{urgency}" 47 | 48 | ;; :kaocha.plugin.notifier/command 49 | ;; "terminal-notifier -message %{message} -title %{title} -appIcon %{icon}" 50 | } 51 | """ 52 | And a file named "test/sample_test.clj" with: 53 | """ clojure 54 | (ns sample-test 55 | (:require [clojure.test :refer :all])) 56 | 57 | (deftest simple-fail-test 58 | (is (= :same :not-same))) 59 | """ 60 | When I run `bin/kaocha` 61 | And I run `cat /tmp/kaocha.txt` 62 | Then the output should contain: 63 | """ 64 | ⛔️ Failing 65 | 1 tests, 1 failures. 66 | true 67 | 1 68 | critical 69 | """ 70 | -------------------------------------------------------------------------------- /src/kaocha/plugin.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin 2 | (:require [kaocha.output :as output] 3 | [clojure.string :as str] 4 | [slingshot.slingshot :refer [try+ throw+]])) 5 | 6 | (def ^:dynamic *current-chain* []) 7 | 8 | (defmacro with-plugins [chain & body] 9 | `(binding [*current-chain* ~chain] ~@body)) 10 | 11 | ;; TODO: duplicated from testable, not sure yet where to put it. 12 | (defn- try-require [n] 13 | (try 14 | (require n) 15 | true 16 | (catch java.io.FileNotFoundException e 17 | false))) 18 | 19 | (defn try-load-third-party-lib [type] 20 | (if (qualified-keyword? type) 21 | (when-not (try-require (symbol (str (namespace type) "." (name type)))) 22 | (try-require (symbol (namespace type)))) 23 | (try-require (symbol (name type))))) 24 | 25 | (defmulti -register "Add your plugin to the stack" 26 | (fn [name plugins] name)) 27 | 28 | (defmethod -register :default [name plugins] 29 | (output/error "Couldn't load plugin " name) 30 | (throw+ {:kaocha/early-exit 254} nil (str "Couldn't load plugin " name))) 31 | 32 | (defn register [name plugins] 33 | (try-load-third-party-lib name) 34 | (-register name plugins)) 35 | 36 | (defn load-all [names] 37 | (reduce #(register %2 %1) [] (distinct names))) 38 | 39 | (defn run-hook* [plugins step value & extra-args] 40 | (reduce (fn [value plugin] 41 | (if-let [step-fn (get plugin step)] 42 | (let [value (apply step-fn value extra-args)] 43 | (when (nil? value) 44 | (output/warn "Plugin " (:id plugin) " hook " step " returned nil.")) 45 | value) 46 | value)) 47 | value 48 | plugins)) 49 | 50 | (defn run-hook [step value & extra-args] 51 | (apply run-hook* *current-chain* step value extra-args)) 52 | 53 | (defmacro defplugin 54 | {:style/indent [1 :form [1]]} 55 | [id & hooks] 56 | (let [plugin-id (keyword id) 57 | var-sym (symbol (str (name id) "-hooks")) 58 | [desc & hooks] (if (string? (first hooks)) 59 | hooks 60 | (cons "" hooks))] 61 | `(do 62 | ~@(map (fn [[hook & fn-tail]] 63 | `(defn ~(symbol (str (name id) "-" hook "-hook")) ~@fn-tail)) 64 | hooks) 65 | 66 | (def ~var-sym 67 | ~(into {:kaocha.plugin/id plugin-id 68 | :kaocha.plugin/description desc} 69 | (map (fn [[hook & _]] 70 | [(keyword "kaocha.hooks" (str hook)) 71 | (symbol (str (name id) "-" hook "-hook"))])) 72 | hooks)) 73 | (defmethod -register ~plugin-id [_# plugins#] 74 | (conj plugins# ~var-sym))))) 75 | 76 | (comment 77 | (= (run-hook [{:foo inc} {:foo inc}] :foo 2) 78 | 4)) 79 | 80 | 81 | ;; HOOKS 82 | 83 | ;; :cli-options 84 | ;; :config 85 | ;; :pre-load 86 | ;; :post-load 87 | ;; :pre-run 88 | ;; :post-run 89 | ;; :wrap-run 90 | ;; :pre-test 91 | ;; :post-test 92 | ;; :pre-report 93 | -------------------------------------------------------------------------------- /doc/05_running_kaocha_repl.md: -------------------------------------------------------------------------------- 1 | # 5. Running Kaocha From the REPL 2 | 3 | For REPL use there's the 4 | [kaocha.repl](https://cljdoc.xyz/d/lambdaisland/kaocha/CURRENT/api/kaocha.repl) 5 | namespace. Its main entry point is the 6 | [run](https://cljdoc.xyz/d/lambdaisland/kaocha/CURRENT/api/kaocha.repl#run) 7 | function. Calling it is similar to starting Kaocha from the CLI, it will load 8 | `tests.edn`, merge in any extra options and flags, and then load and run your 9 | test suites. 10 | 11 | ## Filtering tests 12 | 13 | As arguments to `run` you pass it one or more identifiers of things you want to 14 | test. This can be a test suite, a namespace, or a specific test var. 15 | 16 | Say you have a single `:unit` test suite, with a `kaocha.random-test` namespace 17 | containing two tests. 18 | 19 | ``` 20 | . 21 | └── :unit 22 |     └── kaocha.random-test 23 |         ├── kaocha.random-test/rand-ints-test 24 |         └── kaocha.random-test/randomize-test 25 | 26 | ``` 27 | 28 | You could run the whole suite 29 | 30 | ``` clojure 31 | (use 'kaocha.repl) 32 | 33 | (run :unit) 34 | ``` 35 | 36 | The namespace 37 | 38 | ``` clojure 39 | (run 'kaocha.random-test) 40 | ``` 41 | 42 | Or specific test vars 43 | 44 | ``` clojure 45 | (run 'kaocha.random-test/rand-ints-test 'kaocha.random-test/randomize-test) 46 | ``` 47 | 48 | These are equivalent to using `--focus` on the command line. `run` also 49 | understand namespace and var objects. 50 | 51 | 52 | ``` clojure 53 | (run *ns*) 54 | (run #'rand-ints-test) 55 | ``` 56 | 57 | `(run)` without any arguments is equivalent to `(run *ns*)`. If you really want to run all test suites without discrimination, use [run-all](https://cljdoc.org/d/lambdaisland/kaocha/CURRENT/api/kaocha.repl#run-all). 58 | 59 | 60 | ## Passing configuration 61 | 62 | If the last argument to `(run)` is a map, then it is considered extra 63 | configuration which is applied on top of what is read from `tests.edn`. The 64 | special key `:config-file` is available to change the location from which 65 | `tests.edn` is read. 66 | 67 | ``` clojure 68 | (run {:config-file "/tmp/my_tests.edn"}) 69 | ``` 70 | 71 | Other keys in the map need to be either fully qualified keywords as used in 72 | Kaocha's configuration, or the short equivalent that is available in `tests.edn` 73 | when using the `#kaocha/v1` reader tag. 74 | 75 | ## In-buffer eval 76 | 77 | `kaocha.repl` is especially useful when used with a editor-connected REPL, so 78 | that code can be evaluated in place. When working on a specific test you can 79 | wrap it in `kaocha.repl/run`. Since `deftest` returns the var it defines, this 80 | redefines and runs the test in one go. 81 | 82 | ``` clojure 83 | (kaocha.repl/run 84 | (deftest my-test 85 | ,,,)) 86 | ``` 87 | 88 | When using CIDER this combines really well with 89 | `cider-pprint-eval-defun-at-point` (binding in CIDER 1.18.1: `C-c C-f`). 90 | 91 | ## Config and Test plan 92 | 93 | The `(kaocha.repl/config)` and `(kaocha.config/test-plan)` functions are very 94 | useful when diagnosing issues, and can be helpful when developing plugins or 95 | test types. 96 | -------------------------------------------------------------------------------- /src/kaocha/matcher_combinators.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.matcher-combinators 2 | (:require [kaocha.report :as report] 3 | [kaocha.output :as output] 4 | [kaocha.hierarchy :as hierarchy] 5 | [clojure.test :as t] 6 | [lambdaisland.deep-diff :as ddiff] 7 | [lambdaisland.deep-diff.printer :as printer] 8 | [puget.printer :as puget] 9 | [fipp.engine :as fipp] 10 | [puget.color :as color])) 11 | 12 | (hierarchy/derive! :mismatch :kaocha/fail-type) 13 | (hierarchy/derive! :mismatch :kaocha/known-key) 14 | 15 | (hierarchy/derive! :matcher-combinators/mismatch :kaocha/fail-type) 16 | (hierarchy/derive! :matcher-combinators/mismatch :kaocha/known-key) 17 | 18 | (def print-handlers {'matcher_combinators.model.Mismatch 19 | (fn [printer expr] 20 | (printer/print-mismatch printer {:- (:expected expr) 21 | :+ (:actual expr)})) 22 | 23 | 'matcher_combinators.model.Missing 24 | (fn [printer expr] 25 | (printer/print-deletion printer {:- (:expected expr)})) 26 | 27 | 'matcher_combinators.model.Unexpected 28 | (fn [printer expr] 29 | (printer/print-insertion printer {:+ (:actual expr)})) 30 | 31 | 'matcher_combinators.model.FailedPredicate 32 | (fn [printer expr] 33 | [:group 34 | [:align 35 | (printer/print-other printer (:form expr)) 36 | (printer/print-insertion printer {:+ (:actual expr)})]]) 37 | 38 | 'matcher_combinators.model.InvalidMatcherType 39 | (fn [printer expr] 40 | [:group 41 | [:align 42 | (color/document printer 43 | ::printer/other 44 | [:span "-" 45 | [:raw (:expected-type-msg expr)]]) 46 | (printer/print-insertion printer {:+ (:provided expr)})]])}) 47 | 48 | (run! #(apply printer/register-print-handler! %) print-handlers) 49 | 50 | (defn fail-summary [{:keys [testing-contexts testing-vars] :as m}] 51 | (let [printer (ddiff/printer {:print-color output/*colored-output*})] 52 | (println (str "\n" (output/colored :red "FAIL") " in") (clojure.test/testing-vars-str m)) 53 | (when (seq t/*testing-contexts*) 54 | (println (t/testing-contexts-str))) 55 | (when-let [message (:message m)] 56 | (println message)) 57 | (fipp/pprint-document 58 | [:span 59 | "Mismatch:" :line 60 | [:nest (puget/format-doc printer (:markup m))]] 61 | {:width (:width printer)}) 62 | (report/print-output m))) 63 | 64 | (defmethod report/fail-summary :mismatch [m] (fail-summary m)) 65 | (defmethod report/fail-summary :matcher-combinators/mismatch [m] (fail-summary m)) 66 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/randomize_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.randomize-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.test-helper :refer :all] 4 | [kaocha.plugin :as plugin] 5 | [kaocha.testable :as testable])) 6 | 7 | 8 | (def plugin-chain (plugin/register :kaocha.plugin/randomize [])) 9 | 10 | (def test-suite {:kaocha.testable/type :kaocha.type/clojure.test 11 | :kaocha.testable/id :c 12 | :kaocha.testable/desc "c (clojure.test)" 13 | :kaocha/source-paths [] 14 | :kaocha/test-paths ["fixtures/c-tests"] 15 | :kaocha/ns-patterns [".*"]}) 16 | 17 | (deftest randomize-test 18 | (plugin/with-plugins plugin-chain 19 | (is (match? {:kaocha.plugin.randomize/randomize? true 20 | :kaocha.plugin.randomize/seed number?} 21 | (plugin/run-hook :kaocha.hooks/config {}))) 22 | 23 | (is (match? {:kaocha.testable/type :kaocha.type/clojure.test 24 | :kaocha.test-plan/tests [{:kaocha.testable/type :kaocha.type/ns 25 | :kaocha.testable/id :foo.hello-test 26 | :kaocha.testable/desc "foo.hello-test" 27 | :kaocha.test-plan/tests 28 | [{:kaocha.testable/id :foo.hello-test/pass-1} 29 | {:kaocha.testable/id :foo.hello-test/fail-1} 30 | {:kaocha.testable/id :foo.hello-test/pass-3} 31 | {:kaocha.testable/id :foo.hello-test/pass-2}]}]} 32 | 33 | (plugin/run-hook :kaocha.hooks/post-load 34 | (-> test-suite 35 | (assoc :kaocha.plugin.randomize/seed 123 36 | :kaocha.plugin.randomize/randomize? true) 37 | testable/load)))) 38 | 39 | (is (match? {:kaocha.testable/type :kaocha.type/clojure.test 40 | :kaocha.test-plan/tests [{:kaocha.testable/type :kaocha.type/ns 41 | :kaocha.testable/id :foo.hello-test 42 | :kaocha.testable/desc "foo.hello-test" 43 | :kaocha.test-plan/tests 44 | [{:kaocha.testable/id :foo.hello-test/pass-2} 45 | {:kaocha.testable/id :foo.hello-test/pass-3} 46 | {:kaocha.testable/id :foo.hello-test/fail-1} 47 | {:kaocha.testable/id :foo.hello-test/pass-1}]}]} 48 | 49 | (plugin/run-hook :kaocha.hooks/post-load 50 | (-> test-suite 51 | (assoc :kaocha.plugin.randomize/seed 456 52 | :kaocha.plugin.randomize/randomize? true) 53 | testable/load)))))) 54 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_helper.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-helper 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :as t] 4 | [kaocha.core-ext :refer :all] 5 | [matcher-combinators.result :as mc.result] 6 | [matcher-combinators.core :as mc.core] 7 | [matcher-combinators.model :as mc.model] 8 | [clojure.spec.alpha :as s] 9 | [expound.alpha :as expound] 10 | [orchestra.spec.test :as orchestra]) 11 | (:import [clojure.lang ExceptionInfo])) 12 | 13 | (orchestra/instrument) 14 | 15 | (extend-protocol mc.core/Matcher 16 | clojure.lang.Var 17 | (match [this actual] 18 | (if (= this actual) 19 | {::mc.result/type :match 20 | ::mc.result/value actual 21 | ::mc.result/weight 0} 22 | {::mc.result/type :mismatch 23 | ::mc.result/value (if (and (keyword? actual) (= ::mc.core/missing actual)) 24 | (mc.model/->Missing this) 25 | (mc.model/->Mismatch this actual)) 26 | ::mc.result/weight 1}))) 27 | 28 | ;; TODO move to kaocha.assertions 29 | 30 | (defmacro thrown-ex-data? 31 | "Verifies that an expression throws an ExceptionInfo with specific data and 32 | message. Message can be string or regex. " 33 | [ex-msg ex-data & body] 34 | (assert nil "thrown-ex-data? used outside (is ) block")) 35 | 36 | (defmethod clojure.test/assert-expr 'thrown-ex-data? [msg form] 37 | (let [[_ ex-msg ex-data & body] form] 38 | `(try 39 | ~@body 40 | (t/do-report {:type :fail 41 | :message ~msg 42 | :expected '~form 43 | :actual nil}) 44 | (catch ExceptionInfo e# 45 | (let [m# (.getMessage e#) 46 | d# (clojure.core/ex-data e#)] 47 | (cond 48 | (not (or (and (string? ~ex-msg) (= ~ex-msg m#)) 49 | (and (regex? ~ex-msg) (re-find ~ex-msg m#)))) 50 | (t/do-report {:type :fail 51 | :message ~msg 52 | :expected '~ex-msg 53 | :actual m#}) 54 | 55 | (not= ~ex-data d#) 56 | (t/do-report {:type :fail 57 | :message ~msg 58 | :expected ~ex-data 59 | :actual d#}) 60 | 61 | :else 62 | (t/do-report {:type :pass 63 | :message ~msg 64 | :expected '~form 65 | :actual e#}))) 66 | true)))) 67 | 68 | (defn spec-valid? 69 | "Asserts that the value matches the spec." 70 | [spec value & [msg]] 71 | (s/valid? spec value)) 72 | 73 | (defmethod t/assert-expr 'spec-valid? [msg form] 74 | `(let [[spec# value#] (list ~@(rest form))] 75 | (t/do-report 76 | (if (s/valid? spec# value#) 77 | {:type :pass 78 | :message ~msg 79 | :expected '~form 80 | :actual '~form} 81 | {:type :fail 82 | :message (or ~msg (expound/expound-str spec# value#)) 83 | :expected '~form 84 | :actual (list '~'not '~form)})))) 85 | -------------------------------------------------------------------------------- /src/kaocha/hierarchy.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.hierarchy 2 | (:refer-clojure :exclude [isa?])) 3 | 4 | (defonce hierarchy (make-hierarchy)) 5 | 6 | (defn derive! 7 | "Add a parent/child relationship to kaocha's keyword hierarchy." 8 | [tag parent] 9 | (alter-var-root #'hierarchy derive tag parent)) 10 | 11 | (derive! :fail :kaocha/fail-type) 12 | (derive! :error :kaocha/fail-type) 13 | 14 | (derive! :pass :kaocha/known-key) 15 | (derive! :fail :kaocha/known-key) 16 | (derive! :error :kaocha/known-key) 17 | (derive! :begin-test-suite :kaocha/known-key) 18 | (derive! :end-test-suite :kaocha/known-key) 19 | (derive! :begin-test-var :kaocha/known-key) 20 | (derive! :end-test-var :kaocha/known-key) 21 | (derive! :summary :kaocha/known-key) 22 | (derive! :kaocha/pending :kaocha/known-key) 23 | 24 | (derive! :begin-test-ns :kaocha/begin-group) 25 | (derive! :end-test-ns :kaocha/end-group) 26 | 27 | (derive! :kaocha/begin-group :kaocha/known-key) 28 | (derive! :kaocha/end-group :kaocha/known-key) 29 | 30 | (derive! :begin-test-var :kaocha/begin-test) 31 | (derive! :end-test-var :kaocha/end-test) 32 | 33 | (derive! :kaocha/begin-test :kaocha/known-key) 34 | (derive! :kaocha/end-test :kaocha/known-key) 35 | 36 | (derive! :kaocha/deferred :kaocha/known-key) 37 | 38 | (defn isa? [tag parent] 39 | (or (clojure.core/isa? tag parent) 40 | (clojure.core/isa? hierarchy tag parent))) 41 | 42 | ;; Test event types 43 | 44 | (defn fail-type? 45 | "Fail-type types indicate a failing test." 46 | [event] 47 | (isa? (:type event) :kaocha/fail-type)) 48 | 49 | (defn error-type? 50 | "Error-type indicates a test that failed because of an exception." 51 | [event] 52 | (isa? (:type event) :error)) 53 | 54 | (defn pass-type? 55 | "Error-type indicates a test that failed because of an exception." 56 | [event] 57 | (isa? (:type event) :pass)) 58 | 59 | (defn known-key? 60 | "Known keys don't get propogated to clojure.test/report, our own reporters 61 | already handle them." 62 | [event] 63 | (isa? (:type event) :kaocha/known-key)) 64 | 65 | (defn deferred? 66 | "Deferred events get propagated to clojure.test/report, but only during the 67 | summary step." 68 | [event] 69 | (isa? (:type event) :kaocha/deferred)) 70 | 71 | (defn pending? 72 | "A test that generates a pending event will not be executed, but explicitly 73 | reported as being pending i.e. still needing to be implemented. Tests with 74 | the :kaocha/pending metadata will automatically generate a pending event." 75 | [event] 76 | (isa? (:type event) :kaocha/pending)) 77 | 78 | ;; Testable types 79 | 80 | (defn suite? 81 | "Top level testables are called suites, e.g. a suite of clojure.test tests." 82 | [testable] 83 | (isa? (:kaocha.testable/type testable) :kaocha.testable.type/suite)) 84 | 85 | (defn group? 86 | "Intermediary testables are called groups, e.g. a namespace of tests." 87 | [testable] 88 | (isa? (:kaocha.testable/type testable) :kaocha.testable.type/group)) 89 | 90 | (defn leaf? 91 | "This is a leaf in the tree of testables, i.e. it's an actual test with 92 | assertions, not just a container for tests. 93 | 94 | :kaocha.type/var is a leaf type, :kaocha.type/ns is not." 95 | [testable] 96 | (isa? (:kaocha.testable/type testable) :kaocha.testable.type/leaf)) 97 | -------------------------------------------------------------------------------- /src/kaocha/specs.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.specs 2 | (:require [clojure.spec.alpha :as s] 3 | [clojure.spec.gen.alpha :as gen] 4 | [clojure.test :as t] 5 | [expound.alpha :as expound])) 6 | 7 | (def global-opts [:kaocha/reporter 8 | :kaocha/color? 9 | :kaocha/fail-fast? 10 | :kaocha/watch? 11 | :kaocha/plugins]) 12 | 13 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 14 | ;; config 15 | 16 | (s/def :kaocha/config (s/keys :req ~(conj global-opts :kaocha/tests))) 17 | 18 | (s/def :kaocha/tests (s/coll-of :kaocha/testable)) 19 | 20 | (s/def :kaocha/testable (s/keys :req [:kaocha.testable/type 21 | :kaocha.testable/id] 22 | :opt [:kaocha.testable/meta 23 | :kaocha.testable/wrap])) 24 | 25 | (s/def :kaocha.testable/type qualified-keyword?) 26 | 27 | (s/def :kaocha.testable/id keyword?) 28 | 29 | ;; Short description as used by the documentation reporter. No newlines. 30 | (s/def :kaocha.testable/desc string?) 31 | 32 | (s/def :kaocha.testable/wrap (s/coll-of fn? :into [])) 33 | 34 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 35 | ;; Test plan 36 | 37 | (s/def :kaocha/test-plan (s/keys :req ~(conj global-opts :kaocha.test-plan/tests))) 38 | 39 | (s/def :kaocha.test-plan/tests (s/coll-of :kaocha.test-plan/testable)) 40 | 41 | (s/def :kaocha.test-plan/testable (s/and :kaocha/testable 42 | (s/keys :req [] 43 | :opt [:kaocha.testable/desc 44 | :kaocha.test-plan/tests 45 | :kaacha.test-plan/load-error]))) 46 | 47 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 48 | ;; result 49 | 50 | (s/def :kaocha/result (s/keys :req ~(conj global-opts :kaocha.result/tests))) 51 | 52 | (s/def :kaocha.result/tests (s/coll-of :kaocha.result/testable)) 53 | 54 | (s/def :kaocha.result/testable (s/and :kaocha.test-plan/testable 55 | (s/keys :opt [:kaocha.result/count 56 | :kaocha.result/tests 57 | :kaocha.result/pass 58 | :kaocha.result/error 59 | :kaocha.result/fail 60 | :kaocha.result/out 61 | :kaocha.result/err 62 | :kaocha.result/time]))) 63 | 64 | (s/def :kaocha.result/count nat-int?) 65 | (s/def :kaocha.result/pass nat-int?) 66 | (s/def :kaocha.result/fail nat-int?) 67 | (s/def :kaocha.result/error nat-int?) 68 | 69 | (s/def :kaocha.result/out string?) 70 | (s/def :kaocha.result/err string?) 71 | 72 | (s/def :kaocha.result/time nat-int?) 73 | 74 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 75 | ;; helpers 76 | 77 | (defn assert-spec [spec value] 78 | (when-not (s/valid? spec value) 79 | (throw (AssertionError. (expound/expound-str spec value))))) 80 | -------------------------------------------------------------------------------- /doc/08_plugins.md: -------------------------------------------------------------------------------- 1 | # 8. Plugins 2 | 3 | This section describes plug-ins that are built-in but not enabled by default. Functionality of default plugins is described in the [Running Kaocha CLI](04_running_kaocha_cli.md) section. 4 | 5 | For information on *writing* plugins, see the section on [extending Kaocha](09_extending.md). 6 | 7 | ## Profiling 8 | 9 | The profiling plugin will output a list of the slowest tests for each test type at the end of the test run. 10 | 11 | ### Enabling 12 | 13 | ``` shell 14 | bin/kaocha --plugin kaocha.plugin/profiling 15 | ``` 16 | 17 | or 18 | 19 | ``` clojure 20 | #kaocha/v1 21 | {:plugins [:kaocha.plugin/profiling]} 22 | ``` 23 | 24 | ### Example output 25 | 26 | ``` 27 | Top 2 slowest kaocha.type/clojure.test (37.97308 seconds, 100.0% of total time) 28 | integration 29 | 37.06616 seconds average (37.06616 seconds / 1 tests) 30 | unit 31 | 0.08245 seconds average (0.90692 seconds / 11 tests) 32 | 33 | Top 3 slowest kaocha.type/ns (37.71151 seconds, 99.3% of total time) 34 | kaocha.integration-test 35 | 37.06421 seconds average (37.06421 seconds / 1 tests) 36 | kaocha.type.var-test 37 | 0.57622 seconds average (0.57622 seconds / 1 tests) 38 | kaocha.runner-test 39 | 0.03554 seconds average (0.07108 seconds / 2 tests) 40 | 41 | Top 3 slowest kaocha.type/var (37.70178 seconds, 99.3% of total time) 42 | kaocha.integration-test/command-line-runner-test 43 | 37.06206 seconds kaocha/integration_test.clj:25 44 | kaocha.type.var-test/run-test 45 | 0.57399 seconds kaocha/type/var_test.clj:12 46 | kaocha.runner-test/main-test 47 | 0.06573 seconds kaocha/runner_test.clj:10 48 | ``` 49 | 50 | ### Plugin-specific command line flags 51 | 52 | ``` 53 | --[no-]profiling Show slowest tests of each type with timing information. 54 | --profiling-count NUM Show this many slow tests of each kind in profile results. 55 | ``` 56 | 57 | ### Plugin-specific configuration options 58 | 59 | Shown with their default values: 60 | 61 | ``` 62 | #kaocha/v1 63 | {:kaocha.plugin.profiling/count 3 64 | :kaocha.plugin.profiling/profiling? true} 65 | ``` 66 | 67 | ## Print invocations 68 | 69 | At the end of the test run, print command invocations that can be copy-pasted to re-run only specific failed tests. 70 | 71 | ### Enabling 72 | 73 | ``` shell 74 | bin/kaocha --plugin kaocha.plugin/print-invocations 75 | ``` 76 | 77 | or 78 | 79 | ``` clojure 80 | #kaocha/v1 81 | {:plugins [:kaocha.plugin/print-invocations]} 82 | ``` 83 | 84 | ### Example output 85 | 86 | ``` shell 87 | [(.)(F..)(..)(...)(............)(....)(...........)(......)(....)(.)(...)] 88 | 89 | FAIL in (clojure-test-summary-test) (kaocha/history_test.clj:5) 90 | ... 91 | 18 test vars, 50 assertions, 1 failures. 92 | 93 | bin/kaocha --focus 'kaocha.history-test/clojure-test-summary-test' 94 | ``` 95 | 96 | ## Notifier 97 | 98 | Pop up a system notifications whenever a test run fails or passes. 99 | 100 | See [Plugins: Notifier](plugins/notifier.md) 101 | 102 | ## Version filter 103 | 104 | Skip tests that aren't compatible with the current version of Java or Clojure. 105 | 106 | See [Plugins: Version Filter](plugins/version_filter.md) 107 | 108 | ## Hooks 109 | 110 | Write functions that hook into various parts of Kaocha 111 | 112 | See [Plugins: Hooks](plugins/hooks.md) 113 | -------------------------------------------------------------------------------- /test/unit/kaocha/output_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.output-test 2 | (:require [kaocha.output :as output] 3 | [clojure.test :refer :all] 4 | [kaocha.test-util :as util])) 5 | 6 | (deftest colored-test 7 | (is (= "foo" (output/colored :green "foo"))) 8 | 9 | (is (= "foo" 10 | (binding [output/*colored-output* false] 11 | (output/colored :green "foo"))))) 12 | 13 | (deftest warn-test 14 | (testing "without color" 15 | (is (= {:err "WARNING: Oh no!\n", :out "", :result nil} 16 | (binding [output/*colored-output* false] 17 | (util/with-out-err 18 | (output/warn "Oh no!")))))) 19 | 20 | (testing "with color" 21 | (is (= {:err "WARNING: Oh no!\n", :out "", :result nil} 22 | (util/with-out-err 23 | (output/warn "Oh no!"))))) 24 | 25 | (testing "multiple arguments" 26 | (is (= {:err "WARNING: one mississippi, two mississippi\n", :out "", :result nil} 27 | (util/with-out-err 28 | (output/warn "one mississippi" ", " "two mississippi")))))) 29 | 30 | (deftest error-test 31 | (testing "without color" 32 | (is (= {:err "ERROR: Oh no!\n", :out "", :result nil} 33 | (binding [output/*colored-output* false] 34 | (util/with-out-err 35 | (output/error "Oh no!")))))) 36 | 37 | (testing "with color" 38 | (is (= {:err "ERROR: Oh no!\n", :out "", :result nil} 39 | (util/with-out-err 40 | (output/error "Oh no!"))))) 41 | 42 | (testing "multiple arguments" 43 | (is (= {:err "ERROR: one mississippi, two mississippi\n", :out "", :result nil} 44 | (util/with-out-err 45 | (output/error "one mississippi" ", " "two mississippi")))))) 46 | 47 | (deftest format-doc-test 48 | (testing "without color" 49 | (is (= '[:group "[" [:align ([:group "{" [:align ([:span ":x" " " ":y"])] "}"])] "]"] 50 | (binding [output/*colored-output* false] 51 | (output/format-doc [{:x :y}]))))) 52 | 53 | (testing "with color" 54 | (is (= '[:group 55 | [:span [:pass ""] "[" [:pass ""]] 56 | [:align 57 | ([:group 58 | [:span [:pass ""] "{" [:pass ""]] 59 | [:align 60 | ([:span 61 | [:span [:pass ""] ":x" [:pass ""]] 62 | " " 63 | [:span [:pass ""] ":y" [:pass ""]]])] 64 | [:span [:pass ""] "}" [:pass ""]]])] 65 | [:span [:pass ""] "]" [:pass ""]]] 66 | (output/format-doc [{:x :y}]))))) 67 | 68 | 69 | (deftest print-doc-test 70 | (testing "prints with fipp" 71 | (is (= {:err "" 72 | :out "[:aaa :bbb :ccc]\n", 73 | :result nil} 74 | (util/with-out-err 75 | (-> (output/format-doc [:aaa :bbb :ccc]) 76 | (output/print-doc)))))) 77 | 78 | (testing "respects *print-length*" 79 | (is (= {:err "", 80 | :out "[:aaa\n :bbb\n :ccc]\n", 81 | :result nil} 82 | (util/with-out-err 83 | (binding [*print-length* 1] 84 | (-> (output/format-doc [:aaa :bbb :ccc]) 85 | (output/print-doc)))))))) 86 | -------------------------------------------------------------------------------- /doc/02_installing.md: -------------------------------------------------------------------------------- 1 | ## 2. Installing 2 | 3 | Kaocha is distributed through [Clojars](https://clojars.org), with the 4 | identifier `lambdaisland/kaocha`. You can find version information for the 5 | latest release at [https://clojars.org/lambdaisland/kaocha](https://clojars.org/lambdaisland/kaocha). 6 | 7 | The main namespace for use at the command line is `kaocha.runner`, regardless of which tool you're using to invoke Clojure. 8 | 9 | For example: 10 | 11 | ``` shell 12 | clojure -Sdeps '{:deps {lambdaisland/kaocha {:mvn/version "0.0-389"}}}' -m kaocha.runner --test-help 13 | ``` 14 | 15 | Below are instructions on the recommended way to set things up for various build tools. 16 | 17 | ### Clojure CLI / deps.edn 18 | 19 | In `deps.edn`, create a `test` "alias" (profile) that loads the `lambdaisland/kaocha` dependency. 20 | 21 | ``` clojure 22 | ;; deps.edn 23 | {:deps { ,,, } 24 | :aliases 25 | {:test {:extra-deps {lambdaisland/kaocha {:mvn/version "0.0-389"}}}}} 26 | ``` 27 | 28 | Other dependencies that are only used for tests like test framework or assertion 29 | libraries can also go here. 30 | 31 | Next create a `bin/kaocha` wrapper script. Having it in this location is 32 | strongly recommended, as its where developers coming from other projects will 33 | expect to find it. 34 | 35 | In it invoke `clojure` with the `:test` alias and the `kaocha.runner` main 36 | namespace. This is what `bin/kaocha` by default looks like. Make sure to add 37 | `"$@"` so that any arguments to `bin/kaocha` are passed on to `kaocha.runner`. 38 | 39 | ``` shell 40 | #!/bin/bash 41 | 42 | clojure -A:test -m kaocha.runner "$@" 43 | ``` 44 | 45 | Make sure the script is executable 46 | 47 | ``` shell 48 | chmod +x bin/kaocha 49 | ``` 50 | 51 | This script provides a useful place to encode extra flags or setup that is 52 | needed in order for tests to run correctly. 53 | 54 | ``` shell 55 | #!/bin/bash 56 | 57 | . secrets.env 58 | clojure -J-Xmx512m -A:dev:test -m kaocha.runner --config-file test/tests.edn "$@" 59 | ``` 60 | 61 | This version also sets an alternative location for Kaocha's configuration file: 62 | `tests.edn`. It is generally recommended to leave it at the root of the project, 63 | but if you do want to move or rename it this is the way to go. 64 | 65 | `--config-file` is the only Kaocha option that makes sense in this script, other 66 | Kaocha configuration should be done through `tests.edn`. 67 | 68 | Now you can invoke Kaocha as such: 69 | 70 | ``` shell 71 | bin/kaocha --version 72 | ``` 73 | 74 | ### Leiningen 75 | 76 | Add a `:kaocha` profile, where the Kaocha dependency is included, then add an 77 | alias that activates the profile, and invokes `lein run -m kaocha.runner`. 78 | 79 | ``` clojure 80 | (defproject my-proj "0.1.0" 81 | :dependencies [,,,] 82 | :profiles {:kaocha {:dependencies [[lambdaisland/kaocha "0.0-389"]]}} 83 | :aliases {"kaocha" ["with-profile" "+kaocha" "run" "-m" "kaocha.runner"]}) 84 | ``` 85 | 86 | Now you can invoke Kaocha as such: 87 | 88 | ``` shell 89 | lein kaocha --version 90 | ``` 91 | 92 | It is still recommeded to create a `bin/kaocha` wrapper for consistency among 93 | projects. The rest of the documentation assumes you can invoke Kaocha with 94 | `bin/kaocha`. 95 | 96 | ``` shell 97 | #!/bin/bash 98 | 99 | lein kaocha "$@" 100 | ``` 101 | 102 | ### Boot 103 | 104 | See [kaocha-boot](https://github.com/lambdaisland/kaocha-boot) for instructions. 105 | -------------------------------------------------------------------------------- /doc/07_watch_mode.md: -------------------------------------------------------------------------------- 1 | ## 7. Watch mode 2 | 3 | You can enable watch mode with the `--watch` command line flag. If you want 4 | watch mode to be the default you can also configure it in `tests.edn` with 5 | `:watch? true`. In that case the `--no-watch` flag turns it off again. 6 | 7 | When running in watch mode Kaocha will keep an eye on your test and source 8 | directories (as configured on the test suites), as well as your Kaocha 9 | configuration in `tests.edn`. Whenever any of these files changes it tries to 10 | reload the changed files, and then runs your tests again. 11 | 12 | Watch mode is based on `tools.namespace`, this library keeps track of the 13 | dependencies between namespaces. When a file changes then any namespace that 14 | depends on it gets unloaded first, completely erasing the namespace and its vars 15 | from Clojure's memory, before loading them again from scratch. 16 | 17 | This fixes a lot of issues that are present with more naive code reloading 18 | schemes, but it comes with its own set of caveats. Refer to the [tools.namespace 19 | README](https://github.com/clojure/tools.namespace) for more information. 20 | 21 | If any tests fail, then upon the next change first the failed tests will be run. 22 | Only when they pass is the complete suite run again. 23 | 24 | Sometimes your source or test directories will contain files that should be 25 | ignored by watch mode, for instance temporary files left by your editor. You can 26 | tell watch mode to ignore these with the `:kaocha.watch/ignore` configuration 27 | key. 28 | 29 | This takes a vector of patterns which largely behave like Unix-shell style 30 | "glob" patterns, although they differ from standard shell behavior in some 31 | subtle ways. These are processed using Java's 32 | [PathMatcher](https://docs.oracle.com/javase/10/docs/api/java/nio/file/FileSystem.html#getPathMatcher(java.lang.String)) 33 | interface, the provided links describes how they work in detail. 34 | 35 | ``` clojure 36 | #kaocha/v1 37 | {:kaocha.watch/ignore ["*.tmp"]} 38 | ``` 39 | 40 | When running in watch mode you can press the Enter (Return) key to manually 41 | trigger a re-run of the tests. This will always run all tests, not just the 42 | tests that failed on the last run. 43 | 44 | Interrupt the process (Ctrl-C) to exit Kaocha's watch mode. 45 | 46 | ## Tips and tricks 47 | 48 | The `:kaocha.plugin/notifier` plugin will cause a system notification to pop up 49 | whenever a test run finishes. This works really well together with watch mode, 50 | as it means you can leave Kaocha running in the background, and only switch over 51 | when a test failed. 52 | 53 | Watch mode is most useful when it can provide quick feedback, this is why it's a 54 | good idea to combine it with `--fail-fast`, so you know as soon as possible when 55 | a test fails, and you can focus on one test at a time. You can disable 56 | randomization (`--no-randomize`) to prevent having to jump back and forth 57 | between different failing tests. 58 | 59 | If your test suites takes a long time to run then watch mode will be a lot less 60 | effective, in that case consider tagging your slowest tests with metadata, and 61 | filtering them out. (`--skip-meta :slow`) 62 | 63 | `tests.edn` is also watched for changes, and gets reloaded on every run, so 64 | there's a lot you can do without having to restart Kaocha. 65 | 66 | - enable `:fail-fast? true` 67 | - focus on specific tests or namespaces 68 | - enable extra plugins 69 | - set a fixed seed (when debugging ordering issue) 70 | - switch to a different reporter 71 | -------------------------------------------------------------------------------- /src/kaocha/plugin/capture_output.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.capture-output 2 | (:require [kaocha.plugin :as plugin :refer [defplugin]] 3 | [kaocha.testable :as testable] 4 | [kaocha.hierarchy :as hierarchy]) 5 | (:import [java.io OutputStream ByteArrayOutputStream PrintStream PrintWriter])) 6 | 7 | ;; Many props to eftest for much of this code 8 | 9 | (def ^:dynamic *test-buffer* nil) 10 | 11 | (def active-buffers (atom #{})) 12 | 13 | (defn make-buffer [] 14 | (ByteArrayOutputStream.)) 15 | 16 | (defn read-buffer [buffer] 17 | (when buffer 18 | (-> buffer (.toByteArray) (String.)))) 19 | 20 | (defmacro with-test-buffer [buffer & body] 21 | `(try 22 | (swap! active-buffers conj ~buffer) 23 | (binding [*test-buffer* ~buffer] 24 | ~@body) 25 | (finally 26 | (swap! active-buffers disj ~buffer)))) 27 | 28 | (defn- doto-capture-buffer [f] 29 | (if *test-buffer* 30 | (f *test-buffer*) 31 | (run! f @active-buffers))) 32 | 33 | (defn create-proxy-output-stream ^OutputStream [] 34 | (proxy [OutputStream] [] 35 | (write 36 | ([data] 37 | (if (instance? Integer data) 38 | (doto-capture-buffer #(.write % ^int data)) 39 | (doto-capture-buffer #(.write % ^bytes data 0 (alength ^bytes data))))) 40 | ([data off len] 41 | (doto-capture-buffer #(.write % data off len)))))) 42 | 43 | (defn init-capture [] 44 | (let [old-out System/out 45 | old-err System/err 46 | proxy-output-stream (create-proxy-output-stream) 47 | new-stream (PrintStream. proxy-output-stream) 48 | new-writer (PrintWriter. proxy-output-stream)] 49 | (System/setOut new-stream) 50 | (System/setErr new-stream) 51 | {:captured-writer new-writer 52 | :old-system-out old-out 53 | :old-system-err old-err})) 54 | 55 | (defn restore-capture [{:keys [old-system-out old-system-err]}] 56 | (System/setOut old-system-out) 57 | (System/setErr old-system-err)) 58 | 59 | (defmacro with-capture [& body] 60 | `(let [context# (init-capture) 61 | writer# (:captured-writer context#)] 62 | (try 63 | (binding [*out* writer#, *err* writer#] 64 | (with-redefs [*out* writer#, *err* writer#] 65 | ~@body)) 66 | (finally 67 | (restore-capture context#))))) 68 | 69 | (defplugin kaocha.plugin/capture-output 70 | (cli-options [opts] 71 | (conj opts [nil "--[no-]capture-output" "Capture output during tests."])) 72 | 73 | (config [config] 74 | (let [cli-flag (get-in config [:kaocha/cli-options :capture-output])] 75 | (assoc config ::capture-output? 76 | (if (some? cli-flag) 77 | cli-flag 78 | (::capture-output? config true))))) 79 | 80 | (wrap-run [run test-plan] 81 | (if (::capture-output? test-plan) 82 | (fn [& args] 83 | (with-capture (apply run args))) 84 | run)) 85 | 86 | (pre-test [testable test-plan] 87 | (if (::capture-output? test-plan) 88 | (let [buffer (make-buffer)] 89 | (cond-> testable 90 | (hierarchy/leaf? testable) 91 | (-> (assoc ::buffer buffer) 92 | (update :kaocha.testable/wrap conj (fn [t] #(with-test-buffer buffer (t))))))) 93 | testable)) 94 | 95 | (post-test [testable test-plan] 96 | (if (and (::capture-output? test-plan) (::buffer testable)) 97 | (-> testable 98 | (assoc ::output (read-buffer (::buffer testable))) 99 | (dissoc ::buffer)) 100 | testable))) 101 | -------------------------------------------------------------------------------- /test/unit/kaocha/type/clojure/test_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.clojure.test-test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :refer :all] 4 | [kaocha.core-ext :refer :all] 5 | [kaocha.testable :as testable] 6 | [kaocha.test-util :refer [with-test-ctx]])) 7 | 8 | (def test-suite {:kaocha.testable/type :kaocha.type/clojure.test 9 | :kaocha.testable/id :a 10 | :kaocha/source-paths [] 11 | :kaocha/test-paths ["fixtures/a-tests"] 12 | :kaocha/ns-patterns [".*"]}) 13 | 14 | (deftest load-test 15 | (is (match? {:kaocha.testable/type :kaocha.type/clojure.test 16 | :kaocha.testable/id :a 17 | :kaocha.testable/desc "a (clojure.test)" 18 | :kaocha/source-paths [] 19 | :kaocha/test-paths ["fixtures/a-tests"] 20 | :kaocha/ns-patterns [".*"] 21 | :kaocha.test-plan/tests [{:kaocha.testable/type :kaocha.type/ns 22 | :kaocha.testable/id :foo.bar-test 23 | :kaocha.ns/name 'foo.bar-test 24 | :kaocha.ns/ns ns? 25 | :kaocha.test-plan/tests [{:kaocha.testable/type :kaocha.type/var 26 | :kaocha.testable/id :foo.bar-test/a-test 27 | :kaocha.var/name 'foo.bar-test/a-test 28 | :kaocha.var/var var? 29 | :kaocha.var/test fn?}]}]} 30 | (testable/load test-suite)))) 31 | 32 | (deftest run-test 33 | (let [test-plan (testable/load test-suite)] 34 | (is (match? {:kaocha.testable/type :kaocha.type/clojure.test 35 | :kaocha.testable/id :a 36 | :kaocha.testable/desc "a (clojure.test)" 37 | :kaocha/source-paths [] 38 | :kaocha/test-paths ["fixtures/a-tests"] 39 | :kaocha/ns-patterns [".*"] 40 | :kaocha.result/tests [{:kaocha.testable/type :kaocha.type/ns 41 | :kaocha.testable/id :foo.bar-test 42 | :kaocha.ns/name 'foo.bar-test 43 | :kaocha.ns/ns ns? 44 | :kaocha.result/tests [{:kaocha.testable/type :kaocha.type/var 45 | :kaocha.testable/id :foo.bar-test/a-test 46 | :kaocha.var/name 'foo.bar-test/a-test 47 | :kaocha.var/var var? 48 | :kaocha.var/test fn? 49 | :kaocha.result/count 1 50 | :kaocha.result/pass 1 51 | :kaocha.result/error 0 52 | :kaocha.result/fail 0 53 | :kaocha.result/pending 0}]}]} 54 | 55 | (:result (with-test-ctx {} (testable/run test-plan test-plan))))))) 56 | -------------------------------------------------------------------------------- /src/kaocha/type/var.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.var 2 | (:require [clojure.test :as t] 3 | [kaocha.type :as type] 4 | [kaocha.testable :as testable] 5 | [kaocha.result :as result] 6 | [kaocha.report :as report] 7 | [kaocha.hierarchy :as hierarchy] 8 | [clojure.spec.alpha :as s] 9 | [clojure.spec.gen.alpha :as gen] 10 | [clojure.string :as str]) 11 | (:import [clojure.lang Var])) 12 | 13 | (hierarchy/derive! ::zero-assertions :kaocha/known-key) 14 | (hierarchy/derive! ::zero-assertions :kaocha/fail-type) 15 | 16 | (defmethod report/fail-summary ::zero-assertions [{:keys [testing-contexts testing-vars] :as m}] 17 | (println "\nFAIL in" (report/testing-vars-str m)) 18 | (when (seq testing-contexts) 19 | (println (str/join " " testing-contexts))) 20 | (println "Test ran without assertions. Did you forget an (is ...)?") 21 | (report/print-output m)) 22 | 23 | (defmethod testable/-run :kaocha.type/var [{test :kaocha.var/test 24 | wrap :kaocha.testable/wrap 25 | the-var :kaocha.var/var 26 | meta' :kaocha.testable/meta 27 | :as testable} test-plan] 28 | (type/with-report-counters 29 | (let [test (reduce #(%2 %1) test wrap)] 30 | (binding [t/*testing-vars* (conj t/*testing-vars* the-var)] 31 | (t/do-report {:type :begin-test-var, :var the-var}) 32 | (try 33 | (test) 34 | (catch clojure.lang.ExceptionInfo e 35 | (when-not (:kaocha/fail-fast (ex-data e)) 36 | (t/do-report {:type :error 37 | :message "Uncaught exception, not in assertion." 38 | :expected nil 39 | :actual e 40 | :kaocha.result/exception e}))) 41 | (catch Throwable e 42 | (t/do-report {:type :error 43 | :message "Uncaught exception, not in assertion." 44 | :expected nil 45 | :actual e 46 | :kaocha.result/exception e})))) 47 | (let [{::result/keys [pass error fail pending] :as result} (type/report-count)] 48 | (when (= pass error fail pending 0) 49 | (binding [testable/*fail-fast?* false 50 | testable/*test-location* {:file (:file meta') :line (:line meta')}] 51 | (t/do-report {:type ::zero-assertions}))) 52 | (t/do-report {:type :end-test-var, :var the-var}) 53 | (merge testable {:kaocha.result/count 1} (type/report-count)))))) 54 | 55 | (s/def :kaocha.type/var (s/keys :req [:kaocha.testable/type 56 | :kaocha.testable/id 57 | :kaocha.var/name 58 | :kaocha.var/var 59 | :kaocha.var/test])) 60 | 61 | (s/def :kaocha.var/name qualified-symbol?) 62 | (s/def :kaocha.var/test (s/spec ifn? 63 | :gen (fn [] 64 | (gen/one-of [(gen/return (fn [] (t/is true))) 65 | (gen/return (fn [] (t/is false)))])))) 66 | (s/def :kaocha.var/var (s/spec var? 67 | :gen (fn [] 68 | (gen/return (.setDynamic (Var/create)))))) 69 | 70 | (hierarchy/derive! :kaocha.type/var :kaocha.testable.type/leaf) 71 | -------------------------------------------------------------------------------- /src/kaocha/plugin/profiling.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.profiling 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as str] 4 | [kaocha.plugin :as plugin :refer [defplugin]] 5 | [kaocha.testable :as testable]) 6 | (:import java.time.Instant 7 | java.time.temporal.ChronoUnit)) 8 | 9 | (defn start [testable] 10 | (assoc testable ::start (Instant/now))) 11 | 12 | (defn stop [testable] 13 | (cond-> testable 14 | (::start testable) 15 | (assoc ::duration (.until (::start testable) 16 | (Instant/now) 17 | ChronoUnit/NANOS)))) 18 | 19 | (defplugin kaocha.plugin/profiling 20 | (pre-run [test-plan] 21 | (start test-plan)) 22 | 23 | (post-run [test-plan] 24 | (stop test-plan)) 25 | 26 | (pre-test [testable _] 27 | (start testable)) 28 | 29 | (post-test [testable _] 30 | (stop testable)) 31 | 32 | (cli-options [opts] 33 | (conj opts 34 | [nil "--[no-]profiling" "Show slowest tests of each type with timing information."] 35 | [nil "--profiling-count NUM" "Show this many slow tests of each kind in profile results." 36 | :parse-fn #(Integer/parseInt %)])) 37 | 38 | (config [{:kaocha/keys [cli-options] :as config}] 39 | (assoc config 40 | ::profiling? (:profiling cli-options (::profiling? config true)) 41 | ::count (:profiling-count cli-options (::count config 3)))) 42 | 43 | (post-summary [result] 44 | (when (::profiling? result) 45 | (let [tests (->> result 46 | testable/test-seq 47 | (remove :kaocha.test-plan/load-error) 48 | (remove ::testable/skip)) 49 | types (group-by :kaocha.testable/type tests) 50 | total-dur (::duration result) 51 | limit (::count result)] 52 | (->> (for [[type tests] types 53 | :when type 54 | :let [slowest (take limit (reverse (sort-by ::duration tests))) 55 | slow-test-dur (apply + (keep ::duration slowest))]] 56 | [(format "\nTop %s slowest %s (%.5f seconds, %.1f%% of total time)\n" 57 | (count slowest) 58 | (subs (str type) 1) 59 | (float (/ slow-test-dur 1e9)) 60 | (float (* (/ slow-test-dur total-dur) 100))) 61 | (for [test slowest 62 | :let [duration (::duration test) 63 | cnt (count (remove ::testable/skip (:kaocha.result/tests test)))] 64 | :when duration] 65 | (if (> cnt 0) 66 | (format " %s\n \033[1m%.5f seconds\033[0m average (%.5f seconds / %d tests)\n" 67 | (subs (str (:kaocha.testable/id test)) 1) 68 | (float (/ duration cnt 1e9)) 69 | (float (/ duration 1e9)) 70 | cnt) 71 | 72 | (when (:file (:kaocha.testable/meta test)) 73 | (format " %s\n \033[1m%.5f seconds\033[0m %s:%d\n" 74 | (subs (str (:kaocha.testable/id test)) 1) 75 | (float (/ duration 1e9)) 76 | (str/replace (:file (:kaocha.testable/meta test)) 77 | (str (.getCanonicalPath (io/file ".")) "/") 78 | "") 79 | (:line (:kaocha.testable/meta test))))))]) 80 | (flatten) 81 | (apply str) 82 | print) 83 | (flush))) 84 | result)) 85 | -------------------------------------------------------------------------------- /doc/04_running_kaocha_cli.md: -------------------------------------------------------------------------------- 1 | # 4. Running Kaocha CLI 2 | 3 | If you followed the [installation](02_installing.md) instructions, you 4 | should have a `bin/kaocha` wrapper ("binstub") in your project that accepts 5 | additional arguments. 6 | 7 | The command line runner takes the names of test suites to run, as well as a 8 | number of flags and options. If you don't give it any suite names it runs all 9 | suites. 10 | 11 | You can get an overview of all available flags with `--test-help`. 12 | 13 | ``` shell 14 | bin/kaocha --test-help 15 | ``` 16 | 17 | ## tests.edn 18 | 19 | Kaocha relies on a `tests.edn` configuration file, see the section on 20 | [configuration](03_configuration.md). To load an alternative configuration, use 21 | the `--config-file` option. 22 | 23 | ``` shell 24 | bin/kaocha --config-file tests_ci.edn 25 | ``` 26 | 27 | Note that plugins specified in the config file will influence the available 28 | command line options. 29 | 30 | ## Plugins and reporters 31 | 32 | The `--plugin` option lets you activate additional plugins, it is followed by 33 | the plugin's name, a namespaced symbol. This option can be specified multiple 34 | times. 35 | 36 | With `--reporter` you can specify an alternative reporter, to change Kaocha's 37 | output. 38 | 39 | For example, to see a colorful progress bar, use 40 | 41 | ``` shell 42 | bin/kaocha --reporter kaocha.report.progress/progress 43 | ``` 44 | 45 | Plugins in the `kaocha.plugin` namespace, and reporters in the `kaocha.report` 46 | namespace can be specified without the namespace. 47 | 48 | ``` shell 49 | bin/kaocha --plugin profiling --reporter documentation 50 | ``` 51 | 52 | ## Fail fast mode 53 | 54 | ``` shell 55 | bin/kaocha --fail-fast 56 | ``` 57 | 58 | Stop the test run as soon as a single assertion fails or an exception is thrown, 59 | and then print the results so far. 60 | 61 | ## Randomization 62 | 63 | Kaocha by default randomizes the order that tests are run: it picks a random 64 | seed, and uses that to re-order the test suites, namespaces, and test vars. 65 | 66 | Tests should be independent, but this is not always the case. This random order 67 | helps to track down unintended dependencies between tests. 68 | 69 | The random seed will be printed at the start of the test run. On the same code 70 | base with the same seed you will always get the same test order. This way you 71 | can e.g. reproduce a test run that failed on a build server. 72 | 73 | ``` shell 74 | bin/kaocha --seed 10761431 75 | ``` 76 | 77 | Use `--no-randomize` to load suites in the order they are specified, and vars in 78 | the order they occur in the source. You can disable randomization in `tests.edn` 79 | 80 | ## Control output 81 | 82 | Kaocha makes liberal use of ANSI escape codes for colorizing the output. If you 83 | prefer or need plain text output use `--no-color`. 84 | 85 | By default Kaocha will capture any output that occurs on stdout or stderr during 86 | a test run. Only when a test fails is the captured output printed as part of the 87 | test result summary. This is generally what you want, since this way tests that 88 | pass don't generate distracting noise. If you do want all the output as it 89 | occurs, use `--no-capture`. 90 | 91 | ## Debug information 92 | 93 | `--version` prints version information, whereas `--test-help` will print the 94 | available command line flags. Note that while the more common `--help` is also 95 | available, it is not usable under Clojure CLI, instead it will print the help 96 | information for Clojure itself. 97 | 98 | Conceptually Kaocha goes through three steps: load configuration, load tests, 99 | run tests. The result of each step is a data structure. You can view these 100 | structures (EDN) with `--print-config`, `--print-test-plan`, and 101 | `--print-result`. 102 | 103 | ``` clojure 104 | --print-config Print out the fully merged and normalized config, then exit. 105 | --print-test-plan Load tests, build up a test plan, then print out the test plan and exit. 106 | --print-result Print the test result map as returned by the Kaocha API. 107 | ``` 108 | -------------------------------------------------------------------------------- /test/unit/kaocha/result_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.result-test 2 | (:require [clojure.test :as t :refer :all] 3 | [kaocha.result :as result] 4 | [kaocha.testable :as testable] 5 | [kaocha.report :as report] 6 | [kaocha.test-factories :as f])) 7 | 8 | (deftest totals-test 9 | (is (= #:kaocha.result{:count 5, :pass 3, :error 1, :fail 1, :pending 0} 10 | (result/totals 11 | [{:kaocha.testable/id :api 12 | :kaocha.testable/type :kaocha.type/clojure.test 13 | :kaocha.result/tests [{:kaocha.testable/id :ddd.bar-test 14 | :kaocha.testable/type :kaocha.type/ns 15 | :kaocha.testable/desc "ddd.bar-test" 16 | :kaocha.result/tests [{:kaocha.testable/id :ddd.bar-test/test-1 17 | :kaocha.testable/type :kaocha.type/var 18 | :kaocha.testable/desc "test-1" 19 | :kaocha.result/count 1 20 | :kaocha.result/fail 0 21 | :kaocha.result/pass 1 22 | :kaocha.result/error 0} 23 | {:kaocha.testable/id :ddd.bar-test/test-2 24 | :kaocha.testable/type :kaocha.type/var 25 | :kaocha.testable/desc "test-2" 26 | :kaocha.result/count 1 27 | :kaocha.result/fail 0 28 | :kaocha.result/pass 1 29 | :kaocha.result/error 0}]} 30 | {:kaocha.testable/id :ddd.foo-test 31 | :kaocha.testable/type :kaocha.type/ns 32 | :kaocha.testable/desc "ddd.foo-test" 33 | :kaocha.result/tests [{:kaocha.testable/id :ddd.foo-test/test-3 34 | :kaocha.testable/type :kaocha.type/var 35 | :kaocha.testable/desc "test-3" 36 | :kaocha.result/count 1 37 | :kaocha.result/fail 0 38 | :kaocha.result/pass 0 39 | :kaocha.result/error 1} 40 | {:kaocha.testable/id :ddd.foo-test/test-1 41 | :kaocha.testable/type :kaocha.type/var 42 | :kaocha.testable/desc "test-1" 43 | :kaocha.result/count 1 44 | :kaocha.result/fail 1 45 | :kaocha.result/pass 0 46 | :kaocha.result/error 0} 47 | {:kaocha.testable/id :ddd.foo-test/test-2 48 | :kaocha.testable/type :kaocha.type/var 49 | :kaocha.testable/desc "test-2" 50 | :kaocha.result/count 1 51 | :kaocha.result/fail 0 52 | :kaocha.result/pass 1 53 | :kaocha.result/error 0}]}]}])))) 54 | -------------------------------------------------------------------------------- /test/unit/kaocha/type/ns_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.ns-test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :as t :refer :all] 4 | [kaocha.core-ext :refer :all] 5 | [kaocha.testable :as testable] 6 | [kaocha.test-helper] 7 | [kaocha.classpath :as classpath] 8 | [kaocha.test-util :as util :refer [with-test-ctx]] 9 | [kaocha.output :as out])) 10 | 11 | (defn var-name? 12 | "Predicate for the name of a var, for use in matchers." 13 | [v n] 14 | (and (var? v) (= (:name (meta v)) n))) 15 | 16 | (deftest load-test 17 | (classpath/add-classpath "fixtures/a-tests") 18 | 19 | (let [test-plan (testable/load {:kaocha.testable/type :kaocha.type/ns 20 | :kaocha.testable/id :foo.bar-test 21 | :kaocha.testable/desc "foo.bar-test" 22 | :kaocha.ns/name 'foo.bar-test})] 23 | 24 | (is (match? {:kaocha.testable/type :kaocha.type/ns 25 | :kaocha.testable/id :foo.bar-test 26 | :kaocha.testable/desc "foo.bar-test" 27 | :kaocha.ns/name 'foo.bar-test 28 | :kaocha.ns/ns #(= % (the-ns 'foo.bar-test)) 29 | :kaocha.testable/meta nil 30 | :kaocha.test-plan/tests 31 | [{:kaocha.testable/type :kaocha.type/var 32 | :kaocha.testable/id :foo.bar-test/a-test 33 | :kaocha.var/name 'foo.bar-test/a-test 34 | :kaocha.var/var #(= % (resolve 'foo.bar-test/a-test)) 35 | :kaocha.var/test #(true? (%)) 36 | :kaocha.testable/meta #(= % (meta (resolve 'foo.bar-test/a-test)))}]} 37 | test-plan))) 38 | 39 | (util/expect-warning #"Could not locate foo/unknown_test" 40 | (is (match? {:kaocha.testable/type :kaocha.type/ns 41 | :kaocha.testable/id :foo.unknown-test 42 | :kaocha.testable/desc "foo.unknown-test" 43 | :kaocha.ns/name 'foo.unknown-test 44 | :kaocha.test-plan/load-error #(instance? java.io.FileNotFoundException %)} 45 | 46 | (testable/load {:kaocha.testable/type :kaocha.type/ns 47 | :kaocha.testable/id :foo.unknown-test 48 | :kaocha.testable/desc "foo.unknown-test" 49 | :kaocha.ns/name 'foo.unknown-test}))))) 50 | 51 | (deftest run-test 52 | (classpath/add-classpath "fixtures/a-tests") 53 | 54 | (let [testable (testable/load {:kaocha.testable/type :kaocha.type/ns 55 | :kaocha.testable/id :foo.bar-test 56 | :kaocha.testable/desc "foo.bar-test" 57 | :kaocha.ns/name 'foo.bar-test})] 58 | (is (match? {:kaocha.testable/type :kaocha.type/ns 59 | :kaocha.testable/id :foo.bar-test 60 | :kaocha.ns/name 'foo.bar-test 61 | :kaocha.ns/ns ns? 62 | :kaocha.result/tests [{:kaocha.testable/type :kaocha.type/var 63 | :kaocha.testable/id :foo.bar-test/a-test 64 | :kaocha.testable/desc "a-test" 65 | :kaocha.var/name 'foo.bar-test/a-test 66 | :kaocha.var/var var? 67 | :kaocha.var/test fn? 68 | :kaocha.result/count 1 69 | :kaocha.result/pass 1 70 | :kaocha.result/error 0 71 | :kaocha.result/pending 0 72 | :kaocha.result/fail 0}]} 73 | (:result 74 | (with-test-ctx {:fail-fast? true} 75 | (testable/run testable testable))))))) 76 | -------------------------------------------------------------------------------- /test/shared/kaocha/integration_helpers.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.integration-helpers 2 | (:require [clojure.java.io :as io] 3 | [clojure.string :as str]) 4 | (:import java.io.File 5 | [java.nio.file Files OpenOption Path Paths] 6 | [java.nio.file.attribute FileAttribute PosixFilePermissions])) 7 | 8 | (require 'kaocha.assertions) 9 | 10 | (defprotocol Joinable 11 | (join [this that])) 12 | 13 | (extend-protocol Joinable 14 | String 15 | (join [this that] (join (Paths/get this (make-array String 0)) that)) 16 | Path 17 | (join [this that] (.resolve this (str that))) 18 | File 19 | (join [this that] (.toPath (io/file this (str that))))) 20 | 21 | (extend-protocol io/IOFactory 22 | Path 23 | (make-output-stream [this opts] 24 | (Files/newOutputStream this (into-array OpenOption []))) 25 | (make-input-stream [this opts] 26 | (Files/newInputStream this (into-array OpenOption []))) 27 | (make-writer [this opts] 28 | (io/make-writer (io/make-output-stream this opts) opts)) 29 | (make-reader [this opts] 30 | (io/make-reader (io/make-input-stream this opts) opts))) 31 | 32 | (extend-protocol io/Coercions 33 | Path 34 | (as-file [path] (.toFile path)) 35 | (as-url [path] (.toURL (.toFile path)))) 36 | 37 | (def default-attributes (into-array FileAttribute [])) 38 | 39 | (defn temp-dir 40 | ([] 41 | (temp-dir "kaocha_integration")) 42 | ([path] 43 | (Files/createTempDirectory path default-attributes))) 44 | 45 | (defonce clj-cpcache-dir (temp-dir "kaocha_cpcache")) 46 | 47 | (defn mkdir [path] 48 | (Files/createDirectories path default-attributes)) 49 | 50 | (defn codecov? [] 51 | (= (System/getenv "CI") "true")) 52 | 53 | (defn project-dir-path [& paths] 54 | (str (reduce join (.getAbsolutePath (io/file "")) paths))) 55 | 56 | (defmacro with-print-namespace-maps [bool & body] 57 | (if (find-var 'clojure.core/*print-namespace-maps*) 58 | `(binding [*print-namespace-maps* ~bool] 59 | ~@body) 60 | ;; pre Clojure 1.9 61 | `(do ~@body))) 62 | 63 | (defn write-deps-edn [path] 64 | (with-open [deps-out (io/writer path)] 65 | (binding [*out* deps-out] 66 | (with-print-namespace-maps false 67 | (clojure.pprint/pprint {:deps {'lambdaisland/kaocha {:local/root (project-dir-path)} 68 | 'lambdaisland/kaocha-cloverage {:mvn/version "RELEASE"}}}))))) 69 | 70 | (defn test-dir-setup [m] 71 | (if (:dir m) 72 | m 73 | (let [dir (temp-dir) 74 | test-dir (join dir "test") 75 | bin-dir (join dir "bin") 76 | config-file (join dir "tests.edn") 77 | deps-edn (join dir "deps.edn") 78 | runner (join dir "bin/kaocha")] 79 | (mkdir test-dir) 80 | (mkdir bin-dir) 81 | (spit (str config-file) 82 | (str "#kaocha/v1\n" 83 | "{:color? false\n" 84 | " :randomize? false}")) 85 | (spit (str runner) 86 | (str/join " " 87 | (cond-> ["clojure" 88 | "-m" "kaocha.runner"] 89 | (codecov?) 90 | (into ["--plugin" "cloverage" 91 | "--cov-output" (project-dir-path "target/coverage" (str (gensym "integration"))) 92 | "--cov-src-ns-path" (project-dir-path "src") 93 | "--codecov"]) 94 | :always 95 | (conj "\"$@\"")))) 96 | (write-deps-edn deps-edn) 97 | (Files/setPosixFilePermissions runner (PosixFilePermissions/fromString "rwxr--r--")); 98 | (assoc m 99 | :dir dir 100 | :test-dir test-dir 101 | :config-file config-file 102 | :runner runner)))) 103 | 104 | (defn ns->fname [ns] 105 | (-> ns 106 | str 107 | (str/replace #"\." "/") 108 | (str/replace #"-" "_") 109 | (str ".clj"))) 110 | 111 | (defn spit-dir [m path] 112 | (let [{:keys [dir] :as m} (test-dir-setup m) 113 | path (join dir path)] 114 | (mkdir path) 115 | m)) 116 | 117 | (defn spit-file [m path contents] 118 | (let [{:keys [dir] :as m} (test-dir-setup m) 119 | path (join dir path)] 120 | (mkdir (.getParent path)) 121 | (spit path contents) 122 | m)) 123 | -------------------------------------------------------------------------------- /src/kaocha/type/ns.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.ns 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :as t] 4 | [kaocha.core-ext :refer :all] 5 | [kaocha.testable :as testable] 6 | [kaocha.hierarchy :as hierarchy] 7 | [clojure.spec.alpha :as s] 8 | [kaocha.type.var] 9 | [kaocha.output :as output] 10 | [kaocha.type :as type])) 11 | 12 | (defn ->testable [ns-name] 13 | {:kaocha.testable/type :kaocha.type/ns 14 | :kaocha.testable/id (keyword (str ns-name)) 15 | :kaocha.testable/desc (str ns-name) 16 | :kaocha.ns/name ns-name}) 17 | 18 | (defmethod testable/-load :kaocha.type/ns [testable] 19 | ;; TODO If the namespace has a test-ns-hook function, call that: 20 | ;; if-let [v (find-var (symbol (:kaocha.ns/name testable) "test-ns-hook"))] 21 | 22 | (let [ns-name (:kaocha.ns/name testable)] 23 | (try 24 | (when-not (find-ns ns-name) 25 | (require ns-name)) 26 | (let [ns-obj (the-ns ns-name) 27 | ns-meta (meta ns-obj) 28 | each-fixture-fn (t/join-fixtures (::t/each-fixtures ns-meta))] 29 | (->> ns-obj 30 | ns-publics 31 | (filter (comp :test meta val)) 32 | (sort-by key) 33 | (map (fn [[sym var]] 34 | (let [nsname (:kaocha.ns/name testable) 35 | test-name (symbol (str nsname) (str sym))] 36 | {:kaocha.testable/type :kaocha.type/var 37 | :kaocha.testable/id (keyword test-name) 38 | :kaocha.testable/meta (meta var) 39 | :kaocha.testable/desc (str sym) 40 | :kaocha.var/name test-name 41 | :kaocha.var/var var 42 | :kaocha.var/test (:test (meta var)) 43 | :kaocha.testable/wrap (if (::t/each-fixtures ns-meta) 44 | [(fn [t] #(each-fixture-fn t))] 45 | [])}))) 46 | (assoc testable 47 | :kaocha.testable/meta (meta ns-obj) 48 | :kaocha.ns/ns ns-obj 49 | :kaocha.test-plan/tests))) 50 | (catch Throwable t 51 | (output/warn "Failed loading " ns-name ": " (.getMessage t)) 52 | #_(kaocha.stacktrace/print-cause-trace t) 53 | (assoc testable :kaocha.test-plan/load-error t))))) 54 | 55 | (defn run-tests [testable test-plan fixture-fn] 56 | ;; It's not guaranteed the the fixture-fn returns the result of calling the 57 | ;; tests function, so we need to put it in a box for reference. 58 | (let [result (atom (:kaocha.test-plan/tests testable))] 59 | (fixture-fn #(swap! result testable/run-testables test-plan)) 60 | @result)) 61 | 62 | (defmethod testable/-run :kaocha.type/ns [testable test-plan] 63 | (let [do-report #(t/do-report (merge {:ns (:kaocha.ns/ns testable)} %))] 64 | (type/with-report-counters 65 | (do-report {:type :begin-test-ns}) 66 | (if-let [load-error (:kaocha.test-plan/load-error testable)] 67 | (do 68 | (do-report {:type :error 69 | :message "Failed to load namespace." 70 | :expected nil 71 | :actual load-error 72 | :kaocha.result/exception load-error}) 73 | (do-report {:type :end-test-ns}) 74 | (assoc testable :kaocha.result/error 1)) 75 | (let [ns-meta (:kaocha.testable/meta testable) 76 | once-fixture-fn (t/join-fixtures (::t/once-fixtures ns-meta)) 77 | tests (run-tests testable test-plan once-fixture-fn) 78 | result (assoc (dissoc testable :kaocha.test-plan/tests) 79 | :kaocha.result/tests 80 | tests)] 81 | (do-report {:type :end-test-ns}) 82 | result))))) 83 | 84 | (s/def :kaocha.type/ns (s/keys :req [:kaocha.testable/type 85 | :kaocha.testable/id 86 | :kaocha.ns/name] 87 | :opt [:kaocha.ns/ns 88 | :kaocha.test-plan/tests])) 89 | 90 | (s/def :kaocha.ns/name simple-symbol?) 91 | (s/def :kaocha.ns/ns ns?) 92 | 93 | (hierarchy/derive! :kaocha.type/ns :kaocha.testable.type/group) 94 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | lambdaisland 5 | kaocha 6 | 0.0-389 7 | kaocha 8 | Full featured next generation test runner for Clojure. 9 | https://github.com/lambdaisland/kaocha 10 | 2018 11 | 12 | Lambda Island 13 | https://lambdaisland.com 14 | 15 | 16 | 17 | Eclipse Public License 1.0 18 | https://www.eclipse.org/legal/epl-v10.html 19 | 20 | 21 | 22 | https://github.com/lambdaisland/kaocha 23 | scm:git:git://github.com/lambdaisland/kaocha.git 24 | scm:git:ssh://git@github.com/lambdaisland/kaocha.git 25 | ca1af114b1ae49e47a25b9922c7aaf9154ee19eb 26 | 27 | 28 | 29 | org.clojure 30 | clojure 31 | 1.10.0 32 | 33 | 34 | expound 35 | expound 36 | 0.7.2 37 | 38 | 39 | org.clojure 40 | spec.alpha 41 | 0.2.176 42 | 43 | 44 | org.clojure 45 | tools.cli 46 | 0.4.1 47 | 48 | 49 | lambdaisland 50 | deep-diff 51 | 0.0-25 52 | 53 | 54 | slingshot 55 | slingshot 56 | 0.12.2 57 | 58 | 59 | aero 60 | aero 61 | 1.1.3 62 | 63 | 64 | lambdaisland 65 | tools.namespace 66 | 0.0-234 67 | 68 | 69 | progrock 70 | progrock 71 | 0.1.2 72 | 73 | 74 | orchestra 75 | orchestra 76 | 2018.12.06-2 77 | 78 | 79 | org.tcrawley 80 | dynapath 81 | 1.0.0 82 | 83 | 84 | hawk 85 | hawk 86 | 0.2.11 87 | 88 | 89 | meta-merge 90 | meta-merge 91 | 1.0.0 92 | 93 | 94 | 95 | src 96 | 97 | 98 | src 99 | 100 | 101 | resources 102 | 103 | 104 | 105 | 106 | org.apache.maven.plugins 107 | maven-jar-plugin 108 | 2.4 109 | 110 | 111 | 112 | ca1af114b1ae49e47a25b9922c7aaf9154ee19eb 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | clojars 122 | https://repo.clojars.org/ 123 | 124 | 125 | 126 | 127 | clojars 128 | Clojars repository 129 | https://clojars.org/repo 130 | 131 | 132 | 133 | -------------------------------------------------------------------------------- /src/kaocha/plugin/notifier.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.notifier 2 | {:authors ["Ryan McCuaig (@rgm)" 3 | "Arne Brasseur (@plexus)"]} 4 | (:require [clojure.java.shell :refer [sh]] 5 | [kaocha.plugin :refer [defplugin]] 6 | [kaocha.result :as result] 7 | [kaocha.shellwords :refer [shellwords]] 8 | [clojure.string :as str] 9 | [clojure.java.io :as io]) 10 | (:import [java.nio.file Files])) 11 | 12 | ;; special thanks for terminal-notify stuff to 13 | ;; https://github.com/glittershark/midje-notifier/blob/master/src/midje/notifier.clj 14 | 15 | (defn exists? [program] 16 | (= 0 (:exit (sh "which" program)))) 17 | 18 | (defn detect-command [] 19 | (cond 20 | (exists? "notify-send") 21 | "notify-send -a Kaocha %{title} %{message} -i %{icon} -u %{urgency}" 22 | 23 | (exists? "terminal-notifier") 24 | "terminal-notifier -message %{message} -title %{title} -appIcon %{icon}")) 25 | 26 | (defn message [result] 27 | (let [{::result/keys [count pass fail error pending]} (result/testable-totals result)] 28 | (str count " tests" 29 | (when (pos-int? error) 30 | (str ", " error " errors")) 31 | (when (pos-int? pending) 32 | (str ", "pending " pending")) 33 | (when (pos-int? fail) 34 | (str ", " fail " failures")) 35 | "."))) 36 | 37 | (defn title [result] 38 | (if (result/failed? result) 39 | "⛔️ Failing" 40 | "✅ Passing")) 41 | 42 | (def icon-path 43 | "Return a local path to the Clojure icon. 44 | 45 | If Kaocha is running from a jar, then extract the icon to a temp file first, 46 | so external programs have access to it. " 47 | (memoize 48 | (fn [] 49 | (let [resource (io/resource "kaocha/clojure_logo.png")] 50 | (if (= "file" (.getProtocol resource)) 51 | (str (io/file resource)) 52 | (let [file (java.io.File/createTempFile "clojure_logo" ".png")] 53 | (io/copy (io/make-input-stream resource {}) file) 54 | (str file))))))) 55 | 56 | (defn expand-command 57 | "Takes a command string including replacement patterns, and a map of 58 | replacements, and returns a vector of command + arguments. 59 | 60 | Replacement patterns are of the form `%{xxx}`" 61 | [command replacements] 62 | (map (fn [cmd] 63 | (reduce 64 | (fn [cmd [k v]] 65 | (str/replace cmd (str "%{" (name k) "}") (str v))) 66 | cmd 67 | replacements)) 68 | (shellwords command))) 69 | 70 | (defn run-command 71 | "Run the given command string, replacing patterns with values based on the given 72 | test result." 73 | [command result] 74 | (let [{::result/keys [count pass fail error pending]} (result/testable-totals result) 75 | message (message result) 76 | title (title result) 77 | icon (icon-path) 78 | failed? (result/failed? result) 79 | urgency (if failed? "critical" "normal")] 80 | (apply sh (expand-command command {:message message 81 | :title title 82 | :icon icon 83 | :urgency urgency 84 | :count count 85 | :pass pass 86 | :fail fail 87 | :error error 88 | :pending pending 89 | :failed? failed?})))) 90 | 91 | (defplugin kaocha.plugin/notifier 92 | "Run a shell command after each completed test run, by default will run a 93 | command that displays a desktop notification showing the test results, so a 94 | `kaocha --watch` terminal process can be hidden and generally ignored until it 95 | goes red. 96 | 97 | Requires https://github.com/julienXX/terminal-notifier on mac or `libnotify` / 98 | `notify-send` on linux." 99 | (cli-options [opts] 100 | (conj opts [nil "--[no-]notifications" "Enable/disable the notifier plugin, providing desktop notifications. Defaults to true."])) 101 | 102 | (config [config] 103 | (let [cli-flag (get-in config [:kaocha/cli-options :notifications])] 104 | (assoc config 105 | ::command (::command config (detect-command)) 106 | ::notifications? 107 | (if (some? cli-flag) 108 | cli-flag 109 | (::notifications? config true))))) 110 | 111 | (post-run [result] 112 | (if-let [command (and (::notifications? result) (::command result))] 113 | (run-command command result)) 114 | result)) 115 | -------------------------------------------------------------------------------- /doc/filtering/skip_meta.md: -------------------------------------------------------------------------------- 1 | # Skipping based on metadata 2 | 3 | You can limit the test run based on test's metadata. How to associate metadata 4 | with a test depends on the test type, for `clojure.test` type tests metadata 5 | can be associated with a test var or test namespace. 6 | 7 | Using the `--skip-meta` command line flag, or `:kaocha.filter/skip-meta` key 8 | in test suite configuration, you can tell Kaocha to completely ignore those 9 | tests where the given metadata key has a truthy value associated with it. 10 | 11 | The default value for `:kaocha.filter/skip-meta` is `[:kaocha/skip]`, so you 12 | can use `^:kaocha/skip` to ignore tests without extra configuration. 13 | 14 | ## Skipping a test based on metadata from the command line 15 | 16 | - Given a file named "test/my/project/sample_test.clj" with: 17 | 18 | ``` clojure 19 | (ns my.project.sample-test 20 | (:require [clojure.test :refer :all])) 21 | 22 | (deftest some-test 23 | (is (= 1 1))) 24 | 25 | (deftest ^:xxx other-test 26 | (is (= 2 2))) 27 | ``` 28 | 29 | 30 | - When I run `bin/kaocha --skip-meta :xxx --reporter documentation` 31 | 32 | - Then the output should contain: 33 | 34 | ``` nil 35 | --- unit (clojure.test) --------------------------- 36 | my.project.sample-test 37 | some-test 38 | 39 | 1 tests, 1 assertions, 0 failures. 40 | ``` 41 | 42 | 43 | 44 | ## Skipping a test group based on metadata from the command line 45 | 46 | - Given a file named "test/my/project/sample_test.clj" with: 47 | 48 | ``` clojure 49 | (ns my.project.sample-test 50 | (:require [clojure.test :refer :all])) 51 | 52 | (deftest some-test 53 | (is (= 1 1))) 54 | ``` 55 | 56 | 57 | - And a file named "test/my/project/other_sample_test.clj" with: 58 | 59 | ``` clojure 60 | (ns ^:xxx my.project.other-sample-test 61 | (:require [clojure.test :refer :all])) 62 | 63 | (deftest other-test 64 | (is (= 1 1))) 65 | ``` 66 | 67 | 68 | - When I run `bin/kaocha --skip-meta :xxx --reporter documentation` 69 | 70 | - Then the output should contain: 71 | 72 | ``` nil 73 | --- unit (clojure.test) --------------------------- 74 | my.project.sample-test 75 | some-test 76 | 77 | 1 tests, 1 assertions, 0 failures. 78 | ``` 79 | 80 | 81 | 82 | ## Skipping based on metadata via configuration 83 | 84 | - Given a file named "tests.edn" with: 85 | 86 | ``` edn 87 | #kaocha/v1 88 | {:tests [{:kaocha.filter/skip-meta [:xxx]}] 89 | :color? false 90 | :randomize? false} 91 | ``` 92 | 93 | 94 | - And a file named "test/my/project/sample_test.clj" with: 95 | 96 | ``` clojure 97 | (ns my.project.sample-test 98 | (:require [clojure.test :refer :all])) 99 | 100 | (deftest some-test 101 | (is (= 1 1))) 102 | 103 | (deftest ^:xxx other-test 104 | (is (= 2 2))) 105 | 106 | (deftest ^:kaocha/skip also-skipped 107 | (is (= 3 3))) 108 | ``` 109 | 110 | 111 | - When I run `bin/kaocha --reporter documentation` 112 | 113 | - Then the output should contain: 114 | 115 | ``` nil 116 | --- unit (clojure.test) --------------------------- 117 | my.project.sample-test 118 | some-test 119 | 120 | 1 tests, 1 assertions, 0 failures. 121 | ``` 122 | 123 | 124 | 125 | ## Skipping using the default `:kaocha/skip` metadata 126 | 127 | - Given a file named "test/my/project/sample_test.clj" with: 128 | 129 | ``` clojure 130 | (ns my.project.sample-test 131 | (:require [clojure.test :refer :all])) 132 | 133 | (deftest some-test 134 | (is (= 1 1))) 135 | 136 | (deftest ^:kaocha/skip other-test 137 | (is (= 2 2))) 138 | ``` 139 | 140 | 141 | - When I run `bin/kaocha --reporter documentation` 142 | 143 | - Then the output should contain: 144 | 145 | ``` nil 146 | --- unit (clojure.test) --------------------------- 147 | my.project.sample-test 148 | some-test 149 | 150 | 1 tests, 1 assertions, 0 failures. 151 | ``` 152 | 153 | 154 | 155 | ## Replacing skip metadata 156 | 157 | - Given a file named "tests.edn" with: 158 | 159 | ``` edn 160 | #kaocha/v1 161 | {:tests [{:kaocha.filter/skip-meta ^:replace [:xxx]}] 162 | :color? false 163 | :randomize? false} 164 | ``` 165 | 166 | 167 | - Given a file named "test/my/project/sample_test.clj" with: 168 | 169 | ``` clojure 170 | (ns my.project.sample-test 171 | (:require [clojure.test :refer :all])) 172 | 173 | (deftest ^:xxx some-test 174 | (is (= 1 1))) 175 | 176 | (deftest ^:kaocha/skip other-test ;; this is ignored now 177 | (is (= 2 2))) 178 | ``` 179 | 180 | 181 | - When I run `bin/kaocha --reporter documentation` 182 | 183 | - Then the output should contain: 184 | 185 | ``` nil 186 | --- unit (clojure.test) --------------------------- 187 | my.project.sample-test 188 | other-test 189 | 190 | 1 tests, 1 assertions, 0 failures. 191 | ``` 192 | 193 | 194 | 195 | -------------------------------------------------------------------------------- /test/features/filtering/skip_meta.feature: -------------------------------------------------------------------------------- 1 | Feature: Skipping based on metadata 2 | 3 | You can limit the test run based on test's metadata. How to associate metadata 4 | with a test depends on the test type, for `clojure.test` type tests metadata 5 | can be associated with a test var or test namespace. 6 | 7 | Using the `--skip-meta` command line flag, or `:kaocha.filter/skip-meta` key 8 | in test suite configuration, you can tell Kaocha to completely ignore those 9 | tests where the given metadata key has a truthy value associated with it. 10 | 11 | The default value for `:kaocha.filter/skip-meta` is `[:kaocha/skip]`, so you 12 | can use `^:kaocha/skip` to ignore tests without extra configuration. 13 | 14 | Scenario: Skipping a test based on metadata from the command line 15 | Given a file named "test/my/project/sample_test.clj" with: 16 | """clojure 17 | (ns my.project.sample-test 18 | (:require [clojure.test :refer :all])) 19 | 20 | (deftest some-test 21 | (is (= 1 1))) 22 | 23 | (deftest ^:xxx other-test 24 | (is (= 2 2))) 25 | """ 26 | When I run `bin/kaocha --skip-meta :xxx --reporter documentation` 27 | Then the output should contain: 28 | """ 29 | --- unit (clojure.test) --------------------------- 30 | my.project.sample-test 31 | some-test 32 | 33 | 1 tests, 1 assertions, 0 failures. 34 | """ 35 | 36 | Scenario: Skipping a test group based on metadata from the command line 37 | Given a file named "test/my/project/sample_test.clj" with: 38 | """clojure 39 | (ns my.project.sample-test 40 | (:require [clojure.test :refer :all])) 41 | 42 | (deftest some-test 43 | (is (= 1 1))) 44 | """ 45 | And a file named "test/my/project/other_sample_test.clj" with: 46 | """clojure 47 | (ns ^:xxx my.project.other-sample-test 48 | (:require [clojure.test :refer :all])) 49 | 50 | (deftest other-test 51 | (is (= 1 1))) 52 | """ 53 | When I run `bin/kaocha --skip-meta :xxx --reporter documentation` 54 | Then the output should contain: 55 | """ 56 | --- unit (clojure.test) --------------------------- 57 | my.project.sample-test 58 | some-test 59 | 60 | 1 tests, 1 assertions, 0 failures. 61 | """ 62 | 63 | Scenario: Skipping based on metadata via configuration 64 | Given a file named "tests.edn" with: 65 | """ edn 66 | #kaocha/v1 67 | {:tests [{:kaocha.filter/skip-meta [:xxx]}] 68 | :color? false 69 | :randomize? false} 70 | """ 71 | And a file named "test/my/project/sample_test.clj" with: 72 | """clojure 73 | (ns my.project.sample-test 74 | (:require [clojure.test :refer :all])) 75 | 76 | (deftest some-test 77 | (is (= 1 1))) 78 | 79 | (deftest ^:xxx other-test 80 | (is (= 2 2))) 81 | 82 | (deftest ^:kaocha/skip also-skipped 83 | (is (= 3 3))) 84 | """ 85 | When I run `bin/kaocha --reporter documentation` 86 | Then the output should contain: 87 | """ 88 | --- unit (clojure.test) --------------------------- 89 | my.project.sample-test 90 | some-test 91 | 92 | 1 tests, 1 assertions, 0 failures. 93 | """ 94 | 95 | Scenario: Skipping using the default `:kaocha/skip` metadata 96 | Given a file named "test/my/project/sample_test.clj" with: 97 | """clojure 98 | (ns my.project.sample-test 99 | (:require [clojure.test :refer :all])) 100 | 101 | (deftest some-test 102 | (is (= 1 1))) 103 | 104 | (deftest ^:kaocha/skip other-test 105 | (is (= 2 2))) 106 | """ 107 | When I run `bin/kaocha --reporter documentation` 108 | Then the output should contain: 109 | """ 110 | --- unit (clojure.test) --------------------------- 111 | my.project.sample-test 112 | some-test 113 | 114 | 1 tests, 1 assertions, 0 failures. 115 | """ 116 | 117 | Scenario: Replacing skip metadata 118 | Given a file named "tests.edn" with: 119 | """ edn 120 | #kaocha/v1 121 | {:tests [{:kaocha.filter/skip-meta ^:replace [:xxx]}] 122 | :color? false 123 | :randomize? false} 124 | """ 125 | Given a file named "test/my/project/sample_test.clj" with: 126 | """clojure 127 | (ns my.project.sample-test 128 | (:require [clojure.test :refer :all])) 129 | 130 | (deftest ^:xxx some-test 131 | (is (= 1 1))) 132 | 133 | (deftest ^:kaocha/skip other-test ;; this is ignored now 134 | (is (= 2 2))) 135 | """ 136 | When I run `bin/kaocha --reporter documentation` 137 | Then the output should contain: 138 | """ 139 | --- unit (clojure.test) --------------------------- 140 | my.project.sample-test 141 | other-test 142 | 143 | 1 tests, 1 assertions, 0 failures. 144 | """ 145 | --------------------------------------------------------------------------------