├── .VERSION_PREFIX ├── .circleci └── config.yml ├── .dir-locals.el ├── .github └── workflows │ ├── add_to_project_board.yml │ ├── bb.yml │ └── canary.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── bb-tests.edn ├── bb.edn ├── bin ├── .VERSION_PREFIX ├── generate_toc ├── kaocha ├── proj ├── update_project_list.ed └── update_toc.ed ├── code_of_conduct.txt ├── deps.edn ├── dev └── user.clj ├── doc ├── 01_introduction.md ├── 02_installing.md ├── 03_configuration.md ├── 04_running_kaocha_cli.md ├── 05_running_kaocha_repl.md ├── 06_focusing_and_skipping.md ├── 07_watch_mode.md ├── 08_plugins.md ├── 09_extending.md ├── 10_hooks.md ├── cljdoc.edn ├── clojure_test │ └── assertions.md ├── command_line │ ├── capability_check.md │ ├── fail_fast.md │ ├── print_config.md │ ├── profile.md │ ├── reporter.md │ └── suite_names.md ├── config │ ├── bindings.md │ └── warnings.md ├── extensions.png ├── filtering │ ├── focus_meta.md │ ├── focusing.md │ ├── skip_meta.md │ └── skipping.md ├── images │ └── deep-diff.png ├── pending.md ├── plugins │ ├── capture_output.md │ ├── hooks_plugin.md │ ├── notifier_plugin.md │ ├── orchestra_plugin.md │ └── version_filter.md ├── spec_test_check.md └── syntax_error.md ├── examples └── blame_report.clj ├── fixtures ├── a-tests │ ├── baz │ │ └── qux_test.clj │ └── foo │ │ └── bar_test.clj ├── b-tests │ └── finn │ │ └── finn_test.clj ├── c-tests │ └── foo │ │ └── hello_test.clj ├── custom_config.edn ├── d-tests │ └── ddd │ │ ├── bar_test.clj │ │ ├── double_once_fixture_test.clj │ │ ├── exception_outside_is.clj │ │ └── foo_test.clj ├── e-tests │ └── eee │ │ └── bad_code_test.clj ├── tests.edn ├── with_compile_error.edn ├── with_exception.edn └── with_failing.edn ├── kaocha.png ├── notes.org ├── pom.xml ├── repl_sessions └── strict_focus_meta_hook.clj ├── resources └── kaocha │ ├── clojure_logo.png │ └── default_config.edn ├── src └── kaocha │ ├── api.clj │ ├── assertions.clj │ ├── classpath.bb │ ├── classpath.clj │ ├── config.clj │ ├── core_ext.clj │ ├── hierarchy.clj │ ├── history.clj │ ├── jit.clj │ ├── load.clj │ ├── matcher_combinators.clj │ ├── monkey_patch.clj │ ├── ns.clj │ ├── output.clj │ ├── platform.clj │ ├── platform │ ├── systray.bb │ └── systray.clj │ ├── plugin.clj │ ├── plugin │ ├── alpha │ │ ├── info.clj │ │ ├── spec_test_check.clj │ │ └── xfail.clj │ ├── capture_output.cljc │ ├── debug.clj │ ├── filter.clj │ ├── gc_profiling.clj │ ├── hooks.clj │ ├── notifier.clj │ ├── orchestra.clj │ ├── preloads.clj │ ├── print_invocations.clj │ ├── profiling.clj │ ├── randomize.clj │ └── version_filter.clj │ ├── random.clj │ ├── repl.clj │ ├── report.clj │ ├── report │ └── progress.clj │ ├── result.clj │ ├── runner.clj │ ├── shellwords.clj │ ├── specs.clj │ ├── stacktrace.clj │ ├── test_suite.clj │ ├── testable.clj │ ├── type.clj │ ├── type │ ├── clojure │ │ └── test.clj │ ├── ns.clj │ ├── spec │ │ └── test │ │ │ ├── check.clj │ │ │ ├── fdef.clj │ │ │ └── ns.clj │ └── var.clj │ ├── util.clj │ ├── version_check.clj │ └── watch.clj ├── test ├── bb │ └── kaocha │ │ ├── bb_test.clj │ │ └── canary.clj ├── features │ ├── clojure_test │ │ └── assertions.feature │ ├── command_line │ │ ├── capability_check.feature │ │ ├── fail_fast.feature │ │ ├── print_config.feature │ │ ├── profile.feature │ │ ├── reporter.feature │ │ └── suite_names.feature │ ├── config │ │ ├── bindings.feature │ │ └── warnings.feature │ ├── filtering │ │ ├── focus_meta.feature │ │ ├── focusing.feature │ │ ├── skip_meta.feature │ │ └── skipping.feature │ ├── pending.feature │ ├── plugins │ │ ├── capture_output.feature │ │ ├── hooks_plugin.feature │ │ ├── notifier_plugin.feature │ │ ├── orchestra_plugin.feature │ │ └── version_filter.feature │ ├── spec_test_check.feature │ └── syntax_error.feature ├── shared │ └── kaocha │ │ ├── integration_helpers.clj │ │ ├── test_factories.clj │ │ ├── test_helper.clj │ │ ├── test_plugins.clj │ │ └── test_util.clj ├── step_definitions │ └── kaocha_integration.clj └── unit │ └── kaocha │ ├── api_test.clj │ ├── config │ ├── included-test.edn │ ├── loaded-test-profile.edn │ ├── loaded-test-resource.edn │ ├── loaded-test-spec-mismatch.edn │ └── loaded-test.edn │ ├── config_test.clj │ ├── core_ext_test.clj │ ├── fixtures_test.clj │ ├── hierarchy_test.clj │ ├── history_test.clj │ ├── monkey_patch_test.clj │ ├── output_test.clj │ ├── plugin │ ├── alpha │ │ └── spec_test_check_test.clj │ ├── capture_output_test.clj │ ├── filter_test.clj │ ├── gc_profiling_test.clj │ ├── hooks_test.clj │ ├── notifier_test.clj │ ├── randomize_test.clj │ └── version_filter_test.clj │ ├── plugin_test.clj │ ├── post_assertion_test.clj │ ├── private_test.clj │ ├── random_test.clj │ ├── repl_test.clj │ ├── report_test.clj │ ├── result_test.clj │ ├── runner_test.clj │ ├── testable_test.clj │ ├── type │ ├── clojure │ │ └── test_test.clj │ ├── ns_test.clj │ └── var_test.clj │ ├── version_check_test.clj │ └── watch_test.clj └── tests.edn /.VERSION_PREFIX: -------------------------------------------------------------------------------- 1 | 1.91 -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | kaocha: lambdaisland/kaocha@0.0.3 5 | clojure: lambdaisland/clojure@0.0.8 6 | win: circleci/windows@2.2.0 7 | 8 | jobs: 9 | test: 10 | parameters: 11 | os: 12 | type: executor 13 | clojure_version: 14 | type: string 15 | executor: << parameters.os >> 16 | steps: 17 | - checkout 18 | - clojure/with_cache: 19 | cache_version: << parameters.clojure_version >> 20 | steps: 21 | - run: clojure -e '(println (System/getProperty "java.runtime.name") (System/getProperty "java.runtime.version") "\nClojure" (clojure-version))' 22 | - kaocha/execute: 23 | args: "unit --reporter documentation --plugin cloverage --codecov" 24 | clojure_version: << parameters.clojure_version >> 25 | - kaocha/upload_codecov: 26 | flags: unit 27 | - kaocha/execute: 28 | args: "integration --reporter documentation --plugin cloverage --codecov" 29 | clojure_version: << parameters.clojure_version >> 30 | - kaocha/upload_codecov: 31 | flags: integration 32 | file: target/coverage/integration*/codecov.json 33 | windows-test: 34 | executor: 35 | name: win/default 36 | steps: 37 | - checkout 38 | - run: 39 | command: $(echo hello | Out-Host; $?) -and $(echo world | Out-Host; $?) 40 | shell: powershell.exe 41 | - run: 42 | command: Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://download.clojure.org/install/win-install-1.10.3.839.ps1') 43 | shell: powershell.exe 44 | - run: 45 | command: clojure -e "(println (System/getProperty \`"java.runtime.name\`") (System/getProperty \`"java.runtime.version\`") \`"`nClojure\`" (clojure-version))" 46 | shell: powershell.exe 47 | - run: 48 | command: | 49 | $ErrorActionPreference = "Stop" 50 | clojure "-J-Dline.separator=`n" -A:dev:test -m kaocha.runner unit 2>&1 51 | exit $lastexitcode 52 | shell: powershell.exe 53 | 54 | workflows: 55 | kaocha_test: 56 | jobs: 57 | - test: 58 | matrix: 59 | parameters: 60 | os: [clojure/openjdk17, clojure/openjdk16, clojure/openjdk15, clojure/openjdk11, clojure/openjdk8] 61 | clojure_version: ["1.9.0", "1.10.3", "1.11.1"] 62 | 63 | # - windows-test 64 | 65 | 66 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((cider-clojure-cli-aliases . ":dev:test")))) 2 | -------------------------------------------------------------------------------- /.github/workflows/add_to_project_board.yml: -------------------------------------------------------------------------------- 1 | name: Add new pr or issue to project board 2 | 3 | on: 4 | issues: {types: [opened]} 5 | pull_request: {types: [opened]} 6 | 7 | jobs: 8 | add-to-project: 9 | uses: lambdaisland/open-source/.github/workflows/add-to-project-board.yml@main 10 | secrets: inherit 11 | 12 | -------------------------------------------------------------------------------- /.github/workflows/bb.yml: -------------------------------------------------------------------------------- 1 | name: bb 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | clojure: 8 | 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | 13 | runs-on: ${{ matrix.os }} 14 | 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | 19 | # It is important to install java before installing clojure tools which needs java 20 | # exclusions: babashka, clj-kondo and cljstyle 21 | - name: Prepare java 22 | uses: actions/setup-java@v3 23 | with: 24 | distribution: 'zulu' 25 | java-version: '11' 26 | 27 | - name: Install clojure tools 28 | uses: DeLaGuardo/setup-clojure@10.1 29 | with: 30 | # Install just one or all simultaneously 31 | # The value must indicate a particular version of the tool, or use 'latest' 32 | # to always provision the latest version 33 | bb: latest 34 | 35 | # Optional step: 36 | - name: Cache clojure dependencies 37 | uses: actions/cache@v3 38 | with: 39 | path: | 40 | ~/.m2/repository 41 | ~/.gitlibs 42 | ~/.deps.clj 43 | # List all files containing dependencies: 44 | key: cljdeps-${{ hashFiles('deps.edn') }} 45 | # key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }} 46 | # key: cljdeps-${{ hashFiles('project.clj') }} 47 | # key: cljdeps-${{ hashFiles('build.boot') }} 48 | restore-keys: cljdeps- 49 | 50 | - name: Execute bb tests 51 | run: | 52 | bb test:bb 53 | -------------------------------------------------------------------------------- /.github/workflows/canary.yml: -------------------------------------------------------------------------------- 1 | name: canary 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | 7 | clojure: 8 | 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | 13 | runs-on: ${{ matrix.os }} 14 | 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | 19 | # It is important to install java before installing clojure tools which needs java 20 | # exclusions: babashka, clj-kondo and cljstyle 21 | - name: Prepare java 22 | uses: actions/setup-java@v3 23 | with: 24 | distribution: 'zulu' 25 | java-version: '11' 26 | 27 | - name: Install clojure tools 28 | uses: DeLaGuardo/setup-clojure@10.1 29 | with: 30 | # Install just one or all simultaneously 31 | # The value must indicate a particular version of the tool, or use 'latest' 32 | # to always provision the latest version 33 | cli: latest 34 | bb: latest 35 | 36 | # Optional step: 37 | - name: Cache clojure dependencies 38 | uses: actions/cache@v3 39 | with: 40 | path: | 41 | ~/.m2/repository 42 | ~/.gitlibs 43 | ~/.deps.clj 44 | # List all files containing dependencies: 45 | key: cljdeps-${{ hashFiles('deps.edn') }} 46 | # key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }} 47 | # key: cljdeps-${{ hashFiles('project.clj') }} 48 | # key: cljdeps-${{ hashFiles('build.boot') }} 49 | restore-keys: cljdeps- 50 | - name: Run tests 51 | run: | 52 | bb test/bb/kaocha/canary.clj 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | .nrepl-port 3 | target 4 | node_modules 5 | package-lock.json 6 | .store 7 | deps.local.edn 8 | /.clj-kondo/ 9 | /.lsp/ 10 | -------------------------------------------------------------------------------- /bb-tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:plugins [:kaocha.plugin.alpha/info 3 | :profiling 4 | :print-invocations 5 | :hooks 6 | :notifier 7 | :kaocha.plugin/version-filter] 8 | 9 | :tests [{:id :bb 10 | :test-paths ["test/bb"]}] 11 | 12 | :kaocha.hooks/pre-load [kaocha.assertions/load-assertions] 13 | 14 | :kaocha/bindings {kaocha.stacktrace/*stacktrace-filters* []} 15 | 16 | :reporter kaocha.report/documentation} 17 | -------------------------------------------------------------------------------- /bb.edn: -------------------------------------------------------------------------------- 1 | {:deps 2 | {lambdaisland/kaocha {:local/root "."} 3 | lambdaisland/open-source {:git/url "https://github.com/lambdaisland/open-source" 4 | :git/sha "b91bbd276360bb0a865d85b48e048b831a35bc3f" 5 | #_#_:local/root "../open-source"}} 6 | :tasks 7 | {test:bb 8 | {:extra-deps {nubank/matcher-combinators {:mvn/version "3.8.5"}} 9 | :extra-paths ["test/bb"] 10 | :requires ([kaocha.runner]) 11 | :task (apply kaocha.runner/-main "bb" "--config-file" "bb-tests.edn" *command-line-args*)}}} 12 | -------------------------------------------------------------------------------- /bin/.VERSION_PREFIX: -------------------------------------------------------------------------------- 1 | 1.0 -------------------------------------------------------------------------------- /bin/generate_toc: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | def uslug(s) 4 | s.downcase.gsub(/[^\p{L}\p{N}]+/, ' ').gsub(/[\p{Z}\s]+/, '-') 5 | end 6 | 7 | Dir['doc/**/*.md'].sort.each do |f| 8 | title = File.read(f).each_line.select {|l| l =~ /^#/}.first.sub(/^#+/, '').strip 9 | puts "- [%s](https://cljdoc.org/d/lambdaisland/kaocha/CURRENT/doc/%s)" % [title, uslug(title)] 10 | end 11 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | clojure -J-Dline.separator=$'\n' -A:dev:test -M -m kaocha.runner "$@" 4 | -------------------------------------------------------------------------------- /bin/proj: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bb 2 | 3 | (ns proj (:require [lioss.main :as lioss] 4 | [lioss.subshell :as subshell] 5 | [clojure.java.io :as io] 6 | [clojure.string :as str] 7 | [lioss.version :as version])) 8 | 9 | (defn update-docs [opts] 10 | (subshell/spawn "clojure" "-M:feature-docs" "-m" "lioss.cucumber" ".") 11 | (doseq [file (filter #(str/ends-with? % ".md") (file-seq (io/file "doc")))] 12 | (version/update-versions-in file (:module-versions opts))) 13 | (subshell/spawn "sh" "-c" "ed < bin/update_toc.ed") 14 | opts) 15 | 16 | ;; TODO: run the cucumber to markdown code from here 17 | 18 | (lioss/main 19 | {:license :epl 20 | :inception-year 2018 21 | :description "Full featured next generation test runner for Clojure." 22 | :pre-release-hook update-docs 23 | :group-id "lambdaisland" 24 | :commands ["update-docs" 25 | {:description "Rebuild cucumber docs, update the ToC and version strings in md files" 26 | :command update-docs}] 27 | :aliases-as-optional-deps [:test]}) 28 | 29 | 30 | ;; Local Variables: 31 | ;; mode:clojure 32 | ;; End: 33 | -------------------------------------------------------------------------------- /bin/update_project_list.ed: -------------------------------------------------------------------------------- 1 | /-- projects --/+,/-- \/projects --/- d 2 | i 3 | | Project | CI | Docs | Release | Coverage | 4 | |---------|----|------|---------|----------| 5 | . 6 | .r ! for X in kaocha kaocha-cljs kaocha-cucumber kaocha-junit-xml kaocha-cloverage kaocha-boot deep-diff; do echo "| [${X}](https://github.com/lambdaisland/${X}) | [![CircleCI](https://circleci.com/gh/lambdaisland/${X}.svg?style=svg)](https://circleci.com/gh/lambdaisland/${X}) | [![cljdoc badge](https://cljdoc.org/badge/lambdaisland/${X})](https://cljdoc.org/d/lambdaisland/${X}) | [![Clojars Project](https://img.shields.io/clojars/v/lambdaisland/${X}.svg)](https://clojars.org/lambdaisland/${X}) | [![codecov](https://codecov.io/gh/lambdaisland/${X}/branch/master/graph/badge.svg)](https://codecov.io/gh/lambdaisland/${X}) |" ; done 7 | wq 8 | -------------------------------------------------------------------------------- /bin/update_toc.ed: -------------------------------------------------------------------------------- 1 | r README.md 2 | 1 3 | / docs-toc/+,/\/docs-toc/- d 4 | - 5 | .r ! bin/generate_toc 6 | w 7 | q -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "resources"] 2 | 3 | :deps 4 | {org.clojure/tools.cli {:mvn/version "1.1.230"} 5 | lambdaisland/tools.namespace {:mvn/version "0.3.256"} 6 | lambdaisland/deep-diff2 {:mvn/version "2.11.216"} 7 | org.tcrawley/dynapath {:mvn/version "1.1.0"} 8 | slingshot/slingshot {:mvn/version "0.12.2"} 9 | hawk/hawk {:mvn/version "0.2.11"} 10 | com.nextjournal/beholder {:mvn/version "1.0.2"} 11 | expound/expound {:mvn/version "0.9.0"} 12 | aero/aero {:mvn/version "1.1.6"} 13 | progrock/progrock {:mvn/version "0.1.2"} 14 | meta-merge/meta-merge {:mvn/version "1.0.0"}} 15 | 16 | :aliases 17 | {:test 18 | {:extra-paths ["test/shared" "test/unit"] 19 | :extra-deps {org.clojure/test.check {:mvn/version "1.1.1"} 20 | lambdaisland/kaocha-cucumber {:mvn/version "0.11.100" :exclusions [lambdaisland/kaocha]} 21 | lambdaisland/kaocha-cloverage {:mvn/version "1.1.89" :exclusions [lambdaisland/kaocha]} 22 | nubank/matcher-combinators {:mvn/version "1.5.2"} 23 | akvo/fs {:mvn/version "20180904-152732.6dad3934"} 24 | orchestra/orchestra {:mvn/version "2021.01.01-1"}}} 25 | 26 | :feature-docs 27 | {:extra-deps 28 | {lambdaisland/kaocha-cucumber {:mvn/version "0.11.100" :exclusions [lambdaisland/kaocha]} 29 | lambdaisland/open-source {:git/url "https://github.com/lambdaisland/open-source" 30 | :git/sha "b91bbd276360bb0a865d85b48e048b831a35bc3f" 31 | #_#_:local/root "../open-source"}}} 32 | 33 | :dev 34 | {:extra-paths ["dev"] 35 | :extra-deps {djblue/portal {:mvn/version "RELEASE"}}} 36 | 37 | :bb 38 | {:extra-deps {nubank/matcher-combinators {:mvn/version "3.9.1"}}}}} 39 | -------------------------------------------------------------------------------- /dev/user.clj: -------------------------------------------------------------------------------- 1 | (ns user) 2 | 3 | (defmacro jit [sym] 4 | `(requiring-resolve '~sym)) 5 | 6 | (def portal-instance (atom nil)) 7 | 8 | (defn portal 9 | "Open a Portal window and register a tap handler for it. The result can be 10 | treated like an atom." 11 | [] 12 | ;; Portal is both an IPersistentMap and an IDeref, which confuses pprint. 13 | (prefer-method @(jit clojure.pprint/simple-dispatch) clojure.lang.IPersistentMap clojure.lang.IDeref) 14 | (let [p ((jit portal.api/open) @portal-instance)] 15 | (reset! portal-instance p) 16 | (add-tap (jit portal.api/submit)) 17 | p)) 18 | -------------------------------------------------------------------------------- /doc/01_introduction.md: -------------------------------------------------------------------------------- 1 | ## 1. Introduction 2 | 3 | > Quality is not an act, it is a habit. — Aristotle 4 | 5 | Kaocha is an all-in-one testing tool, its core task is to load tests and execute 6 | them, reporting on their progress and final result. It does this in a way that 7 | encourages good habits, supports multiple workflow styles, and optimizes for 8 | ergonomics. 9 | 10 | Kaocha has a modular architecture. It understands different types of tests: 11 | `clojure.test`, ClojureScript, Cucumber, Fudje, Expectations, so that all of a 12 | project's tests can be handled in the same way, and so that more can be added 13 | without requiring changes to the core. 14 | 15 | It aims to deliver all the features a developer might expect from their test 16 | tooling. Different people have different workflows, different styles of writing 17 | and running tests. We want to make sure we got you covered. 18 | 19 | Much of this is achieved through plugins. This way the Kaocha core can remain 20 | focused, while making it easy to experiment with new features. 21 | 22 | To use Kaocha you create a `tests.edn` at the root of your project, and run 23 | tests from the command line or from the REPL. 24 | 25 | Features include: 26 | 27 | - Filtering tests based on test names or metadata 28 | - Watch mode: watch the file system for changes and re-run tests 29 | - Pretty, pluggable reporting 30 | - Randomize test order 31 | - Detect when interrupted with Ctrl-C and print report 32 | - Fail fast mode: stop at first failure and print report 33 | - Profiling (show slowest tests) 34 | - Dynamic classpath handling 35 | - Tests as data (get test config, test plan, or test results as EDN) 36 | - Extensible test types (clojure.test, Cucumber, ...) 37 | - Extensible through plugins 38 | - Tool agnostic (Clojure CLI, Leiningen, boot) 39 | 40 | Currently Kaocha's versioning scheme is `1.${release count}-${commit count}`, 41 | and releases are made often. Kaocha is stable and we try to avoid breaking 42 | changes. If they aren't avoidable, we minimize their impact and the number of 43 | people affected. We especially avoid changes to APIs. However, breaking changes are 44 | sometimes necessary to fix a bug or UI pain point, so it's still good to keep 45 | an eye on the CHANGELOG. 46 | 47 | Kaocha requires Clojure 1.9. ClojureScript support requires Clojure and 48 | ClojureScript 1.10. 49 | -------------------------------------------------------------------------------- /doc/05_running_kaocha_repl.md: -------------------------------------------------------------------------------- 1 | # 5. Running Kaocha From the REPL 2 | 3 | For REPL use there's the 4 | [kaocha.repl](https://cljdoc.xyz/d/lambdaisland/kaocha/CURRENT/api/kaocha.repl) 5 | namespace. Its main entry point is the 6 | [run](https://cljdoc.xyz/d/lambdaisland/kaocha/CURRENT/api/kaocha.repl#run) 7 | function. Calling it is similar to starting Kaocha from the CLI: It will load 8 | `tests.edn`, merge in any extra options and flags, and then load and run your 9 | test suites. 10 | 11 | ## Filtering tests 12 | 13 | As arguments to `run` you pass it one or more identifiers of things you want to 14 | test. This can be a test suite, a namespace, or a specific test var. 15 | 16 | Say you have a single `:unit` test suite, with a `kaocha.random-test` namespace 17 | containing two tests. 18 | 19 | ``` 20 | . 21 | └── :unit 22 |     └── kaocha.random-test 23 |         ├── kaocha.random-test/rand-ints-test 24 |         └── kaocha.random-test/randomize-test 25 | 26 | ``` 27 | 28 | You could run the whole suite... 29 | 30 | ``` clojure 31 | (require '[kaocha.repl :as k]) 32 | 33 | (k/run :unit) 34 | ``` 35 | 36 | ...the namespace... 37 | 38 | ``` clojure 39 | (k/run 'kaocha.random-test) 40 | ``` 41 | 42 | ...or specific test vars: 43 | 44 | ``` clojure 45 | (k/run 'kaocha.random-test/rand-ints-test 'kaocha.random-test/randomize-test) 46 | ``` 47 | 48 | These are equivalent to using `--focus` on the command line. `run` also 49 | understand namespace and var objects. 50 | 51 | 52 | ``` clojure 53 | (k/run *ns*) 54 | (k/run #'rand-ints-test) 55 | ``` 56 | 57 | `(k/run)` without any arguments is equivalent to `(k/run *ns*)`. If you really want to run all test suites without discrimination, use [k/run-all](https://cljdoc.org/d/lambdaisland/kaocha/CURRENT/api/kaocha.repl#run-all). 58 | 59 | 60 | ## Passing configuration 61 | 62 | If the last argument to `(k/run)` is a map, then it is considered extra 63 | configuration and is applied on top of what is read from `tests.edn`. The 64 | special key `:config-file` is available to change the location from which 65 | `tests.edn` is read. 66 | 67 | ``` clojure 68 | (k/run {:config-file "/tmp/my_tests.edn"}) 69 | ``` 70 | 71 | Other keys in the map need to be either fully qualified keywords as used in 72 | Kaocha's configuration, or the short equivalent that is available in `tests.edn` 73 | when using the `#kaocha/v1` reader tag. 74 | 75 | ## In-buffer eval 76 | 77 | `kaocha.repl` is especially useful when used with a editor-connected REPL, so 78 | that code can be evaluated in place. When working on a specific test you can 79 | wrap it in `kaocha.repl/run`. Since `deftest` returns the var it defines, this 80 | redefines and runs the test in one go. 81 | 82 | ``` clojure 83 | (kaocha.repl/run 84 | (deftest my-test 85 | ,,,)) 86 | ``` 87 | 88 | When using CIDER this combines really well with 89 | `cider-pprint-eval-defun-at-point` (binding in CIDER 1.18.1: `C-c C-f`). 90 | 91 | ## Live reload at the REPL 92 | 93 | To enable live reloading of tests in your REPL session, you can call 94 | `(kaocha.watch/run (kaocha.repl/config))`. This will use all your standard 95 | config options, including watching the tests.edn file. 96 | 97 | ## Accessing configuration and test plans 98 | 99 | The `(kaocha.repl/config)` and `(kaocha.repl/test-plan)` functions are very 100 | useful when diagnosing issues, and can be helpful when developing plugins or 101 | test types. 102 | -------------------------------------------------------------------------------- /doc/06_focusing_and_skipping.md: -------------------------------------------------------------------------------- 1 | # 6. Focusing and Skipping 2 | 3 | Often you will want to *skip* certain tests, so they don't get run, or you want 4 | to *focus* on specific tests, so only those get run. 5 | 6 | For example, you can: 7 | 8 | - Skip tests that aren't finished yet 9 | - Skip tests marked as slow 10 | - Focus on a test that previously failed 11 | 12 | You can skip tests, or focus on tests, either based on a test `ID`, or on test 13 | or namespace metadata, based on four command line flags and configuration keys. 14 | 15 | ``` shell 16 | --skip SYM-OR-KW Skip tests with this ID and their children. 17 | --focus SYM-OR-KW Only run this test, skip others. 18 | --skip-meta SYM-OR-KW Skip tests where this metadata key is truthy. 19 | --focus-meta SYM-OR-KW Only run tests where this metadata key is truthy. 20 | ``` 21 | 22 | ## Matching 23 | 24 | Before running Kaocha builds a test plan where all tests are 25 | identified by a test `ID` keyword. The command line then 26 | canonicalises any IDs you supply into this keyword form, before 27 | matching them for focussing or skipping. 28 | 29 | ### On a test suite 30 | 31 | Assuming you have a test suite `:unit` specified in `tests.edn`: 32 | 33 | ``` clojure 34 | #kaocha/v1 35 | {:tests [{:id :unit 36 | :skip [...] 37 | :focus [...] 38 | :skip-meta [...] 39 | :focus-meta [...]}]} 40 | ``` 41 | 42 | You can focus on this by running: 43 | 44 | ``` shell 45 | bin/kaocha --focus :unit 46 | ``` 47 | 48 | ### On a namespace 49 | 50 | If you have tests in a namespace `com.my.project-test` and you want to 51 | run them all you can focus on them with the command: 52 | 53 | ``` shell 54 | bin/kaocha --focus com.my.project-test 55 | ``` 56 | 57 | ### On a test var 58 | 59 | Assuming you have a test var defined with for example `clojure.test` 60 | `deftest`, you can focus on it by supplying its fully qualified name 61 | like so: 62 | 63 | ``` shell 64 | bin/kaocha --focus com.my.project-test/foo-test 65 | ``` 66 | 67 | ### On metadata 68 | 69 | Suppose you have tests that are checked into version control, but that still need 70 | work. You can mark these with a metadata tag: 71 | 72 | ``` clojure 73 | (deftest ^:pending my-test 74 | ,,,) 75 | ``` 76 | 77 | To ignore such tests, add a `:skip-meta` key to the test suite config: 78 | 79 | ``` clojure 80 | #kaocha/v1 81 | {:tests [{:id :unit 82 | :skip-meta [:pending]}]} 83 | ``` 84 | 85 | And then run via the command: 86 | 87 | ``` shell 88 | bin/kaocha --focus :unit 89 | ``` 90 | 91 | This also works for metadata placed in the test's namespace, or any other 92 | metadata that a given test type implementation exposes. For example, 93 | kaocha-cucumber converts scenario tags into metadata. 94 | 95 | ### Focusing on metadata: special case 96 | 97 | `--focus-meta` will only work if at least one test has this metadata tag. If not 98 | a single test matches then this metadata is ignored. Assuming no other filters 99 | are in effect, this will result in running all tests. 100 | 101 | This way you can configure a certain key in `tests.edn` that you can use when 102 | you want to zone in on a specific test. Add the metadata to the test and only 103 | this test runs, remove it and the whole suite runs: 104 | 105 | ``` clojure 106 | #kaocha/v1 107 | {:tests [{:focus-meta [:xxx]}]} 108 | ``` 109 | 110 | ```clojure 111 | (deftest ^:xxx my-test 112 | ,,,) 113 | ``` 114 | -------------------------------------------------------------------------------- /doc/cljdoc.edn: -------------------------------------------------------------------------------- 1 | {:cljdoc/languages ["clj"]} 2 | -------------------------------------------------------------------------------- /doc/clojure_test/assertions.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # `clojure.test` assertion extensions 4 | 5 | \When running `clojure.test` based tests through Kaocha, some of the behavior 6 | is a little different. Kaocha tries to detect certain scenarios that are 7 | likely mistakes which make a test pass trivially, and turns them into errors 8 | so you can investigate and see what's up. 9 | 10 | Kaocha will also render failures differently, and provides extra multimethods 11 | to influence how certain failures are presented. 12 | 13 | ## Detecting missing assertions 14 | 15 | - Given a file named "test/sample_test.clj" with: 16 | 17 | ``` clojure 18 | (ns sample-test 19 | (:require [clojure.test :refer :all])) 20 | 21 | (deftest my-test 22 | (= 4 5)) 23 | ``` 24 | 25 | 26 | - When I run `bin/kaocha` 27 | 28 | - Then the output should contain: 29 | 30 | ``` text 31 | FAIL in sample-test/my-test (sample_test.clj:4) 32 | Test ran without assertions. Did you forget an (is ...)? 33 | ``` 34 | 35 | 36 | 37 | ## Detecting single argument `=` 38 | 39 | - Given a file named "test/sample_test.clj" with: 40 | 41 | ``` clojure 42 | (ns sample-test 43 | (:require [clojure.test :refer :all])) 44 | 45 | (deftest my-test 46 | (is (= 4) 5)) 47 | ``` 48 | 49 | 50 | - When I run `bin/kaocha` 51 | 52 | - Then the output should contain: 53 | 54 | ``` text 55 | FAIL in sample-test/my-test (sample_test.clj:5) 56 | Equality assertion expects 2 or more values to compare, but only 1 arguments given. 57 | Expected: 58 | (= 4 arg2) 59 | Actual: 60 | (= 4) 61 | 1 tests, 1 assertions, 1 failures. 62 | ``` 63 | 64 | 65 | 66 | ## Pretty printed diffs 67 | 68 | - Given a file named "test/sample_test.clj" with: 69 | 70 | ``` clojure 71 | (ns sample-test 72 | (:require [clojure.test :refer :all])) 73 | 74 | (defn my-fn [] 75 | {:xxx [1 2 3] 76 | :blue :red 77 | "hello" {:world :!}}) 78 | 79 | (deftest my-test 80 | (is (= {:xxx [1 3 4] 81 | "hello" {:world :?}} 82 | {:xxx [1 2 3] 83 | :blue :red 84 | "hello" {:world :!}}))) 85 | ``` 86 | 87 | 88 | - When I run `bin/kaocha` 89 | 90 | - Then the output should contain: 91 | 92 | ``` text 93 | FAIL in sample-test/my-test (sample_test.clj:10) 94 | Expected: 95 | {"hello" {:world :?}, :xxx [1 3 4]} 96 | Actual: 97 | {"hello" {:world -:? +:!}, :xxx [1 +2 3 -4], +:blue :red} 98 | 1 tests, 1 assertions, 1 failures. 99 | ``` 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /doc/command_line/capability_check.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Capability check for org.clojure/tools.cli 4 | 5 | If a project's dependency pulls in an old version of tools.cli, then this may 6 | break command line flags of the form `--[no-]xxx`. Before starting the main 7 | command line runner, Kaocha verifies that tools.cli has the necessary 8 | capabilities. 9 | 10 | ## With an outdated tools.cli 11 | 12 | - When I run `clojure -Sdeps '{:deps {org.clojure/tools.cli {:mvn/version "0.3.5"}}}' --main kaocha.runner` 13 | 14 | - Then stderr should contain: 15 | 16 | ``` nil 17 | org.clojure/tools.cli does not have all the capabilities that Kaocha needs. Make sure you are using version 0.3.6 or greater. 18 | ``` 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /doc/command_line/fail_fast.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # CLI: `--fail-fast` option 4 | 5 | Kaocha by default runs all tests it can find, providing a final summary on 6 | failures and errors when all tests have finished. With the `--fail-fast` 7 | option the test run will be interrupted as soon as a single failure or error 8 | has occured. Afterwards a summary of the test run so far is printed. 9 | 10 | ## Failing fast 11 | 12 | - Given a file named "test/my/project/fail_fast_test.clj" with: 13 | 14 | ``` clojure 15 | (ns my.project.fail-fast-test 16 | (:require [clojure.test :refer :all])) 17 | 18 | (deftest test-1 19 | (is true)) 20 | 21 | (deftest test-2 22 | (is true) 23 | (is false) 24 | (is true)) 25 | 26 | (deftest test-3 27 | (is true)) 28 | ``` 29 | 30 | 31 | - When I run `bin/kaocha --fail-fast` 32 | 33 | - Then the exit-code should be 1 34 | 35 | - And the output should contain: 36 | 37 | ``` nil 38 | [(..F)] 39 | 40 | FAIL in my.project.fail-fast-test/test-2 (fail_fast_test.clj:9) 41 | expected: false 42 | actual: false 43 | 2 tests, 3 assertions, 1 failures. 44 | ``` 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /doc/command_line/print_config.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # CLI: Print the Kaocha configuration 4 | 5 | A Kaocha test run starts with building up a Kaocha configuration map, based on 6 | default values, the contents of `tests.edn`, command line flags, and active 7 | plugins. 8 | 9 | Debugging issues with Kaocha often starts with inspecting the configuration, 10 | which is why a `--print-config` flag is provided. This builds up the 11 | configuration from any available sources, runs it through any active plugins, 12 | and then pretty prints the result, an EDN map. 13 | 14 | Note that the ordering, while not expected to change, is not guaranteed. We 15 | recommend parsing the configuration as EDN and not relying on order. If you 16 | are manipulating the output as text (say, on the command line) and can't 17 | avoid relying on the order, run it through a tool like 18 | [puget](https://github.com/greglook/puget) or 19 | [zprint](https://github.com/kkinnear/zprint) that sorts the keys 20 | alphabetically first. 21 | 22 | ## Using `--print-config` 23 | 24 | - When I run `bin/kaocha --print-config` 25 | 26 | - Then the EDN output should contain: 27 | 28 | ``` clojure 29 | {:kaocha.plugin.randomize/randomize? false, 30 | :kaocha/reporter [kaocha.report/dots], 31 | :kaocha/color? false, 32 | :kaocha/fail-fast? false} 33 | ``` 34 | 35 | 36 | - And the EDN output should contain: 37 | 38 | ``` clojure 39 | {:kaocha/tests 40 | [{:kaocha.testable/type :kaocha.type/clojure.test, 41 | :kaocha.testable/id :unit, 42 | :kaocha/ns-patterns ["-test$"], 43 | :kaocha/source-paths ["src"], 44 | :kaocha/test-paths ["test"], 45 | :kaocha.filter/skip-meta [:kaocha/skip]}]} 46 | ``` 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /doc/command_line/profile.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # CLI: `--profile` option 4 | 5 | The `--profile KEYWORD` flags sets the profile that is used to read the 6 | `tests.edn` configuration file. By using the `#profile {}` tagged reader 7 | literal you can provide different configuration values for different 8 | scenarios. 9 | 10 | If the `CI` environment value is set to `"true"`, as is the case on most CI 11 | platforms, then the profile will default to `:ci`. Otherwise it defaults to 12 | `:default`. 13 | 14 | ## Specifying profile on the command line 15 | 16 | - Given a file named "tests.edn" with: 17 | 18 | ``` clojure 19 | #kaocha/v1 20 | {:reporter #profile {:ci kaocha.report/documentation 21 | :default kaocha.report/dots}} 22 | ``` 23 | 24 | 25 | - And a file named "test/my/project/my_test.clj" with: 26 | 27 | ``` clojure 28 | (ns my.project.my-test 29 | (:require [clojure.test :refer :all])) 30 | 31 | (deftest test-1 32 | (is true)) 33 | ``` 34 | 35 | 36 | - When I run `bin/kaocha --profile :ci` 37 | 38 | - And the output should contain: 39 | 40 | ``` nil 41 | --- unit (clojure.test) --------------------------- 42 | my.project.my-test 43 | test-1 44 | ``` 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /doc/command_line/reporter.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # CLI: `--reporter` option 4 | 5 | The progress and summary printed by Kaocha are done by one or more "reporter" 6 | functions. A reporter can be specified with the `--reporter` option followed 7 | by a fully qualified function name. 8 | 9 | Reporters in the `kaocha.report` namespace can be specified without a 10 | namespace prefix. 11 | 12 | See the 13 | [`kaocha.report`](https://github.com/lambdaisland/kaocha/blob/master/src/kaocha/report.clj) 14 | namespace for built-in reporters. 15 | 16 | ## Background: An example test 17 | 18 | - Given a file named "test/my/project/reporter_test.clj" with: 19 | 20 | ``` clojure 21 | (ns my.project.reporter-test 22 | (:require [clojure.test :refer :all])) 23 | 24 | (deftest test-1 25 | (is (= 1 0))) 26 | 27 | (deftest test-2 28 | (is true) 29 | (is (throw (Exception. ""))) 30 | (is true)) 31 | 32 | (deftest test-3 33 | (is true)) 34 | ``` 35 | 36 | 37 | 38 | ## Using a fully qualified function as a reporter 39 | 40 | - When I run `bin/kaocha --reporter kaocha.report/documentation` 41 | 42 | - And the output should contain: 43 | 44 | ``` nil 45 | my.project.reporter-test 46 | test-1 FAIL 47 | test-2 ERROR 48 | test-3 49 | ``` 50 | 51 | 52 | 53 | ## Specifying a reporter via shorthand 54 | 55 | - When I run `bin/kaocha --reporter documentation` 56 | 57 | - Then the exit-code should be 2 58 | 59 | - And the output should contain: 60 | 61 | ``` nil 62 | my.project.reporter-test 63 | test-1 FAIL 64 | test-2 ERROR 65 | test-3 66 | ``` 67 | 68 | 69 | 70 | ## Using a reporter which does not exist 71 | 72 | - When I run `bin/kaocha --reporter does/not-exist` 73 | 74 | - Then stderr should contain 75 | 76 | ``` nil 77 | ERROR: Failed to resolve reporter var: does/not-exist 78 | ``` 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /doc/command_line/suite_names.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # CLI: Selecting test suites 4 | 5 | Each test suite has a unique id, given as a keyword in the test configuration. 6 | You can supply one or more of these ids on the command line to run only those 7 | test suites. 8 | 9 | ## Background: Given two test suites, `:aaa` and `:bbb` 10 | 11 | - Given a file named "tests.edn" with: 12 | 13 | ``` clojure 14 | #kaocha/v1 15 | {:tests [{:id :aaa 16 | :test-paths ["tests/aaa"]} 17 | {:id :bbb 18 | :test-paths ["tests/bbb"]}]} 19 | ``` 20 | 21 | 22 | - And a file named "tests/aaa/aaa_test.clj" with: 23 | 24 | ``` clojure 25 | (ns aaa-test (:require [clojure.test :refer :all])) 26 | (deftest foo-test (is true)) 27 | ``` 28 | 29 | 30 | - And a file named "tests/bbb/bbb_test.clj" with: 31 | 32 | ``` clojure 33 | (ns bbb-test (:require [clojure.test :refer :all])) 34 | (deftest bbb-test (is true)) 35 | ``` 36 | 37 | 38 | 39 | ## Specifying a test suite on the command line 40 | 41 | - When I run `bin/kaocha aaa --reporter documentation` 42 | 43 | - And the output should contain: 44 | 45 | ``` nil 46 | aaa-test 47 | foo-test 48 | ``` 49 | 50 | 51 | - And the output should not contain 52 | 53 | ``` nil 54 | bbb-test 55 | ``` 56 | 57 | 58 | 59 | ## Specifying a test suite using keyword syntax 60 | 61 | - When I run `bin/kaocha :aaa --reporter documentation` 62 | 63 | - And the output should contain: 64 | 65 | ``` nil 66 | aaa-test 67 | foo-test 68 | ``` 69 | 70 | 71 | - And the output should not contain 72 | 73 | ``` nil 74 | bbb-test 75 | ``` 76 | 77 | 78 | 79 | ## Specifying an unknown suite 80 | 81 | - When I run `bin/kaocha suite-name` 82 | 83 | - Then the output should contain: 84 | 85 | ``` nil 86 | No such suite: :suite-name, valid options: :aaa, :bbb. 87 | ``` 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /doc/config/warnings.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Configuration: Warnings 4 | 5 | Kaocha will warn about common mistakes. 6 | 7 | ## No config 8 | 9 | - Given a file named "test/my/foo_test.clj" with: 10 | 11 | ``` clojure 12 | (ns my.foo-test 13 | (:require [clojure.test :refer :all])) 14 | 15 | (deftest var-test 16 | (is (= 456 456))) 17 | ``` 18 | 19 | 20 | - When I run `bin/kaocha -c alt-tests.edn` 21 | 22 | - Then stderr should contain: 23 | 24 | ``` nil 25 | Did not load a configuration file and using the defaults. 26 | ``` 27 | 28 | 29 | 30 | ## Warn about bad configuration 31 | 32 | - Given a file named "tests.edn" with: 33 | 34 | ``` clojure 35 | #kaocha/v1 36 | {:plugins notifier} 37 | ``` 38 | 39 | 40 | - And a file named "test/my/foo_test.clj" with: 41 | 42 | ``` clojure 43 | (ns my.foo-test 44 | (:require [clojure.test :refer :all])) 45 | 46 | (deftest var-test 47 | (is (= 456 456))) 48 | ``` 49 | 50 | 51 | - When I run `bin/kaocha` 52 | 53 | - Then stderr should contain: 54 | 55 | ``` nil 56 | Invalid configuration file: 57 | ``` 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /doc/extensions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaisland/kaocha/525af0545afeab42f9d1e27eb5331fbc5119ca90/doc/extensions.png -------------------------------------------------------------------------------- /doc/filtering/focus_meta.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Focusing based on metadata 4 | 5 | You can limit the test run based on test's metadata. How to associate metadata 6 | with a test depends on the test type, for `clojure.test` type tests metadata 7 | can be associated with a test var or test namespace. 8 | 9 | Using the `--focus-meta` command line flag, or `:kaocha.filter/focus-meta` key 10 | in test suite configuration, you can limit the tests being run to only those 11 | where the given metadata key has a truthy value associated with it. 12 | 13 | ## Background: Some tests with metadata 14 | 15 | - Given a file named "test/my/project/sample_test.clj" with: 16 | 17 | ``` clojure 18 | (ns ^:xxx my.project.sample-test 19 | (:require [clojure.test :refer :all])) 20 | 21 | (deftest some-test 22 | (is (= 1 1))) 23 | 24 | (deftest other-test 25 | (is (= 2 2))) 26 | ``` 27 | 28 | 29 | - And a file named "test/my/project/other_sample_test.clj" with: 30 | 31 | ``` clojure 32 | (ns my.project.other-sample-test 33 | (:require [clojure.test :refer :all])) 34 | 35 | (deftest ^:yyy other-test 36 | (is (= 3 3))) 37 | ``` 38 | 39 | 40 | 41 | ## Focusing by metadata from the command line 42 | 43 | - When I run `bin/kaocha --focus-meta :xxx --reporter documentation` 44 | 45 | - Then the output should contain: 46 | 47 | ``` nil 48 | --- unit (clojure.test) --------------------------- 49 | my.project.sample-test 50 | other-test 51 | some-test 52 | 53 | 2 tests, 2 assertions, 0 failures. 54 | ``` 55 | 56 | 57 | 58 | ## Focusing on a test group by metadata from the command line 59 | 60 | - When I run `bin/kaocha --focus-meta :yyy --reporter documentation` 61 | 62 | - Then the output should contain: 63 | 64 | ``` nil 65 | --- unit (clojure.test) --------------------------- 66 | my.project.other-sample-test 67 | other-test 68 | 69 | 1 tests, 1 assertions, 0 failures. 70 | ``` 71 | 72 | 73 | 74 | ## Focusing based on metadata via configuration 75 | 76 | - Given a file named "tests.edn" with: 77 | 78 | ``` edn 79 | #kaocha/v1 80 | {:tests [{:kaocha.filter/focus-meta [:yyy]}] 81 | :color? false 82 | :randomize? false} 83 | ``` 84 | 85 | 86 | - When I run `bin/kaocha --reporter documentation` 87 | 88 | - Then the output should contain: 89 | 90 | ``` nil 91 | --- unit (clojure.test) --------------------------- 92 | my.project.other-sample-test 93 | other-test 94 | 95 | 1 tests, 1 assertions, 0 failures. 96 | ``` 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /doc/filtering/focusing.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Focusing on specific tests 4 | 5 | You can limit the test run to only specific tests or test groups (e.g. 6 | namespaces) using the `--focus` command line flag, or `:kaocha.filter/focus` 7 | key in test suite configuration. 8 | 9 | ## Background: A simple test suite 10 | 11 | - Given a file named "test/my/project/sample_test.clj" with: 12 | 13 | ``` clojure 14 | (ns my.project.sample-test 15 | (:require [clojure.test :refer :all])) 16 | 17 | (deftest some-test 18 | (is (= 1 1))) 19 | 20 | (deftest other-test 21 | (is (= 2 2))) 22 | ``` 23 | 24 | 25 | - And a file named "test/my/project/other_sample_test.clj" with: 26 | 27 | ``` clojure 28 | (ns my.project.other-sample-test 29 | (:require [clojure.test :refer :all])) 30 | 31 | (deftest other-test 32 | (is (= 1 2))) 33 | ``` 34 | 35 | 36 | 37 | ## Focusing on test id from the command line 38 | 39 | - When I run `bin/kaocha --focus my.project.sample-test/some-test --reporter documentation` 40 | 41 | - Then the output should contain: 42 | 43 | ``` nil 44 | --- unit (clojure.test) --------------------------- 45 | my.project.sample-test 46 | some-test 47 | 48 | 1 tests, 1 assertions, 0 failures. 49 | ``` 50 | 51 | 52 | 53 | ## Focusing on test group id from the command line 54 | 55 | - When I run `bin/kaocha --focus my.project.sample-test --reporter documentation` 56 | 57 | - Then the output should contain: 58 | 59 | ``` nil 60 | --- unit (clojure.test) --------------------------- 61 | my.project.sample-test 62 | other-test 63 | some-test 64 | 65 | 2 tests, 2 assertions, 0 failures. 66 | ``` 67 | 68 | 69 | 70 | ## Focusing via configuration 71 | 72 | - Given a file named "tests.edn" with: 73 | 74 | ``` edn 75 | #kaocha/v1 76 | {:tests [{:kaocha.filter/focus [my.project.sample-test/other-test]}] 77 | :color? false 78 | :randomize? false} 79 | ``` 80 | 81 | 82 | - When I run `bin/kaocha --reporter documentation` 83 | 84 | - Then the output should contain: 85 | 86 | ``` nil 87 | --- unit (clojure.test) --------------------------- 88 | my.project.sample-test 89 | other-test 90 | 91 | 1 tests, 1 assertions, 0 failures. 92 | ``` 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /doc/filtering/skipping.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Skipping test based on ids 4 | 5 | You can tell Kaocha to completely ignore certain tests or test groups, either 6 | with the `--skip` command line flag, or the `:kaocha.filter/skip` test suite 7 | configuration key. 8 | 9 | Both of these take test ids or test group ids (e.g. the fully qualified name 10 | of a test var, or the name of a test namespace). 11 | 12 | ## Background: A simple test suite 13 | 14 | - Given a file named "test/my/project/sample_test.clj" with: 15 | 16 | ``` clojure 17 | (ns my.project.sample-test 18 | (:require [clojure.test :refer :all])) 19 | 20 | (deftest some-test 21 | (is (= 1 1))) 22 | 23 | (deftest other-test 24 | (is (= 2 2))) 25 | ``` 26 | 27 | 28 | - And a file named "test/my/project/other_sample_test.clj" with: 29 | 30 | ``` clojure 31 | (ns my.project.other-sample-test 32 | (:require [clojure.test :refer :all])) 33 | 34 | (deftest other-test 35 | (is (= 3 3))) 36 | ``` 37 | 38 | 39 | 40 | ## Skipping test id from the command line 41 | 42 | - When I run `bin/kaocha --skip my.project.sample-test/some-test --reporter documentation` 43 | 44 | - Then the output should contain: 45 | 46 | ``` nil 47 | --- unit (clojure.test) --------------------------- 48 | my.project.other-sample-test 49 | other-test 50 | 51 | my.project.sample-test 52 | other-test 53 | 54 | 2 tests, 2 assertions, 0 failures. 55 | ``` 56 | 57 | 58 | 59 | ## Skipping a test group id from the command line 60 | 61 | - When I run `bin/kaocha --skip my.project.sample-test --reporter documentation` 62 | 63 | - Then the output should contain: 64 | 65 | ``` nil 66 | --- unit (clojure.test) --------------------------- 67 | my.project.other-sample-test 68 | other-test 69 | 70 | 1 tests, 1 assertions, 0 failures. 71 | ``` 72 | 73 | 74 | 75 | ## Skipping via configuration 76 | 77 | - Given a file named "tests.edn" with: 78 | 79 | ``` edn 80 | #kaocha/v1 81 | {:tests [{:kaocha.filter/skip [my.project.sample-test]}] 82 | :color? false 83 | :randomize? false} 84 | ``` 85 | 86 | 87 | - When I run `bin/kaocha --reporter documentation` 88 | 89 | - Then the output should contain: 90 | 91 | ``` nil 92 | --- unit (clojure.test) --------------------------- 93 | my.project.other-sample-test 94 | other-test 95 | 96 | 1 tests, 1 assertions, 0 failures. 97 | ``` 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /doc/images/deep-diff.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaisland/kaocha/525af0545afeab42f9d1e27eb5331fbc5119ca90/doc/images/deep-diff.png -------------------------------------------------------------------------------- /doc/pending.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Marking tests as pending 4 | 5 | Pending tests are tests that are not yet implemented, or that need fixing, and 6 | that you don't want to forget about. Pending tests are similar to skipped 7 | tests (see the section on "Filtering"), in that the runner will skip over them 8 | without trying to run them. 9 | 10 | The difference is that pending tests are explicitly reported in the test 11 | result. At the end of each test run you get to see the number of pending 12 | tests, followed by a list of their test ids and file/line information. This 13 | constant reminder is there to make sure pending tests are not left 14 | unaddressed. 15 | 16 | Add the `^:kaocha/pending` metadata to a test to mark it as pending. The 17 | metadata check is done inside the Kaocha runner itself, not in the specific 18 | test type implementation, so this metadata is supported on any test type that 19 | allows setting metadata tags, including ClojureScript and Cucumber tests. 20 | 21 | ## Marking a test as pending 22 | 23 | - Given a file named "test/sample/sample_test.clj" with: 24 | 25 | ``` clojure 26 | (ns sample.sample-test 27 | (:require [clojure.test :refer :all])) 28 | 29 | (deftest ^:kaocha/pending my-test) 30 | ``` 31 | 32 | 33 | - When I run `bin/kaocha` 34 | 35 | - Then the output should contain: 36 | 37 | ``` nil 38 | [(P)] 39 | 1 tests, 0 assertions, 1 pending, 0 failures. 40 | 41 | PENDING sample.sample-test/my-test (sample/sample_test.clj:4) 42 | ``` 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /doc/plugins/capture_output.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Plugin: Capture output 4 | 5 | Kaocha has a plugin which will capture all output written to stdout or stderr 6 | during the test run. When tests pass this output is hidden, when they fail the 7 | output is made visible to help understand the problem. 8 | 9 | This plugin is loaded by default, but can be disabled with `--no-capture-output` 10 | 11 | ## Show output of failing test 12 | 13 | - Given a file named "test/sample_test.clj" with: 14 | 15 | ``` clojure 16 | (ns sample-test 17 | (:require [clojure.test :refer :all])) 18 | 19 | (deftest stdout-pass-test 20 | (println "You peng zi yuan fang lai") 21 | (is (= :same :same))) 22 | 23 | (deftest stdout-fail-test 24 | (println "Bu yi le hu?") 25 | (is (= :same :not-same))) 26 | ``` 27 | 28 | 29 | - When I run `bin/kaocha` 30 | 31 | - Then the output should contain: 32 | 33 | ``` nil 34 | FAIL in sample-test/stdout-fail-test (sample_test.clj:10) 35 | Expected: 36 | :same 37 | Actual: 38 | -:same +:not-same 39 | ╭───── Test output ─────────────────────────────────────────────────────── 40 | │ Bu yi le hu? 41 | ╰───────────────────────────────────────────────────────────────────────── 42 | 2 tests, 2 assertions, 1 failures. 43 | ``` 44 | 45 | 46 | 47 | ## Bypass output capturing 48 | 49 | - Given a file named "test/sample_test.clj" with: 50 | 51 | ``` clojure 52 | (ns sample-test 53 | (:require [clojure.test :refer :all] 54 | [kaocha.plugin.capture-output :as capture])) 55 | 56 | (deftest stdout-pass-test 57 | (capture/bypass 58 | (println "You peng zi yuan fang lai")) 59 | (is (= :same :same))) 60 | ``` 61 | 62 | 63 | - When I run `bin/kaocha` 64 | 65 | - Then the output should contain: 66 | 67 | ``` nil 68 | [(You peng zi yuan fang lai 69 | .)] 70 | 1 tests, 1 assertions, 0 failures. 71 | ``` 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /doc/plugins/hooks_plugin.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Plugin: Hooks 4 | 5 | The hooks plugin allows hooking into Kaocha's process with arbitrary 6 | functions. This is very similar to using writing a plugin, but requires less 7 | boilerplate. 8 | 9 | See the documentation for extending Kaocha for a description of the different 10 | hooks. The supported hooks are: config, pre-load, post-load, pre-run, post-run, wrap-run, 11 | pre-test, post-test, pre-report, post-summary. 12 | 13 | The hooks plugin also provides hooks at the test suite level, which in order 14 | are `:kaocha.hooks/pre-load-test`, `:kaocha.hooks/post-load-test`, 15 | `:kaocha.hooks/pre-test`, `:kaocha.hooks/post-test`. 16 | 17 | Hooks can be specified as a fully qualified symbol referencing a function, or 18 | a collection thereof. The referenced namespaces will be loaded during the 19 | config phase. It's also possible to have functions directly inline, but due to 20 | limitations of the EDN reader this is of limited use, and you are generally 21 | better off sticking your hooks into a proper namespace. 22 | 23 | ## Implementing a hook 24 | 25 | - Given a file named "tests.edn" with: 26 | 27 | ``` clojure 28 | #kaocha/v1 29 | {:plugins [:kaocha.plugin/hooks] 30 | :kaocha.hooks/pre-test [my.kaocha.hooks/sample-hook]} 31 | ``` 32 | 33 | 34 | - And a file named "src/my/kaocha/hooks.clj" with: 35 | 36 | ``` clojure 37 | (ns my.kaocha.hooks) 38 | 39 | (println "ok") 40 | 41 | (defn sample-hook [test test-plan] 42 | (if (re-find #"fail" (str (:kaocha.testable/id test))) 43 | (assoc test :kaocha.testable/pending true) 44 | test)) 45 | ``` 46 | 47 | 48 | - And a file named "test/sample_test.clj" with: 49 | 50 | ``` clojure 51 | (ns sample-test 52 | (:require [clojure.test :refer :all])) 53 | 54 | (deftest stdout-pass-test 55 | (println "You peng zi yuan fang lai") 56 | (is (= :same :same))) 57 | 58 | (deftest stdout-fail-test 59 | (println "Bu yi le hu?") 60 | (is (= :same :not-same))) 61 | ``` 62 | 63 | 64 | - When I run `bin/kaocha` 65 | 66 | - Then the output should contain: 67 | 68 | ``` nil 69 | PENDING sample-test/stdout-fail-test (sample_test.clj:8) 70 | ``` 71 | 72 | 73 | 74 | ## Implementing a test-suite specific hook 75 | 76 | - Given a file named "tests.edn" with: 77 | 78 | ``` clojure 79 | #kaocha/v1 80 | {:plugins [:kaocha.plugin/hooks] 81 | :tests [{:id :unit 82 | :kaocha.hooks/before [my.kaocha.hooks/sample-before-hook] 83 | :kaocha.hooks/after [my.kaocha.hooks/sample-after-hook]}]} 84 | ``` 85 | 86 | 87 | - And a file named "src/my/kaocha/hooks.clj" with: 88 | 89 | ``` clojure 90 | (ns my.kaocha.hooks) 91 | 92 | (defn sample-before-hook [suite test-plan] 93 | (println "before suite:" (:kaocha.testable/id suite)) 94 | suite) 95 | 96 | (defn sample-after-hook [suite test-plan] 97 | (println "after suite:" (:kaocha.testable/id suite)) 98 | suite) 99 | ``` 100 | 101 | 102 | - And a file named "test/sample_test.clj" with: 103 | 104 | ``` clojure 105 | (ns sample-test 106 | (:require [clojure.test :refer :all])) 107 | 108 | (deftest stdout-pass-test 109 | (println "You peng zi yuan fang lai") 110 | (is (= :same :same))) 111 | ``` 112 | 113 | 114 | - When I run `bin/kaocha` 115 | 116 | - Then the output should contain: 117 | 118 | ``` nil 119 | before suite: :unit 120 | [(.)]after suite: :unit 121 | ``` 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /doc/plugins/notifier_plugin.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Plugin: Notifier (desktop notifications) 4 | 5 | Desktop notifications can be enabled with the `:kaocha.plugin/notifier` 6 | plugin. This will pop up a fail/pass notification bubble including a summary 7 | of tests passed/errored/failed at the end of each test run. It's particularly 8 | useful in combination with `--watch`, e.g. `bin/kaocha --plugin notifier 9 | --watch`. 10 | 11 | It does this by invoking a shell command which can be configured, so it can be 12 | used to invoke an arbitrary command or script. By default it will try to 13 | detect which command to use, using either `notify-send` (Linux) or 14 | `terminal-notifier` (Mac OS X), either of which may need to be installed 15 | first. 16 | 17 | Several replacement patterns are available: 18 | 19 | - `%{title}` : The notification title, either `⛔️ Failing` or `✅ Passing` 20 | - `%{message}` : Test result summary, e.g. `5 tests, 12 assertions, 0 failures` 21 | - `%{icon}` : Full local path to an icon to use (currently uses the Clojure icon) 22 | - `%{failed?}` : `true` if any tests failed or errored, `false` otherwise 23 | - `%{count}` : the number of tests 24 | - `%{pass}` : the number of passing assertions 25 | - `%{fail}` : the number of failing assertions 26 | - `%{error}` : the number of errors 27 | - `%{pending}` : the number of pending tests 28 | - `%{urgency}` : `normal` if the tests pass, `critical` otherwise, meant for use with `notify-send` 29 | 30 | If no command is configured, and neither notification command is found, then 31 | the plugin will silently do nothing. You can explicitly inhibit its behaviour 32 | with `--no-notifications`. 33 | 34 | ## Enabling Desktop Notifications 35 | 36 | - Given a file named "tests.edn" with: 37 | 38 | ``` clojure 39 | #kaocha/v1 40 | {:plugins [:kaocha.plugin/notifier] 41 | 42 | ;; Configuring a command is optional. Since CI does not have desktop 43 | ;; notifications we pipe to a file instead. 44 | :kaocha.plugin.notifier/command 45 | "sh -c 'echo \"%{title}\n%{message}\n%{failed?}\n%{count}\n%{urgency}\" > /tmp/kaocha.txt'" 46 | 47 | ;; Fallbacks: 48 | 49 | ;; :kaocha.plugin.notifier/command 50 | ;; "notify-send -a Kaocha %{title} %{message} -i %{icon} -u %{urgency}" 51 | 52 | ;; :kaocha.plugin.notifier/command 53 | ;; "terminal-notifier -message %{message} -title %{title} -appIcon %{icon}" 54 | } 55 | ``` 56 | 57 | 58 | - And a file named "test/sample_test.clj" with: 59 | 60 | ``` clojure 61 | (ns sample-test 62 | (:require [clojure.test :refer :all])) 63 | 64 | (deftest simple-fail-test 65 | (is (= :same :not-same))) 66 | ``` 67 | 68 | 69 | - When I run `bin/kaocha` 70 | 71 | - And I run `cat /tmp/kaocha.txt` 72 | 73 | - Then the output should contain: 74 | 75 | ``` nil 76 | ⛔️ Failing 77 | 1 tests, 1 failures. 78 | true 79 | 1 80 | critical 81 | ``` 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /doc/plugins/orchestra_plugin.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Orchestra (spec instrumentation) 4 | 5 | You can enable spec instrumentation of your functions before running 6 | tests with the `:kaocha.plugin/orchestra` plugin. This uses the 7 | [Orchestra](https://github.com/jeaye/orchestra) library to instrument 8 | `:args`, `:ret`, and `:fn` specs. 9 | 10 | You can use the `:kaocha.plugin/preloads` plugin to ensure namespaces 11 | are required (similar to ClojureScript's preloads feature). This is 12 | useful to ensure that your specs required before the orchestra plugin 13 | instruments your functions. 14 | 15 | ## Enabling Orchestra 16 | 17 | - Given a file named "tests.edn" with: 18 | 19 | ``` clojure 20 | #kaocha/v1 21 | {:plugins [:orchestra 22 | :preloads] 23 | :kaocha.plugin.preloads/ns-names [my.specs] 24 | :color? false} 25 | ``` 26 | 27 | 28 | - And a file named "test/orchestra_test.clj" with: 29 | 30 | ``` clojure 31 | (ns orchestra-test 32 | (:require [clojure.test :refer :all] 33 | [clojure.spec.alpha :as spec])) 34 | 35 | (defn simple-fn [] 36 | "x") 37 | 38 | (spec/fdef simple-fn :ret :simple/int) 39 | 40 | (deftest spec-fail-test 41 | (is (= "x" (simple-fn)) "Just testing simple-fn")) 42 | ``` 43 | 44 | 45 | - And a file named "src/my/specs.clj" with: 46 | 47 | ``` clojure 48 | (ns my.specs 49 | (:require [clojure.spec.alpha :as spec])) 50 | 51 | (spec/def :simple/int int?) 52 | ``` 53 | 54 | 55 | - When I run `bin/kaocha` 56 | 57 | - Then the output should contain: 58 | 59 | ``` nil 60 | ERROR in orchestra-test/spec-fail-test (orchestra_test.clj:11) 61 | Just testing simple-fn 62 | Call to orchestra-test/simple-fn did not conform to spec. 63 | orchestra_test.clj:11 64 | 65 | -- Spec failed -------------------- 66 | 67 | Return value 68 | 69 | "x" 70 | 71 | should satisfy 72 | 73 | int? 74 | 75 | -- Relevant specs ------- 76 | 77 | :simple/int: 78 | clojure.core/int? 79 | 80 | ------------------------- 81 | Detected 1 error 82 | ``` 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /doc/plugins/version_filter.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Plugin: Clojure/Java Version filter 4 | 5 | The `version-filter` plugin will look for test metadata specifying the minimum 6 | or maximum version of Clojure or Java the test is designed to work with, and 7 | skip the test unless it falls within the range specified. 8 | 9 | The recognized metadata keys are `:min-clojure-version`, 10 | `:max-clojure-version`, `:min-java-version`, and `:max-java-version`. The 11 | associated value is a version string, such as `"1.10.0"`. 12 | 13 | You can set both a minimum and a maximum to limit to a certain range. The 14 | boundaries are always inclusive, so `^{:max-clojure-version "1.9"}` will run 15 | on Clojure `1.9.*` or earlier. 16 | 17 | Specificty matters, a test with a max version of `"1.10" will also run on 18 | version `"1.10.2"`, whereas if the max version is `"1.10.0"` it will not. 19 | 20 | Note that the Java version is based on the "java.runtime.version" system 21 | property. Before Java 9 this was the so called "developer version", which 22 | started with `1.`, e.g. `"1.8.0"`, so Java (JDK) versions effectivel jumped 23 | from `1.8` to `9`. 24 | [1](https://blogs.oracle.com/java-platform-group/a-new-jdk-9-version-string-scheme) 25 | [2](https://en.wikipedia.org/wiki/Java_version_history#Versioning_change) 26 | 27 | ## Enabling in `tests.edn` 28 | 29 | - Given a file named "tests.edn" with: 30 | 31 | ``` clojure 32 | #kaocha/v1 33 | {:plugins [:kaocha.plugin/version-filter] 34 | :color? false} 35 | ``` 36 | 37 | 38 | - And a file named "test/my/sample_test.clj" with: 39 | 40 | ``` clojure 41 | (ns my.sample-test 42 | (:require [clojure.test :refer :all])) 43 | 44 | (deftest ^{:max-java-version "1.7"} this-test-gets-skipped 45 | (is false)) 46 | 47 | (deftest ^{:min-clojure-version "1.6.0"} this-test-runs 48 | (is true)) 49 | ``` 50 | 51 | 52 | - When I run `bin/kaocha --reporter documentation` 53 | 54 | - Then the output should contain: 55 | 56 | ``` nil 57 | --- unit (clojure.test) --------------------------- 58 | my.sample-test 59 | this-test-runs 60 | 61 | 1 tests, 1 assertions, 0 failures. 62 | ``` 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /doc/syntax_error.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Syntax errors are preserved 4 | 5 | Syntax errors should be passed along. 6 | 7 | ## Show output of failing test 8 | 9 | - Given a file named "test/sample_test.clj" with: 10 | 11 | ``` clojure 12 | (ns sample-test 13 | (:require [clojure.test :refer :all])) 14 | 15 | stray-symbol 16 | 17 | (deftest stdout-pass-test 18 | (is (= :same :same))) 19 | ``` 20 | 21 | 22 | - When I run `bin/kaocha` 23 | 24 | - Then the output should contain: 25 | 26 | ``` nil 27 | Exception: clojure.lang.Compiler$CompilerException 28 | ``` 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /examples/blame_report.clj: -------------------------------------------------------------------------------- 1 | (ns my.project.kaocha-hooks 2 | (:require [kaocha.testable :as testable] 3 | [kaocha.hierarchy :as hierarchy] 4 | [kaocha.result :as result] 5 | [clojure.java.shell :as sh] 6 | [clojure.java.io :as io] 7 | [clojure.string :as str])) 8 | 9 | ;; use as a post-summary hook 10 | (defn blame-report [result] 11 | (let [clojure-test-suites (filter (comp #{:kaocha.type/clojure.test} :kaocha.testable/type) 12 | (:kaocha.result/tests result))] 13 | (doseq [suite clojure-test-suites 14 | ns-testable (:kaocha.result/tests suite) 15 | :when (result/failed? ns-testable) 16 | :let [ns-name (:kaocha.ns/name ns-testable)]] 17 | (println 18 | ns-name "last touched by" 19 | (re-find #"Author:.*" 20 | (:out 21 | (sh/sh "git" "log" "-1" (str/replace (str (or (io/resource (str (.. (name ns-name) 22 | (replace \- \_) 23 | (replace \. \/)) 24 | ".clj")) 25 | (io/resource (str (.. (name ns-name) 26 | (replace \- \_) 27 | (replace \. \/)) 28 | ".cljc")))) 29 | "file:" ""))))))) 30 | result) 31 | 32 | 33 | (comment 34 | (require '[kaocha.repl :as repl] 35 | '[kaocha.api :as api]) 36 | 37 | (def test-result (api/run (repl/config {}))) 38 | 39 | (blame-report test-result) 40 | ) 41 | -------------------------------------------------------------------------------- /fixtures/a-tests/baz/qux_test.clj: -------------------------------------------------------------------------------- 1 | (ns baz.qux-test 2 | (:require [clojure.test :as t :refer [deftest is use-fixtures]])) 3 | 4 | (defn once-fix 5 | [f] 6 | (try (f) 7 | (catch Throwable _))) 8 | 9 | (defn each-fix 10 | [f] 11 | (try (f) 12 | (catch Throwable _))) 13 | 14 | (use-fixtures :once #'once-fix) 15 | (use-fixtures :each #'each-fix) 16 | 17 | (deftest nested-test 18 | (is (= 1 1)) 19 | (throw (Exception. "fake exception")) 20 | (is (= 2 1))) 21 | -------------------------------------------------------------------------------- /fixtures/a-tests/foo/bar_test.clj: -------------------------------------------------------------------------------- 1 | (ns foo.bar-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (def a-var :ok) 5 | 6 | (deftest a-test 7 | (is true)) 8 | -------------------------------------------------------------------------------- /fixtures/b-tests/finn/finn_test.clj: -------------------------------------------------------------------------------- 1 | (ns finn.finn-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest the-test 5 | (is true)) 6 | -------------------------------------------------------------------------------- /fixtures/c-tests/foo/hello_test.clj: -------------------------------------------------------------------------------- 1 | (ns foo.hello-test 2 | (:require [clojure.test :as t])) 3 | 4 | (t/deftest pass-1 5 | (t/is true)) 6 | 7 | (t/deftest pass-2 8 | (t/is true)) 9 | 10 | (t/deftest fail-1 11 | (t/is true) 12 | (t/is false) 13 | (t/is true)) 14 | 15 | (t/deftest pass-3 16 | (t/is true)) 17 | -------------------------------------------------------------------------------- /fixtures/custom_config.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :foo 3 | :test-paths ["test/foo"]}] 4 | :reporter [kaocha.report.progress/report] 5 | :color? false 6 | :fail-fast? true 7 | :plugins [:kaocha.plugin.alpha/xfail]} 8 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/bar_test.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.bar-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest test-1 5 | (is true)) 6 | 7 | (deftest test-2 8 | (is true)) 9 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/double_once_fixture_test.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.double-once-fixture-test 2 | (:require [clojure.test :refer [deftest is use-fixtures]])) 3 | 4 | (def ^:dynamic *val* nil) 5 | 6 | (defn doubled-fixture [f] 7 | (binding [*val* :one] 8 | (f)) 9 | (binding [*val* :two] 10 | (f))) 11 | 12 | (use-fixtures :once doubled-fixture) 13 | 14 | (deftest example-fail-test 15 | (if (= :one *val*) 16 | (is (= 1 2)) 17 | (is (= 2 2)))) 18 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/exception_outside_is.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.exception-outside-is 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest exception-outside-is-test 5 | (throw (Exception. "booo"))) 6 | -------------------------------------------------------------------------------- /fixtures/d-tests/ddd/foo_test.clj: -------------------------------------------------------------------------------- 1 | (ns ddd.foo-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | (deftest test-1 5 | (is false)) 6 | 7 | (deftest test-2 8 | (is true)) 9 | 10 | (deftest test-3 11 | (is (throw (Exception. "fail!")))) 12 | -------------------------------------------------------------------------------- /fixtures/e-tests/eee/bad_code_test.clj: -------------------------------------------------------------------------------- 1 | (ns eee.bad-code-test 2 | (:require [clojure.test :refer :all])) 3 | 4 | ;; (def oops-iam-commented-out "oops") 5 | 6 | (deftest test-1 7 | (is (= "oops" oops-iam-commented-out))) 8 | -------------------------------------------------------------------------------- /fixtures/tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :a 3 | :test-paths ["fixtures/a-tests"]} 4 | {:id :b 5 | :test-paths ["fixtures/b-tests"]}]} 6 | -------------------------------------------------------------------------------- /fixtures/with_compile_error.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :bad-code 3 | :test-paths ["fixtures/e-tests"] 4 | :ns-patterns [""] 5 | :kaocha.filter/focus [eee.bad-code-test]}]} 6 | -------------------------------------------------------------------------------- /fixtures/with_exception.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :inside-is 3 | :test-paths ["fixtures/d-tests"] 4 | :kaocha.filter/focus [ddd.foo-test/test-3]} 5 | {:id :outside-is 6 | :test-paths ["fixtures/d-tests"] 7 | :ns-patterns [""] 8 | :kaocha.filter/focus [ddd.exception-outside-is]}]} 9 | -------------------------------------------------------------------------------- /fixtures/with_failing.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:tests [{:id :a 3 | :test-paths ["fixtures/a-tests"]} 4 | 5 | {:id :c 6 | :test-paths ["fixtures/c-tests"]} 7 | 8 | {:id :b 9 | :test-paths ["fixtures/b-tests"]}]} 10 | -------------------------------------------------------------------------------- /kaocha.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaisland/kaocha/525af0545afeab42f9d1e27eb5331fbc5119ca90/kaocha.png -------------------------------------------------------------------------------- /repl_sessions/strict_focus_meta_hook.clj: -------------------------------------------------------------------------------- 1 | (ns strict-focus-meta-hook 2 | (:require [kaocha.repl :as repl] 3 | [clojure.walk :as w] 4 | [kaocha.hierarchy :as hierarchy] 5 | [kaocha.testable :as testable])) 6 | 7 | ;; #kaocha @madstap 2020-07-08 2:50 PM 8 | ;; 9 | ;; Hi, when using kaocha in our ci we were surprised by the behavior of 10 | ;; --focus-meta when there are no tests tagged with that metadata, which is to 11 | ;; run all tests. What we would like is to run no tests and succeed, because not 12 | ;; all our projects will have that kind of test. Is there a way to configure 13 | ;; this? 14 | 15 | #_ 16 | (def plan (repl/test-plan)) 17 | 18 | (defn my-post-load-hook [test-plan] 19 | (w/postwalk 20 | (fn [testable] 21 | (if (and (hierarchy/leaf? testable) (not (:really-focus (::testable/meta testable)))) 22 | (assoc testable ::testable/skip true) 23 | testable)) 24 | plan)) 25 | 26 | 27 | (comment 28 | ;; tests.edn: 29 | ;; #kaocha/v1 30 | {:plugins [:hooks] 31 | :kaocha.hooks/post-load [strict-focus-meta-hook/my-post-load-hook]}) 32 | -------------------------------------------------------------------------------- /resources/kaocha/clojure_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lambdaisland/kaocha/525af0545afeab42f9d1e27eb5331fbc5119ca90/resources/kaocha/clojure_logo.png -------------------------------------------------------------------------------- /resources/kaocha/default_config.edn: -------------------------------------------------------------------------------- 1 | {:kaocha/reporter [kaocha.report/dots] 2 | :kaocha/color? true 3 | :kaocha/fail-fast? false 4 | :kaocha/plugins [:kaocha.plugin/randomize 5 | :kaocha.plugin/filter 6 | :kaocha.plugin/capture-output] 7 | :kaocha/tests [{:kaocha.testable/type :kaocha.type/clojure.test 8 | :kaocha.testable/id :unit 9 | :kaocha/ns-patterns ["-test$"] 10 | :kaocha/source-paths ["src"] 11 | :kaocha/test-paths ["test"] 12 | :kaocha.filter/skip-meta [:kaocha/skip]}]} 13 | -------------------------------------------------------------------------------- /src/kaocha/assertions.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.assertions 2 | (:require [kaocha.output :as output] 3 | [clojure.test :as t] 4 | [kaocha.report :as report] 5 | [lambdaisland.deep-diff2.puget.color :as color] 6 | [clojure.string :as str])) 7 | 8 | (defmethod t/assert-expr 'substring? [msg form] 9 | (let [[_ s1 s2] form] 10 | `(if (.contains ~s2 ~s1) 11 | (t/do-report {:type :pass :message ~msg}) 12 | (t/do-report {:type :fail 13 | :expected (list '~'substring? ~s1 ~s2) 14 | :actual (list '~'not '~form) 15 | :message ~msg})))) 16 | 17 | (def x-last 18 | "Reducing version of [[clojure.core/last]]" 19 | (completing (fn [_ x] x))) 20 | 21 | (defn longest-substring [s1 s2] 22 | (transduce (comp (map #(subs s1 0 (inc %))) 23 | (take-while #(.contains s2 %))) 24 | x-last 25 | nil 26 | (range (count s1)))) 27 | 28 | (defn show-trailing-whitespace [s] 29 | (str/replace s 30 | #"(?m)[ \h\x0B\f\r\x85\u2028\u2029]+$" 31 | (fn [s] 32 | (output/colored :red-bg s)))) 33 | 34 | (defmethod report/print-expr 'substring? [{:keys [expected] :as m}] 35 | (let [[_ s1 s2] expected 36 | long-sub (longest-substring s1 s2) 37 | remainder (subs s1 (count long-sub)) 38 | printer (output/printer {:color-scheme {::long-sub [:green] 39 | ::header [:blue]}})] 40 | (output/print-doc 41 | [:span 42 | "Expected: (substring? needle haystack)" 43 | :break 44 | (color/document printer ::header (output/colored :underline "Haystack:")) 45 | :break 46 | (show-trailing-whitespace s2) 47 | :break 48 | (color/document printer ::header (output/colored :underline "Needle:")) 49 | :break 50 | (color/document printer ::long-sub long-sub) 51 | (show-trailing-whitespace remainder)]))) 52 | 53 | #_ 54 | (defmethod t/assert-expr 'thrown+? [msg form] 55 | (let [expr (second form) 56 | body (nthnext form 2)] 57 | `(try+ 58 | ~@body 59 | (t/do-report {:type :fail, :message ~msg, 60 | :expected '~form, :actual nil}) 61 | (catch ~expr e# 62 | (t/do-report {:type :pass, :message ~msg, 63 | :expected '~form, :actual e#}) 64 | e#)))) 65 | 66 | ;; Configured as a pre-load hook 67 | (defn load-assertions [config] 68 | (require 'matcher-combinators.test) 69 | config) 70 | -------------------------------------------------------------------------------- /src/kaocha/classpath.bb: -------------------------------------------------------------------------------- 1 | (ns kaocha.classpath 2 | "On babashka we use bb's version of add-classpath" 3 | (:refer-clojure :exclude [add-classpath]) 4 | (:require [babashka.classpath :as bbcp])) 5 | 6 | (def add-classpath bbcp/add-classpath) 7 | -------------------------------------------------------------------------------- /src/kaocha/classpath.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.classpath 2 | "This is the add-classpath function from Pomegranate 1.0.0, extracted so we 3 | don't need to pull in Aether." 4 | (:refer-clojure :exclude [add-classpath]) 5 | (:require [dynapath.util :as dp] 6 | [clojure.java.io :as io])) 7 | 8 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 9 | ;; Pomegranate 10 | 11 | (defn ensure-compiler-loader 12 | "Ensures the clojure.lang.Compiler/LOADER var is bound to a DynamicClassLoader, 13 | so that we can add to Clojure's classpath dynamically." 14 | [] 15 | (when-not (bound? Compiler/LOADER) 16 | (.bindRoot Compiler/LOADER (clojure.lang.DynamicClassLoader. (clojure.lang.RT/baseLoader))))) 17 | 18 | (defn- classloader-hierarchy 19 | "Returns a seq of classloaders, with the tip of the hierarchy first. 20 | Uses the current thread context ClassLoader as the tip ClassLoader 21 | if one is not provided." 22 | ([] 23 | (ensure-compiler-loader) 24 | (classloader-hierarchy (deref clojure.lang.Compiler/LOADER))) 25 | ([tip] 26 | (->> tip 27 | (iterate #(.getParent ^ClassLoader %)) 28 | (take-while boolean)))) 29 | 30 | (defn- modifiable-classloader? 31 | "Returns true iff the given ClassLoader is of a type that satisfies 32 | the dynapath.dynamic-classpath/DynamicClasspath protocol, and it can 33 | be modified." 34 | [cl] 35 | (dp/addable-classpath? cl)) 36 | 37 | (defn add-classpath 38 | "A corollary to the (deprecated) `add-classpath` in clojure.core. This implementation 39 | requires a java.io.File or String path to a jar file or directory, and will attempt 40 | to add that path to the right classloader (with the search rooted at the current 41 | thread's context classloader)." 42 | ([jar-or-dir classloader] 43 | (if-not (dp/add-classpath-url classloader (.toURL (.toURI (io/file jar-or-dir)))) 44 | (throw (IllegalStateException. (str classloader " is not a modifiable classloader"))))) 45 | ([jar-or-dir] 46 | (let [classloaders (classloader-hierarchy)] 47 | (if-let [cl (filter modifiable-classloader? classloaders)] 48 | ;; Add to all classloaders that allow it. Brute force but doesn't hurt. 49 | (run! #(add-classpath jar-or-dir %) cl) 50 | (throw (IllegalStateException. (str "Could not find a suitable classloader to modify from " 51 | classloaders))))))) 52 | 53 | ;; /Pomegranate 54 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 55 | -------------------------------------------------------------------------------- /src/kaocha/core_ext.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.core-ext 2 | "Core language extensions" 3 | (:require [kaocha.platform :as platform]) 4 | (:refer-clojure :exclude [symbol]) 5 | (:import (java.util.regex Pattern))) 6 | 7 | (defn regex? [x] 8 | (instance? Pattern x)) 9 | 10 | (defn exception? [x] 11 | (instance? java.lang.Exception x)) 12 | 13 | (defn error? [x] 14 | (instance? java.lang.Error x)) 15 | 16 | (defn throwable? [x] 17 | (instance? java.lang.Throwable x)) 18 | 19 | (defn ns? [x] 20 | (instance? clojure.lang.Namespace x)) 21 | 22 | (defn file? [x] 23 | (and (instance? java.io.File x) (.isFile ^java.io.File x))) 24 | 25 | (defn directory? [x] 26 | (and (instance? java.io.File x) (.isDirectory ^java.io.File x))) 27 | 28 | (defn path? [x] 29 | (instance? java.nio.file.Path x)) 30 | 31 | (defn regex 32 | ([x & xs] 33 | (regex (apply str x xs))) 34 | ([x] 35 | (cond 36 | (regex? x) x 37 | (string? x) (Pattern/compile x) 38 | :else (throw (ex-info (str "Can't coerce " (class x) " to regex.") {}))))) 39 | 40 | (defn mapply 41 | "Applies a function f to the argument list formed by concatenating 42 | everything but the last element of args with the last element of 43 | args. This is useful for applying a function that accepts keyword 44 | arguments to a map." 45 | [f & args] 46 | (apply f (apply concat (butlast args) (last args)))) 47 | 48 | (defn symbol 49 | "Backport from Clojure 1.10, symbol function that's a bit more lenient on its 50 | inputs. 51 | 52 | Returns a Symbol with the given namespace and name. Arity-1 works on strings, 53 | keywords, and vars." 54 | ^clojure.lang.Symbol 55 | ([name] 56 | (cond 57 | (symbol? name) name 58 | (instance? String name) (clojure.lang.Symbol/intern name) 59 | (instance? clojure.lang.Var name) (.toSymbol ^clojure.lang.Var name) 60 | (instance? clojure.lang.Keyword name) (.sym ^clojure.lang.Keyword name) 61 | :else (throw (IllegalArgumentException. "no conversion to symbol")))) 62 | ([ns name] 63 | (clojure.core/symbol ns name))) 64 | 65 | ;; 1.10 backport 66 | (when-not (resolve 'clojure.core/requiring-resolve) 67 | ;; using defn generates a warning even when not evaluated 68 | (intern *ns* 69 | ^{:doc "Resolves namespace-qualified sym per 'resolve'. If initial resolve 70 | fails, attempts to require sym's namespace and retries."} 71 | 'requiring-resolve 72 | (fn [sym] 73 | (if (qualified-symbol? sym) 74 | (or (resolve sym) 75 | (do (-> sym namespace symbol require) 76 | (resolve sym))) 77 | (throw (IllegalArgumentException. (str "Not a qualified symbol: " sym))))))) 78 | -------------------------------------------------------------------------------- /src/kaocha/history.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.history 2 | (:require [clojure.test :as t] 3 | [kaocha.hierarchy :as hierarchy])) 4 | 5 | (def ^:dynamic *history* nil) 6 | 7 | (defmulti track :type :hierarchy #'hierarchy/hierarchy) 8 | 9 | (defmethod track :default [m] 10 | (when *history* 11 | (swap! *history* conj m))) 12 | 13 | (defmethod track :kaocha/fail-type [m] 14 | (swap! *history* conj (assoc m 15 | :testing-contexts t/*testing-contexts* 16 | :testing-vars t/*testing-vars*)) ) 17 | 18 | (defmethod track :error [m] 19 | (swap! *history* conj (assoc m 20 | :testing-contexts t/*testing-contexts* 21 | :testing-vars t/*testing-vars*))) 22 | 23 | (defn clojure-test-summary 24 | ([] 25 | (clojure-test-summary @*history*)) 26 | ([history] 27 | (reduce 28 | (fn [m {type :type :as event}] 29 | (cond 30 | (some #{type} [:pass :error :kaocha/pending]) (update m type inc) 31 | (hierarchy/isa? type :kaocha/begin-test) (update m :test inc) 32 | (hierarchy/fail-type? event) (update m :fail inc) 33 | :else m)) 34 | {:type :summary 35 | :test 0 36 | :pass 0 37 | :fail 0 38 | :error 0} 39 | history))) 40 | -------------------------------------------------------------------------------- /src/kaocha/jit.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.jit) 2 | 3 | (defmacro jit 4 | "Just in time loading of dependencies." 5 | [sym] 6 | `(do 7 | (require '~(symbol (namespace sym))) 8 | (find-var '~sym))) 9 | -------------------------------------------------------------------------------- /src/kaocha/load.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.load 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [kaocha.core-ext :refer :all] 4 | [kaocha.testable :as testable] 5 | [clojure.java.io :as io] 6 | [lambdaisland.tools.namespace.find :as ctn-find])) 7 | 8 | (set! *warn-on-reflection* true) 9 | 10 | (def clj ctn-find/clj) 11 | (def cljs ctn-find/cljs) 12 | 13 | (defn ns-match? [ns-patterns ns-sym-or-error] 14 | (or (ctn-find/reader-exception? ns-sym-or-error) 15 | (some #(re-find % (name ns-sym-or-error)) ns-patterns))) 16 | 17 | (defn find-test-nss [test-paths ns-patterns & [platform]] 18 | (sequence (comp 19 | (map io/file) 20 | (mapcat #(ctn-find/find-namespaces-in-dir % platform)) 21 | (filter (partial ns-match? ns-patterns))) 22 | test-paths)) 23 | 24 | (defn load-error-testable [file exception] 25 | {::testable/type :kaocha.type/ns 26 | ::testable/id (keyword (str file)) 27 | ::testable/desc (str "ns form could not be read in " file) 28 | ::testable/load-error exception 29 | ::testable/load-error-file (str file) 30 | ::testable/load-error-line 1 31 | ::testable/load-error-message (str "Failed reading ns form in " file "\n" 32 | "Caused by: " (.getMessage ^Throwable exception)) 33 | :kaocha.ns/name 'kaocha.load-error}) 34 | 35 | (defn load-test-namespaces [testable ns-testable-fn & [platform]] 36 | (let [test-paths (:kaocha/test-paths testable) 37 | ns-patterns (map regex (:kaocha/ns-patterns testable)) 38 | ns-names (find-test-nss test-paths ns-patterns platform) 39 | testables (map (fn [sym-or-error] 40 | (if (ctn-find/reader-exception? sym-or-error) 41 | (let [[_ file exception] sym-or-error] 42 | (load-error-testable file exception)) 43 | (ns-testable-fn sym-or-error))) 44 | ns-names)] 45 | (assoc testable 46 | :kaocha.test-plan/tests 47 | (testable/load-testables testables)))) 48 | -------------------------------------------------------------------------------- /src/kaocha/matcher_combinators.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.matcher-combinators 2 | (:require [kaocha.report :as report] 3 | [kaocha.output :as output] 4 | [clojure.test :as t] 5 | [lambdaisland.deep-diff2 :as ddiff] 6 | [lambdaisland.deep-diff2.printer-impl :as printer] 7 | [lambdaisland.deep-diff2.puget.printer :as puget] 8 | [fipp.engine :as fipp] 9 | [lambdaisland.deep-diff2.puget.color :as color])) 10 | 11 | (def print-handlers {'matcher_combinators.model.Mismatch 12 | (fn [printer expr] 13 | (printer/print-mismatch printer {:- (:expected expr) 14 | :+ (:actual expr)})) 15 | 16 | 'matcher_combinators.model.Missing 17 | (fn [printer expr] 18 | (printer/print-deletion printer {:- (:expected expr)})) 19 | 20 | 'matcher_combinators.model.Unexpected 21 | (fn [printer expr] 22 | (printer/print-insertion printer {:+ (:actual expr)})) 23 | 24 | 'matcher_combinators.model.FailedPredicate 25 | (fn [printer expr] 26 | [:group 27 | [:align 28 | (printer/print-other printer (:form expr)) 29 | (printer/print-insertion printer {:+ (:actual expr)})]]) 30 | 31 | 'matcher_combinators.model.InvalidMatcherType 32 | (fn [printer expr] 33 | [:group 34 | [:align 35 | (color/document printer 36 | ::printer/other 37 | [:span "-" 38 | [:raw (:expected-type-msg expr)]]) 39 | (printer/print-insertion printer {:+ (:provided expr)})]])}) 40 | 41 | (run! #(apply printer/register-print-handler! %) print-handlers) 42 | 43 | (defn fail-summary [{:keys [testing-contexts testing-vars] :as m}] 44 | (let [printer (ddiff/printer {:print-color output/*colored-output*})] 45 | (println (str "\n" (output/colored :red "FAIL") " in") (clojure.test/testing-vars-str m)) 46 | (when (seq t/*testing-contexts*) 47 | (println (t/testing-contexts-str))) 48 | (when-let [message (:message m)] 49 | (println message)) 50 | (fipp/pprint-document 51 | [:span 52 | "Mismatch:" :line 53 | [:nest (puget/format-doc printer (:markup m))]] 54 | {:width (:width printer)}) 55 | (report/print-output m))) 56 | 57 | (defmethod report/fail-summary :mismatch [m] (fail-summary m)) 58 | (defmethod report/fail-summary :matcher-combinators/mismatch [m] (fail-summary m)) 59 | -------------------------------------------------------------------------------- /src/kaocha/ns.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.ns 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.spec.alpha :as spec] 4 | [kaocha.core-ext :refer :all])) 5 | 6 | (defn required-ns [ns-name] 7 | (when-not (and (find-ns ns-name) 8 | (contains? (loaded-libs) (symbol ns-name))) 9 | (require ns-name)) 10 | (try 11 | (the-ns ns-name) 12 | (catch Exception _))) 13 | 14 | (spec/def ::name simple-symbol?) 15 | (spec/def ::ns ns?) 16 | -------------------------------------------------------------------------------- /src/kaocha/output.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.output 2 | (:require [kaocha.jit :refer [jit]] 3 | [slingshot.slingshot :refer [throw+]])) 4 | 5 | (def ^:dynamic *colored-output* true) 6 | 7 | (def ESC \u001b) 8 | 9 | (def colors 10 | {:black (str ESC "[30m") 11 | :red-bg (str ESC "[41m") 12 | :red (str ESC "[31m") 13 | :green (str ESC "[32m") 14 | :yellow (str ESC "[33m") 15 | :blue (str ESC "[34m") 16 | :magenta (str ESC "[35m") 17 | :cyan (str ESC "[36m") 18 | :white (str ESC "[37m") 19 | :underline (str ESC "[4m") 20 | :reset (str ESC "[m")}) 21 | 22 | (defn colored [color string] 23 | (if *colored-output* 24 | (str (get colors color) string (:reset colors)) 25 | string)) 26 | 27 | (defn warn [& args] 28 | (binding [*out* *err*] 29 | (println (apply str (colored :yellow "WARNING: ") args)))) 30 | 31 | (defn error [& args] 32 | (binding [*out* *err*] 33 | (println (apply str (colored :red "ERROR: ") args)))) 34 | 35 | (defn error-and-throw [object cause? & args] 36 | (apply error args) 37 | (throw+ object cause? (apply str args))) 38 | 39 | (defn printer [& [opts]] 40 | ((jit lambdaisland.deep-diff2/printer) (merge {:print-color *colored-output*} opts))) 41 | 42 | (defn print-doc 43 | ([doc] 44 | (print-doc doc (printer))) 45 | ([doc printer] 46 | ((jit fipp.engine/pprint-document) doc {:width (:width printer)}))) 47 | 48 | (defn format-doc 49 | ([doc] 50 | (format-doc doc (printer))) 51 | ([doc printer] 52 | ((jit lambdaisland.deep-diff2.puget.printer/format-doc) printer doc))) 53 | -------------------------------------------------------------------------------- /src/kaocha/platform.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.platform 2 | "Utility functions for specific operating systems") 3 | 4 | (defn on-windows? 5 | "Return whether we're running on Windows." 6 | [] 7 | (re-find #"Windows" (System/getProperty "os.name"))) 8 | 9 | (defn on-posix? 10 | "Return whether we're running on a Posix system." 11 | [] 12 | (re-find #"(?ix)(MacOS|Linux)" (System/getProperty "os.name"))) 13 | -------------------------------------------------------------------------------- /src/kaocha/platform/systray.bb: -------------------------------------------------------------------------------- 1 | (ns kaocha.platform.systray 2 | "Null-implementation for babashka") 3 | 4 | 5 | (defn display-message 6 | [title message urgency] 7 | :unsupported) 8 | -------------------------------------------------------------------------------- /src/kaocha/platform/systray.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.platform.systray 2 | (:import (java.nio.file Files) 3 | (java.io IOException) 4 | (java.awt SystemTray 5 | TrayIcon 6 | TrayIcon$MessageType 7 | Toolkit))) 8 | 9 | (def tray-icon 10 | "Creates a system tray icon." 11 | (memoize 12 | (fn [icon-path] 13 | (let [^java.awt.Toolkit toolkit (Toolkit/getDefaultToolkit) 14 | tray-icon (-> toolkit 15 | (.getImage ^String icon-path) 16 | (TrayIcon. "Kaocha Notification"))] 17 | (doto (SystemTray/getSystemTray) 18 | (.add tray-icon)) 19 | tray-icon)))) 20 | 21 | (defn display-message 22 | "Use Java's built-in functionality to display a notification. 23 | 24 | Not preferred over shelling out because the built-in notification sometimes 25 | looks out of place, and isn't consistently available on Linux." 26 | [title message urgency] 27 | (try 28 | (.displayMessage (tray-icon "kaocha/clojure_logo.png") 29 | title 30 | message 31 | (get {:error TrayIcon$MessageType/ERROR 32 | :info TrayIcon$MessageType/INFO} 33 | urgency)) 34 | :ok 35 | (catch java.awt.HeadlessException _e 36 | :headless) 37 | (catch java.lang.UnsupportedOperationException _e 38 | :unsupported))) 39 | -------------------------------------------------------------------------------- /src/kaocha/plugin/alpha/info.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.alpha.info 2 | (:require [kaocha.api :as api] 3 | [kaocha.plugin :as plugin :refer [defplugin]] 4 | [kaocha.testable :as testable] 5 | [slingshot.slingshot :refer [throw+]])) 6 | 7 | (def cli-opts 8 | [[nil "--print-test-ids" "Print all known test ids"] 9 | [nil "--print-env" "Print Clojure and Java version."]]) 10 | 11 | (defplugin kaocha.plugin.alpha/info 12 | (cli-options [opts] 13 | (into opts cli-opts)) 14 | 15 | (main [config] 16 | (cond 17 | (:print-test-ids (:kaocha/cli-options config)) 18 | (binding [api/*active?* true] 19 | (let [test-plan (api/test-plan (plugin/run-hook :kaocha.hooks/config config))] 20 | (doseq [test (testable/test-seq test-plan)] 21 | (println (:kaocha.testable/id test))) 22 | (throw+ {:kaocha/early-exit 0}))) 23 | 24 | (:print-env (:kaocha/cli-options config)) 25 | (do 26 | (println "Clojure" (clojure-version)) 27 | (println (System/getProperty "java.runtime.name") (System/getProperty "java.runtime.version")) 28 | config) 29 | 30 | :else 31 | config))) 32 | -------------------------------------------------------------------------------- /src/kaocha/plugin/alpha/xfail.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.alpha.xfail 2 | "This plugin inverses fail & pass for tests marked with the ^:kaocha/xfail 3 | metadata. A failing test can be marked with this metadata, and will now be 4 | considered passing. Once the test passes again, it fails. 5 | 6 | Note that this current implementation inverses each assertion in turn, so if 7 | you have multiple assertions in a xfail test, then they all must fail for the 8 | test to pass." 9 | (:require [kaocha.plugin :as plugin] 10 | [kaocha.testable :as testable] 11 | [kaocha.hierarchy :as hierarchy])) 12 | 13 | (plugin/defplugin kaocha.plugin.alpha/xfail 14 | (post-load [test-plan] 15 | (update 16 | test-plan 17 | :kaocha/reporter 18 | (fn [reporter] 19 | (fn [event] 20 | (reporter 21 | (if (-> event :kaocha/testable ::testable/meta :kaocha/xfail) 22 | (cond 23 | (hierarchy/fail-type? event) 24 | (assoc event :type :pass) 25 | 26 | (= (:type event) :pass) 27 | (assoc event :type :fail) 28 | 29 | :else 30 | event) 31 | event))))))) 32 | -------------------------------------------------------------------------------- /src/kaocha/plugin/debug.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.debug 2 | "Plugin that implements all possible plugin hooks and prints out a debug message 3 | with the hook name, and a subset of the first argument passed to the hook. 4 | 5 | For hooks that receive a testable it prints the testable id and type, for 6 | hooks that receive a test event it prints the type, file and line. 7 | 8 | Customize which values are printed with 9 | 10 | ``` clojure 11 | :kaocha/bindings {kaocha.plugin.debug/*keys* [,,,]} 12 | ```" 13 | (:require [kaocha.plugin :as plugin])) 14 | 15 | (def id :kaocha.plugin/debug) 16 | 17 | (def ^:dynamic *keys* [:kaocha.testable/id 18 | :kaocha.testable/type 19 | :type 20 | :file 21 | :line]) 22 | 23 | (defmethod plugin/-register id [_ plugins] 24 | (conj plugins 25 | (into {:kaocha.plugin/id id 26 | :kaocha.plugin/description "Show all hooks that are invoked"} 27 | (map (fn [hook] 28 | [hook (fn [& args] 29 | (print "[DEBUG]" hook "") 30 | (println (cond-> (first args) 31 | (map? (first args)) 32 | (select-keys *keys*))) 33 | (first args))])) 34 | plugin/all-hooks))) 35 | -------------------------------------------------------------------------------- /src/kaocha/plugin/orchestra.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.orchestra 2 | "Instrument/unstrument namespaces with Orchestra, to get validation of function 3 | arguments and return values based on clojure.spec.alpha." 4 | (:require [kaocha.plugin :refer [defplugin]] 5 | [orchestra.spec.test :as orchestra] 6 | [clojure.spec.alpha :as spec])) 7 | 8 | (defplugin kaocha.plugin/orchestra 9 | (post-load [test-plan] 10 | ;; Instrument specs after all of the test namespaces have been loaded 11 | (orchestra/instrument) 12 | test-plan) 13 | 14 | (post-run [result] 15 | ;; Unstrument specs after tests have run. This isn't so important 16 | ;; for CLI testing as the process will exit shortly after the post-run 17 | ;; step, but is helpful for running Kaocha tests from the REPL. 18 | (orchestra/unstrument) 19 | result) 20 | 21 | (pre-report [{:keys [type actual] :as event}] 22 | ;; Render the explain-out string and add it to the clojure.test :error 23 | ;; event's message, since orchestra no longer adds the explain-str to the 24 | ;; exception. 25 | (let [data (and (instance? clojure.lang.ExceptionInfo actual) 26 | (ex-data actual))] 27 | (if (and (= :error type) (:clojure.spec.alpha/problems data)) 28 | (assoc event :kaocha.report/printed-expression 29 | (str (.getMessage actual) "\n" 30 | (with-out-str (spec/explain-out data)))) 31 | event)))) 32 | -------------------------------------------------------------------------------- /src/kaocha/plugin/preloads.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.preloads 2 | "Preload namespaces 3 | 4 | Useful for preloading specs and other instrumentation. 5 | 6 | This plugin calls `require` on the given namespace names before loading any 7 | tests. 8 | 9 | This plugin works for only Clojure namespaces. For ClojureScript namespaces, 10 | use the :preloads functionality of the ClojureScript compiler." 11 | (:require [kaocha.plugin :refer [defplugin]])) 12 | 13 | (defplugin kaocha.plugin/preloads 14 | (pre-load [config] 15 | (when-let [ns-names (::ns-names config)] 16 | (apply require ns-names)) 17 | config)) 18 | -------------------------------------------------------------------------------- /src/kaocha/plugin/print_invocations.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.print-invocations 2 | (:require [kaocha.plugin :as plugin :refer [defplugin]] 3 | [kaocha.testable :as testable] 4 | [kaocha.result :as result] 5 | [clojure.string :as str])) 6 | 7 | 8 | ;; This is an attempt to print out invocations that each re-run a single failing 9 | ;; test. The hard part is recreating a full command line invocation, which might 10 | ;; not be fully feasible. 11 | 12 | (defplugin kaocha.plugin/print-invocations 13 | (post-summary [results] 14 | (when (result/failed? results) 15 | (println) 16 | (doseq [test (testable/test-seq results)] 17 | (if (and (not (seq (::result/tests test))) (result/failed? test)) 18 | (let [id (str (::testable/id test))] 19 | (println 20 | (str/join 21 | " " 22 | (-> ["bin/kaocha"] 23 | (into 24 | (mapcat (fn [[k v]] 25 | (cond 26 | (vector? v) 27 | (mapcat (fn [v] [(str "--" (name k)) v]) v) 28 | 29 | (true? v) 30 | [(str "--" (name k))] 31 | 32 | (false? v) 33 | [(str "--no-" (name k))] 34 | 35 | :else 36 | [(str "--" (name k)) v])) 37 | (cond-> (dissoc (:kaocha/cli-options results) :focus) 38 | (= "tests.edn" (:config-file (:kaocha/cli-options results))) 39 | (dissoc :config-file)))) 40 | (conj "--focus" 41 | (str 42 | "'" (cond-> id (= (first id) \:) (subs 1)) "'"))))))))) 43 | results)) 44 | -------------------------------------------------------------------------------- /src/kaocha/plugin/profiling.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.profiling 2 | (:require [clojure.java.io :as io] 3 | [clojure.spec.alpha :as spec] 4 | [clojure.string :as str] 5 | [kaocha.plugin :as plugin :refer [defplugin]] 6 | [kaocha.testable :as testable]) 7 | (:import java.time.Instant 8 | java.time.temporal.ChronoUnit)) 9 | 10 | (spec/def ::start #(instance? Instant %)) 11 | (spec/def ::duration nat-int?) 12 | (spec/def ::profiling? boolean?) 13 | (spec/def ::count nat-int?) 14 | 15 | (defn start [testable] 16 | (assoc testable ::start (Instant/now))) 17 | 18 | (defn stop [testable] 19 | (cond-> testable 20 | (::start testable) 21 | (assoc ::duration (.until (::start testable) 22 | (Instant/now) 23 | ChronoUnit/NANOS)))) 24 | 25 | (defplugin kaocha.plugin/profiling 26 | (pre-run [test-plan] 27 | (start test-plan)) 28 | 29 | (post-run [test-plan] 30 | (stop test-plan)) 31 | 32 | (pre-test [testable _] 33 | (start testable)) 34 | 35 | (post-test [testable _] 36 | (stop testable)) 37 | 38 | (cli-options [opts] 39 | (conj opts 40 | [nil "--[no-]profiling" "Show slowest tests of each type with timing information."] 41 | [nil "--profiling-count NUM" "Show this many slow tests of each kind in profile results." 42 | :parse-fn #(Integer/parseInt %)])) 43 | 44 | (config [{:kaocha/keys [cli-options] :as config}] 45 | (assoc config 46 | ::profiling? (:profiling cli-options (::profiling? config true)) 47 | ::count (:profiling-count cli-options (::count config 3)))) 48 | 49 | (post-summary [result] 50 | (when (::profiling? result) 51 | (let [tests (->> result 52 | testable/test-seq 53 | (remove ::testable/load-error) 54 | (remove ::testable/skip)) 55 | types (group-by :kaocha.testable/type tests) 56 | total-dur (::duration result) 57 | limit (::count result)] 58 | (->> (for [[type tests] types 59 | :when type 60 | :let [slowest (take limit (reverse (sort-by ::duration tests))) 61 | slow-test-dur (apply + (keep ::duration slowest))]] 62 | [(format "\nTop %s slowest %s (%.5f seconds, %.1f%% of total time)\n" 63 | (count slowest) 64 | (subs (str type) 1) 65 | (float (/ slow-test-dur 1e9)) 66 | (float (* (/ slow-test-dur total-dur) 100))) 67 | (for [test slowest 68 | :let [duration (::duration test) 69 | cnt (count (remove ::testable/skip (:kaocha.result/tests test)))] 70 | :when duration] 71 | (if (> cnt 0) 72 | (format " %s\n \033[1m%.5f seconds\033[0m average (%.5f seconds / %d tests)\n" 73 | (subs (str (:kaocha.testable/id test)) 1) 74 | (float (/ duration cnt 1e9)) 75 | (float (/ duration 1e9)) 76 | cnt) 77 | 78 | (when (:file (:kaocha.testable/meta test)) 79 | (format " %s\n \033[1m%.5f seconds\033[0m %s:%d\n" 80 | (subs (str (:kaocha.testable/id test)) 1) 81 | (float (/ duration 1e9)) 82 | (str/replace (:file (:kaocha.testable/meta test)) 83 | (str (.getCanonicalPath (io/file ".")) "/") 84 | "") 85 | (:line (:kaocha.testable/meta test))))))]) 86 | (flatten) 87 | (apply str) 88 | print) 89 | (flush))) 90 | result)) 91 | -------------------------------------------------------------------------------- /src/kaocha/plugin/randomize.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.randomize 2 | (:require [kaocha.plugin :as plugin :refer [defplugin]] 3 | [kaocha.result :as result]) 4 | (:import (java.util Random))) 5 | 6 | (defn rng [seed] 7 | (let [rng (java.util.Random. seed)] 8 | (fn [& _] (.nextInt rng)))) 9 | 10 | (defn straight-sort [test-plan] 11 | (if-let [tests (:kaocha.test-plan/tests test-plan)] 12 | (assoc test-plan 13 | :kaocha.test-plan/tests 14 | (->> tests 15 | (sort-by :kaocha.testable/id) 16 | (map straight-sort))) 17 | test-plan)) 18 | 19 | (defn rng-sort [rng test-plan] 20 | (if-let [tests (:kaocha.test-plan/tests test-plan)] 21 | (assoc test-plan 22 | :kaocha.test-plan/tests 23 | (->> tests 24 | (map #(assoc % ::sort-key (rng))) 25 | (sort-by ::sort-key) 26 | (map (partial rng-sort rng)))) 27 | test-plan)) 28 | 29 | (defplugin kaocha.plugin/randomize 30 | (cli-options [opts] 31 | (conj opts 32 | [nil "--[no-]randomize" "Run test namespaces and vars in random order."] 33 | [nil "--seed SEED" "Provide a seed to determine the random order of tests." 34 | :parse-fn #(Integer/parseInt %)])) 35 | 36 | (config [config] 37 | (let [randomize? (get-in config [:kaocha/cli-options :randomize]) 38 | seed (get-in config [:kaocha/cli-options :seed]) 39 | config (merge {::randomize? true} 40 | config 41 | (when (some? randomize?) 42 | {::randomize? randomize?}))] 43 | (if (::randomize? config) 44 | (merge {::seed (or seed (rand-int Integer/MAX_VALUE))} config) 45 | config))) 46 | 47 | (post-load [test-plan] 48 | (if (::randomize? test-plan) 49 | (let [rng (rng (::seed test-plan))] 50 | (->> test-plan 51 | straight-sort 52 | (rng-sort rng))) 53 | test-plan)) 54 | 55 | (post-run [test-plan] 56 | (if (and (::randomize? test-plan) (result/failed? test-plan)) 57 | (print "\nRandomized with --seed" (::seed test-plan))) 58 | test-plan)) 59 | -------------------------------------------------------------------------------- /src/kaocha/plugin/version_filter.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.version-filter 2 | "Filter tests based on the Clojure or Java version. 3 | 4 | This plugin will look for test metadata specifying the minimum or maximum 5 | version of Clojure or Java this test is designed to work with. 6 | 7 | The recognized metadata keys are `:min-clojure-version`, 8 | `:max-clojure-version`, `:min-java-version`, and `:max-java-version`. The 9 | associated value is a version string, such as `\"1.10.0\"`. 10 | 11 | You can set both a minimum and a maximum to limit to a certain range. The 12 | boundaries are always inclusive, so `^{:max-clojure-version \"1.9\"}` will run 13 | on Clojure `1.9.*` or earlier. 14 | 15 | Specificty matters, a test with a max version of `\"1.10\" will also run on 16 | version `\"1.10.2\"`, whereas if the max version is `\"1.10.0\"` it will not." 17 | (:require [clojure.string :as str] 18 | [kaocha.plugin :refer [defplugin]] 19 | [kaocha.testable :as testable])) 20 | 21 | (defn version-vector [v] 22 | (let [v (first (str/split v #"\+"))] 23 | ;; if the segment starts with digits then parse those and compare them 24 | ;; numerically, else keep the segment and compare it as a string. 25 | (mapv #(if-let [num (re-find #"^\d+" %)] 26 | (Integer/parseInt num) 27 | %) 28 | (clojure.string/split v #"\.")))) 29 | 30 | (defn java-version [] 31 | (System/getProperty "java.runtime.version")) 32 | 33 | (defn compare-versions [v1 v2] 34 | (let [v1 (version-vector v1) 35 | v2 (version-vector v2) 36 | significance (min (count v1) (count v2))] 37 | (compare (vec (take significance v1)) 38 | (vec (take significance v2))))) 39 | 40 | (defn version>=? [v1 v2] 41 | (if (and v1 v2) 42 | (>= (compare-versions v1 v2) 0) 43 | true)) 44 | 45 | (defn skip? [testable] 46 | (let [{:keys [min-clojure-version 47 | max-clojure-version 48 | min-java-version 49 | max-java-version]} 50 | (::testable/meta testable)] 51 | (not 52 | (and 53 | (version>=? (clojure-version) min-clojure-version) 54 | (version>=? max-clojure-version (clojure-version)) 55 | (version>=? (java-version) min-java-version) 56 | (version>=? max-java-version (java-version)))))) 57 | 58 | (defplugin kaocha.plugin/version-filter 59 | (pre-test [testable test-plan] 60 | (if (skip? testable) 61 | (assoc testable ::testable/skip true) 62 | testable))) 63 | -------------------------------------------------------------------------------- /src/kaocha/random.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.random) 2 | 3 | (defn rng [seed] 4 | (let [rng (java.util.Random. seed)] 5 | (fn [& _] (.nextInt rng)))) 6 | 7 | (defn randomize-tests [seed ns->tests] 8 | (let [next-int (rng seed) 9 | ns->tests' (->> ns->tests 10 | (map (fn [[k v]] [k (sort-by str v)])) 11 | (sort-by first))] 12 | (->> ns->tests' 13 | (map (fn [[k v]] [k (sort-by next-int v)])) 14 | (sort-by next-int)))) 15 | -------------------------------------------------------------------------------- /src/kaocha/report/progress.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.report.progress 2 | (:require [clojure.test :as t] 3 | [progrock.core :as pr] 4 | [kaocha.testable :as testable] 5 | [kaocha.output :as output] 6 | [kaocha.report :as report] 7 | [kaocha.hierarchy :as hierarchy])) 8 | 9 | (def bar (atom nil)) 10 | 11 | (defn color [{:keys [failed? pending?]}] 12 | (cond 13 | failed? :red 14 | pending? :yellow 15 | :else :green)) 16 | 17 | (defn format-bar [{:keys [label label-width] :as bar}] 18 | (str (format (str "%" label-width "s") label) 19 | ": :percent% [" 20 | (output/colored (color bar) ":bar") 21 | "] :progress/:total")) 22 | 23 | (defn print-bar [] 24 | (t/with-test-out 25 | (pr/print @bar {:format (format-bar @bar)}))) 26 | 27 | (defmulti progress :type :hierarchy #'hierarchy/hierarchy) 28 | (defmethod progress :default [_]) 29 | 30 | (defmethod progress :begin-test-suite [m] 31 | (let [testable (:kaocha/testable m) 32 | test-plan (:kaocha/test-plan m) 33 | leaf-tests (->> testable 34 | testable/test-seq 35 | (filter hierarchy/leaf?))] 36 | (reset! bar (assoc (pr/progress-bar (count leaf-tests)) 37 | :label (name (:kaocha.testable/id testable)) 38 | :label-width (->> test-plan 39 | :kaocha.test-plan/tests 40 | (remove :kaocha.testable/skip) 41 | (map (comp count name :kaocha.testable/id)) 42 | (apply max)))) 43 | (print-bar))) 44 | 45 | (defmethod progress :kaocha/end-test [m] 46 | (swap! bar pr/tick) 47 | (print-bar)) 48 | 49 | (defmethod progress :end-test-suite [m] 50 | (swap! bar assoc :done? true) 51 | (print-bar)) 52 | 53 | (defmethod progress :kaocha/fail-type [m] 54 | (swap! bar assoc :failed? true) 55 | (print-bar)) 56 | 57 | (defmethod progress :kaocha/pending [m] 58 | (swap! bar assoc :pending? true) 59 | (print-bar)) 60 | 61 | (defmethod progress :error [m] 62 | (when @bar 63 | (swap! bar assoc :failed? true) 64 | (print-bar))) 65 | 66 | (def report [progress report/result]) 67 | -------------------------------------------------------------------------------- /src/kaocha/result.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.result 2 | (:require [clojure.spec.alpha :as spec])) 3 | 4 | (defn diff-test-result 5 | "Subtract two clojure.test style summary maps." 6 | [before after] 7 | {::pass (apply - (map :pass [after before])) 8 | ::error (apply - (map :error [after before])) 9 | ::fail (apply - (map :fail [after before])) 10 | ::pending (apply - (map :pending [after before]))}) 11 | 12 | (defn sum 13 | "Sum up kaocha result maps." 14 | [& rs] 15 | {::count (apply + (map #(::count % 0) rs)) 16 | ::pass (apply + (map #(::pass % 0) rs)) 17 | ::error (apply + (map #(::error % 0) rs)) 18 | ::fail (apply + (map #(::fail % 0) rs)) 19 | ::pending (apply + (map #(::pending % 0) rs))}) 20 | 21 | (spec/def ::result-map (spec/keys :req [::count ::pass ::error ::fail ::pending])) 22 | 23 | (spec/fdef sum 24 | :args (spec/cat :args (spec/* ::result-map)) 25 | :ret ::result-map) 26 | 27 | (declare testable-totals) 28 | 29 | (defn totals 30 | "Return a map of summed up results for a collection of testables." 31 | [testables] 32 | (apply sum (map testable-totals testables))) 33 | 34 | (defn ^:no-gen testable-totals 35 | "Return a map of summed up results for a testable, including descendants." 36 | [testable] 37 | (if-let [testables (::tests testable)] 38 | (merge testable (totals testables)) 39 | (merge (sum) testable))) 40 | 41 | (spec/fdef testable-totals 42 | :args (spec/cat :testable (spec/or :group (spec/keys :req [:kaocha.result/tests]) 43 | :leaf (spec/keys :opt [::count ::pass ::error ::fail ::pending]))) 44 | :ret ::result-map) 45 | 46 | (defn failed? 47 | "Did this testable, or one of its children, fail or error?" 48 | [testable] 49 | (let [{::keys [error fail]} (testable-totals testable)] 50 | (or (> error 0) (> fail 0)))) 51 | 52 | (defn failed-one? 53 | "Did this testable fail or error, does not recurse." 54 | [{::keys [error fail] :or {error 0 fail 0}}] 55 | (or (> error 0) (> fail 0))) 56 | 57 | (defn totals->clojure-test-summary 58 | "Turn a kaocha-style result map into a clojure.test style summary map." 59 | [totals] 60 | {:type :summary 61 | :test (::count totals) 62 | :pass (::pass totals) 63 | :fail (::fail totals) 64 | :pending (::pending totals) 65 | :error (::error totals)}) 66 | -------------------------------------------------------------------------------- /src/kaocha/shellwords.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.shellwords) 2 | 3 | (def shellwords-pattern #"[^\s'\"]+|[']([^']*)[']|[\"]([^\"]*)[\"]") 4 | 5 | ;; ported from cucumber.runtime.ShellWords, which was ported from Ruby 6 | (defn shellwords [cmdline] 7 | (let [matcher (re-matcher shellwords-pattern cmdline)] 8 | (loop [res []] 9 | (if (.find matcher) 10 | (recur 11 | (if-let [word (.group matcher 1)] 12 | (conj res word) 13 | (let [word (.group matcher)] 14 | (if (and (= \" (first word)) 15 | (= \" (last word))) 16 | (conj res (subs word 1 (dec (count word)))) 17 | (conj res word))))) 18 | res)))) 19 | -------------------------------------------------------------------------------- /src/kaocha/stacktrace.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.stacktrace 2 | (:require [clojure.stacktrace :as st] 3 | [clojure.string :as str])) 4 | 5 | (def ^:dynamic *stacktrace-filters* ["java.lang." 6 | "clojure.test$" 7 | "clojure.lang." 8 | "clojure.core" 9 | "clojure.main" 10 | "kaocha.monkey_patch"]) 11 | 12 | (defn elide-element? [e] 13 | (some #(str/starts-with? (.getClassName ^StackTraceElement e) %) *stacktrace-filters*)) 14 | 15 | (def ^:dynamic *stacktrace-stop-list* ["kaocha.ns" 16 | "lambdaisland.tools.namespace.reload"]) 17 | 18 | (defn sentinel-element? [e] 19 | (some #(str/starts-with? (.getClassName ^StackTraceElement e) %) *stacktrace-stop-list*)) 20 | 21 | (defn print-stack-trace 22 | "Prints a Clojure-oriented stack trace of tr, a Throwable. 23 | Prints a maximum of n stack frames (default: unlimited). Does not print 24 | chained exceptions (causes)." 25 | ([tr] 26 | (print-stack-trace tr nil)) 27 | ([^Throwable tr n] 28 | (let [st (.getStackTrace tr)] 29 | (st/print-throwable tr) 30 | (newline) 31 | (print " at ") 32 | (if-let [e (first st)] 33 | (st/print-trace-element e) ;; always print the first element 34 | (print "[empty stack trace]")) 35 | (newline) 36 | (loop [[e & st] (next st) 37 | eliding? false 38 | n n] 39 | (when e 40 | (let [n (cond-> n n dec)] 41 | (if (= 0 n) 42 | (println " ... and " (count st) "more") 43 | (if (sentinel-element? e) 44 | (println "(Rest of stacktrace elided)") 45 | (if (elide-element? e) 46 | (do 47 | (when (not eliding?) 48 | (println " ...")) 49 | (recur st true n)) 50 | (do 51 | (print " ") 52 | (st/print-trace-element e) 53 | (newline) 54 | (recur st false n))))))))))) 55 | 56 | (defn print-cause-trace 57 | "Like print-stack-trace but prints chained exceptions (causes)." 58 | ([tr] 59 | (print-cause-trace tr nil)) 60 | ([tr n] 61 | (print-stack-trace tr n) 62 | (when-let [cause (.getCause ^Throwable tr)] 63 | (print "Caused by: ") 64 | (recur cause n)))) 65 | -------------------------------------------------------------------------------- /src/kaocha/test_suite.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-suite 2 | (:require [clojure.test :as t] 3 | [kaocha.testable :as testable])) 4 | 5 | (defn run [testable test-plan] 6 | (t/do-report {:type :begin-test-suite}) 7 | (let [results (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) 8 | testable (-> testable 9 | (dissoc :kaocha.test-plan/tests) 10 | (assoc :kaocha.result/tests results))] 11 | (t/do-report {:type :end-test-suite 12 | :kaocha/testable testable}) 13 | testable)) 14 | -------------------------------------------------------------------------------- /src/kaocha/type.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type 2 | "Utilities for writing new test types." 3 | (:require [clojure.test :as t] 4 | [kaocha.result :as result])) 5 | 6 | (def initial-counters {:test 0, :pass 0, :fail 0, :error 0, :pending 0}) 7 | 8 | (def ^:dynamic *intermediate-report*) 9 | 10 | (defmacro with-report-counters 11 | {:style/indent [0]} 12 | [& body] 13 | `(binding [*intermediate-report* (or (some-> t/*report-counters* deref) ~initial-counters)] 14 | (binding [t/*report-counters* (ref *intermediate-report*)] 15 | ~@body))) 16 | 17 | (defn report-count [] 18 | (result/diff-test-result *intermediate-report* @t/*report-counters*)) 19 | -------------------------------------------------------------------------------- /src/kaocha/type/clojure/test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.clojure.test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [kaocha.core-ext :refer :all] 4 | [clojure.spec.alpha :as spec] 5 | [kaocha.type.ns :as type.ns] 6 | [kaocha.testable :as testable] 7 | [kaocha.hierarchy :as hierarchy] 8 | [kaocha.load :as load] 9 | [kaocha.specs] 10 | [kaocha.test-suite :as test-suite])) 11 | 12 | (defmethod testable/-load :kaocha.type/clojure.test [testable] 13 | (-> testable 14 | (load/load-test-namespaces type.ns/->testable) 15 | (testable/add-desc "clojure.test"))) 16 | 17 | (defmethod testable/-run :kaocha.type/clojure.test [testable test-plan] 18 | (test-suite/run testable test-plan)) 19 | 20 | (spec/def :kaocha.type/clojure.test (spec/keys :req [:kaocha/source-paths 21 | :kaocha/test-paths 22 | :kaocha/ns-patterns])) 23 | 24 | (hierarchy/derive! :kaocha.type/clojure.test :kaocha.testable.type/suite) 25 | (hierarchy/derive! :kaocha.type/ns :kaocha.testable.type/group) 26 | (hierarchy/derive! :kaocha.type/var :kaocha.testable.type/leaf) 27 | -------------------------------------------------------------------------------- /src/kaocha/type/ns.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.ns 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.spec.alpha :as spec] 4 | [clojure.test :as t] 5 | [kaocha.core-ext :refer :all] 6 | [kaocha.ns :as ns] 7 | [kaocha.testable :as testable] 8 | [kaocha.type :as type])) 9 | 10 | (defn ->testable [ns-name] 11 | {:kaocha.testable/type :kaocha.type/ns 12 | :kaocha.testable/id (keyword (str ns-name)) 13 | :kaocha.testable/desc (str ns-name) 14 | :kaocha.ns/name ns-name}) 15 | 16 | (defn run-tests [testable test-plan fixture-fn] 17 | ;; It's not guaranteed the the fixture-fn returns the result of calling the 18 | ;; tests function, so we need to put it in a box for reference. 19 | (let [testables (:kaocha.test-plan/tests testable) 20 | result (atom [])] 21 | (fixture-fn #(let [test-result (testable/run-testables testables test-plan)] 22 | (swap! result into test-result))) 23 | @result)) 24 | 25 | (defmethod testable/-load :kaocha.type/ns [testable] 26 | ;; TODO If the namespace has a test-ns-hook function, call that: 27 | ;; if-let [v (find-var (symbol (:kaocha.ns/name testable) "test-ns-hook"))] 28 | 29 | (let [ns-name (:kaocha.ns/name testable) 30 | ns-obj (ns/required-ns ns-name) 31 | ns-meta (meta ns-obj) 32 | each-fixture-fn (t/join-fixtures (::t/each-fixtures ns-meta))] 33 | (assoc testable 34 | :kaocha.testable/meta (meta ns-obj) 35 | :kaocha.ns/ns ns-obj 36 | :kaocha.test-plan/tests 37 | (->> ns-obj 38 | ns-interns 39 | (filter (comp :test meta val)) 40 | (sort-by key) 41 | (map (fn [[sym var]] 42 | (let [nsname (:kaocha.ns/name testable) 43 | test-name (symbol (str nsname) (str sym))] 44 | {:kaocha.testable/type :kaocha.type/var 45 | :kaocha.testable/id (keyword test-name) 46 | :kaocha.testable/meta (meta var) 47 | :kaocha.testable/desc (str sym) 48 | :kaocha.var/name test-name 49 | :kaocha.var/var var 50 | :kaocha.var/test (:test (meta var)) 51 | :kaocha.testable/wrap (if (::t/each-fixtures ns-meta) 52 | [(fn [t] #(each-fixture-fn t))] 53 | [])}))))))) 54 | 55 | (defmethod testable/-run :kaocha.type/ns [testable test-plan] 56 | (let [do-report #(t/do-report (merge {:ns (:kaocha.ns/ns testable)} %))] 57 | (type/with-report-counters 58 | (do-report {:type :begin-test-ns}) 59 | (let [ns-meta (:kaocha.testable/meta testable) 60 | once-fixture-fn (t/join-fixtures (::t/once-fixtures ns-meta)) 61 | tests (run-tests testable test-plan once-fixture-fn) 62 | result (assoc (dissoc testable :kaocha.test-plan/tests) 63 | :kaocha.result/tests 64 | tests)] 65 | (do-report {:type :end-test-ns}) 66 | result)))) 67 | 68 | (spec/def :kaocha.type/ns (spec/keys :req [:kaocha.testable/type 69 | :kaocha.testable/id 70 | :kaocha.ns/name] 71 | :opt [:kaocha.ns/ns 72 | :kaocha.test-plan/tests])) 73 | -------------------------------------------------------------------------------- /src/kaocha/type/spec/test/check.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.spec.test.check 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.spec.alpha :as spec] 4 | [clojure.spec.test.alpha] 5 | [kaocha.core-ext :refer :all] 6 | [kaocha.hierarchy :as hierarchy] 7 | [kaocha.load :as load] 8 | [kaocha.specs] 9 | [kaocha.test-suite :as test-suite] 10 | [kaocha.testable :as testable] 11 | [kaocha.type.spec.test.fdef :as type.fdef] 12 | [kaocha.type.spec.test.ns :as type.spec.ns])) 13 | 14 | ;; This namespace does not actually exist, but is created by 15 | ;; requiring clojure.spec.test.alpha 16 | (alias 'stc 'clojure.spec.test.check) 17 | 18 | (def check-defaults {:kaocha.spec.test.check/ns-patterns [".*"] 19 | :kaocha.spec.test.check/syms :all-fdefs}) 20 | 21 | (defn all-fdef-tests [{:kaocha/keys [source-paths] 22 | :kaocha.spec.test.check/keys [ns-patterns] 23 | :as testable}] 24 | (let [ns-patterns (map regex ns-patterns) 25 | ns-names (load/find-test-nss source-paths ns-patterns) 26 | testables (map #(type.spec.ns/->testable testable %) ns-names)] 27 | (testable/load-testables testables))) 28 | 29 | (defn check-tests [check] 30 | (let [{syms :kaocha.spec.test.check/syms :as check} (merge check-defaults check)] 31 | (condp = syms 32 | :all-fdefs (all-fdef-tests check) 33 | :other-fdefs nil ;; TODO: this requires orchestration from the plugin 34 | ;; else 35 | (type.fdef/load-testables check syms)))) 36 | 37 | (defn checks [{checks :kaocha.spec.test.check/checks :as testable}] 38 | (let [checks (or checks [{}])] 39 | (map #(merge testable %) checks))) 40 | 41 | (defmethod testable/-load :kaocha.type/spec.test.check [testable] 42 | (-> (checks testable) 43 | (->> (map check-tests) 44 | (apply concat) 45 | (assoc testable :kaocha.test-plan/tests)) 46 | (testable/add-desc "clojure.spec.test.check"))) 47 | 48 | (defmethod testable/-run :kaocha.type/spec.test.check [testable test-plan] 49 | (test-suite/run testable test-plan)) 50 | 51 | (spec/def :kaocha.spec.test.check/syms 52 | (spec/or :given-symbols (spec/coll-of symbol?) 53 | :catch-all #{:all-fdefs :other-fdefs})) 54 | 55 | (spec/def :kaocha.spec.test.check/ns-patterns :kaocha/ns-patterns) 56 | 57 | (spec/def :kaocha.spec.test.check/check 58 | (spec/keys :opt [:kaocha.spec.test.check/syms 59 | ::stc/instrument? 60 | ::stc/check-asserts? 61 | ::stc/opts 62 | :kaocha.spec.test.check/ns-patterns])) 63 | 64 | (spec/def :kaocha.spec.test.check/checks (spec/coll-of :kaocha.spec.test.check/check)) 65 | 66 | (spec/def :kaocha.type/spec.test.check 67 | (spec/merge (spec/keys :req [:kaocha.testable/type 68 | :kaocha.testable/id 69 | :kaocha/source-paths] 70 | :opt [:kaocha.filter/skip-meta 71 | :kaocha.spec.test.check/ns-patterns 72 | :kaocha.spec.test.check/checks]) 73 | :kaocha.spec.test.check/check)) 74 | 75 | (hierarchy/derive! :kaocha.type/spec.test.check 76 | :kaocha.testable.type/suite) 77 | -------------------------------------------------------------------------------- /src/kaocha/type/spec/test/ns.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.spec.test.ns 2 | (:require [clojure.spec.alpha :as spec] 3 | [clojure.spec.test.alpha :as stest] 4 | [clojure.spec.test.alpha] 5 | [clojure.test :as t] 6 | [kaocha.hierarchy :as hierarchy] 7 | [kaocha.ns :as ns] 8 | [kaocha.testable :as testable] 9 | [kaocha.type :as type] 10 | [kaocha.type.spec.test.fdef :as type.fdef])) 11 | 12 | ;; This namespace does not actually exist, but is created by 13 | ;; requiring clojure.spec.test.alpha 14 | (alias 'stc 'clojure.spec.test.check) 15 | 16 | (defn ->testable [check ns-name] 17 | (->> {:kaocha.testable/type :kaocha.type/spec.test.ns 18 | :kaocha.testable/id (keyword (str ns-name)) 19 | :kaocha.testable/desc (str ns-name) 20 | :kaocha.ns/name ns-name} 21 | (merge check))) 22 | 23 | (defn starts-with-namespace? [ns-name sym-or-kw] 24 | (-> sym-or-kw namespace (= (str ns-name)))) 25 | 26 | (ns/required-ns 'kaocha.result) 27 | 28 | (stest/checkable-syms) 29 | 30 | (type.fdef/load-testables '[kaocha.result/sum] {}) 31 | 32 | (defmethod testable/-load :kaocha.type/spec.test.ns [testable] 33 | (let [ns-name (:kaocha.ns/name testable) 34 | ns-obj (ns/required-ns ns-name) 35 | tests (->> (stest/checkable-syms) 36 | (filter (partial starts-with-namespace? ns-name)) 37 | (type.fdef/load-testables testable))] 38 | (assoc testable 39 | :kaocha.testable/meta (meta ns-obj) 40 | :kaocha.ns/ns ns-obj 41 | :kaocha.test-plan/tests tests))) 42 | 43 | (defmethod testable/-run :kaocha.type/spec.test.ns [testable test-plan] 44 | (let [do-report #(t/do-report (merge {:ns (:kaocha.ns/ns testable)} %))] 45 | (type/with-report-counters 46 | (do-report {:type :kaocha.stc/begin-ns}) 47 | (let [tests (testable/run-testables (:kaocha.test-plan/tests testable) test-plan) 48 | result (-> testable 49 | (dissoc :kaocha.test-plan/tests) 50 | (assoc :kaocha.result/tests tests))] 51 | (do-report {:type :kaocha.stc/end-ns}) 52 | result)))) 53 | 54 | (spec/def :kaocha.type/spec.test.ns (spec/keys :req [:kaocha.testable/type 55 | :kaocha.testable/id 56 | :kaocha.ns/name] 57 | :opt [:kaocha.ns/ns 58 | :kaocha.test-plan/tests 59 | ::stc/opts])) 60 | 61 | (hierarchy/derive! :kaocha.type/spec.test.ns :kaocha.testable.type/group) 62 | (hierarchy/derive! :kaocha.stc/begin-ns :kaocha/begin-group) 63 | (hierarchy/derive! :kaocha.stc/end-ns :kaocha/end-group) 64 | -------------------------------------------------------------------------------- /src/kaocha/type/var.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.type.var 2 | (:require [clojure.test :as t] 3 | [kaocha.type :as type] 4 | [kaocha.testable :as testable] 5 | [kaocha.result :as result] 6 | [kaocha.report :as report] 7 | [kaocha.hierarchy :as hierarchy] 8 | [clojure.spec.alpha :as spec] 9 | [clojure.spec.gen.alpha :as gen] 10 | [clojure.string :as str]) 11 | (:import (clojure.lang Var))) 12 | 13 | (defmethod report/fail-summary ::zero-assertions [{:keys [testing-contexts testing-vars] :as m}] 14 | (println "\nFAIL in" (report/testing-vars-str m)) 15 | (when (seq testing-contexts) 16 | (println (str/join " " testing-contexts))) 17 | (println "Test ran without assertions. Did you forget an (is ...)?") 18 | (report/print-output m)) 19 | 20 | (defn test-var [test the-var] 21 | (binding [t/*testing-vars* (conj t/*testing-vars* the-var)] 22 | (t/do-report {:type :begin-test-var, :var the-var}) 23 | (try 24 | (test) 25 | (catch clojure.lang.ExceptionInfo e 26 | (when-not (:kaocha/fail-fast (ex-data e)) 27 | (report/report-exception e))) 28 | (catch Throwable e (report/report-exception e))))) 29 | 30 | (defmethod testable/-run :kaocha.type/var [{test :kaocha.var/test 31 | wrap :kaocha.testable/wrap 32 | the-var :kaocha.var/var 33 | meta' :kaocha.testable/meta 34 | :as testable} test-plan] 35 | (type/with-report-counters 36 | (let [wrapped-test (fn [] (test-var test the-var)) 37 | wrapped-test (reduce #(%2 %1) wrapped-test wrap)] 38 | (wrapped-test) 39 | (let [{::result/keys [pass error fail pending] :as result} (type/report-count)] 40 | (when (= pass error fail pending 0) 41 | (binding [testable/*fail-fast?* false 42 | testable/*test-location* {:file (:file meta') :line (:line meta')}] 43 | (t/do-report {:type ::zero-assertions})))) 44 | (t/do-report {:type :end-test-var, :var the-var}) 45 | (merge testable {:kaocha.result/count 1} (type/report-count))))) 46 | 47 | (spec/def :kaocha.type/var (spec/keys :req [:kaocha.testable/type 48 | :kaocha.testable/id 49 | :kaocha.var/name 50 | :kaocha.var/var 51 | :kaocha.var/test])) 52 | 53 | (spec/def :kaocha.var/name qualified-symbol?) 54 | (spec/def :kaocha.var/test (spec/spec ifn? 55 | :gen (fn [] 56 | (gen/one-of [(gen/return (fn [] (t/is true))) 57 | (gen/return (fn [] (t/is false)))])))) 58 | (spec/def :kaocha.var/var (spec/spec var? 59 | :gen (fn [] 60 | (gen/return (.setDynamic (Var/create)))))) 61 | 62 | (hierarchy/derive! :kaocha/begin-var :kaocha/begin-test) 63 | (hierarchy/derive! :kaocha/end-var :kaocha/end-test) 64 | -------------------------------------------------------------------------------- /src/kaocha/util.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.util 2 | (:require [kaocha.platform :as platform])) 3 | 4 | (defn lib-path 5 | "Returns the path for a lib" 6 | ^String [lib ext] 7 | (str 8 | (.. (name lib) 9 | (replace \- \_) 10 | (replace \. \/)) 11 | "." ext)) 12 | 13 | (defn ns-file 14 | "Find the file for a given namespace (symbol), tries to mimic the resolution 15 | logic of [[clojure.core/load]]." 16 | [ns-sym] 17 | (some 18 | #(.getResource (clojure.lang.RT/baseLoader) (lib-path ns-sym %)) 19 | ["class" "cljc" "clj"])) 20 | 21 | (defn compiler-exception-file-and-line 22 | "Try to get the file and line number from a CompilerException" 23 | [^Throwable error] 24 | ;; On Clojure we get a clojure.lang.Compiler$CompilerException, on babashka we 25 | ;; get a clojure.lang.ExceptionInfo. Both implement the IExceptioninfo 26 | ;; interface, so we have a uniform way of getting the location info, although 27 | ;; what Clojure calls `:source` babashka calls `:file`. Calling `ex-data` on 28 | ;; other exceptions will return `nil`. Note that we can't actually test 29 | ;; for `(instance? IExceptioninfo)`, since babashka includes the ExceptionInfo 30 | ;; class, but not the IExceptionInfo interface. Instead we assume that if we 31 | ;; get the right `ex-data` that this is the exception we're looking for. 32 | (let [{:keys [type line file source]} (ex-data error) 33 | file (or source file)] 34 | (if (and type file line) 35 | [file line] 36 | (when-let [error (.getCause error)] 37 | (recur error))))) 38 | 39 | (defn minimal-test-event 40 | "Return a reduced version of a test event map, so debug output doesn't blow up 41 | too much, e.g. in case of deeply nested testables in there." 42 | [m] 43 | (cond-> (select-keys m [:type 44 | :file 45 | :line 46 | :var 47 | :ns 48 | :expected 49 | :actual 50 | :message 51 | :kaocha/testable 52 | :debug 53 | ::printed-expression]) 54 | (:kaocha/testable m) 55 | (update :kaocha/testable select-keys [:kaocha.testable/id :kaocha.testable/type]))) 56 | -------------------------------------------------------------------------------- /src/kaocha/version_check.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.version-check 2 | (:require [kaocha.output :as output] 3 | [slingshot.slingshot :refer [throw+]])) 4 | 5 | (defn check-version-minimum 6 | "Checks that Clojure has at least a minimum version" 7 | [major minor] 8 | (when-not (or (and (= (:major *clojure-version*) major) (>= (:minor *clojure-version*) minor)) 9 | (>= (:major *clojure-version*) (inc major) )) 10 | (let [msg (format "Kaocha requires Clojure %d.%d or later." major minor)] 11 | (output/error msg) 12 | (throw+ {:kaocha/early-exit 251} msg)))) 13 | 14 | (check-version-minimum 1 9) 15 | -------------------------------------------------------------------------------- /test/bb/kaocha/bb_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.bb-test 2 | (:require [clojure.test :as t :refer [deftest is]])) 3 | 4 | (deftest smoke-test 5 | (is (= 1 1))) 6 | -------------------------------------------------------------------------------- /test/bb/kaocha/canary.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.canary 2 | "Babashka script for checking out and running test suites" 3 | (:require [babashka.fs :as fs] 4 | [babashka.process :refer [shell]])) 5 | 6 | (def github-clone-url "https://github.com") 7 | 8 | (def test-suites [{:repo-name "lambdaisland/deep-diff2" 9 | :suite "clj"} 10 | {:repo-name "lambdaisland/regal" 11 | :suite "clj"} 12 | {:repo-name "lambdaisland/uri" 13 | :suite "clj"} 14 | {:repo-name "lambdaisland/kaocha-cucumber" 15 | :suite "unit"} 16 | {:repo-name "lambdaisland/kaocha-doctest" 17 | :suite "unit"} 18 | {:repo-name "lambdaisland/kaocha-cloverage" 19 | :suite ""} 20 | {:repo-name "lambdaisland/kaocha-junit-xml" 21 | :suite ""}]) 22 | 23 | (def current-wd (System/getProperty "user.dir")) 24 | 25 | (let [temp-dir (fs/create-temp-dir)] 26 | (doseq [{:keys [repo-name suite]} test-suites] 27 | (println "Testing " repo-name) 28 | (let [repo-dir (str temp-dir "/" repo-name)] 29 | (shell (format "git clone %s %s" (str github-clone-url "/" repo-name) repo-dir)) 30 | (shell {:dir repo-dir} 31 | (format "clojure -Sdeps '{:aliases {:test-local {:override-deps {lambdaisland/kaocha {:local/root \"%s/\"}}}}}' -A:test:test-local -m kaocha.runner %s" current-wd suite))))) 32 | -------------------------------------------------------------------------------- /test/features/clojure_test/assertions.feature: -------------------------------------------------------------------------------- 1 | Feature: `clojure.test` assertion extensions 2 | 3 | \When running `clojure.test` based tests through Kaocha, some of the behavior 4 | is a little different. Kaocha tries to detect certain scenarios that are 5 | likely mistakes which make a test pass trivially, and turns them into errors 6 | so you can investigate and see what's up. 7 | 8 | Kaocha will also render failures differently, and provides extra multimethods 9 | to influence how certain failures are presented. 10 | 11 | Scenario: Detecting missing assertions 12 | Given a file named "test/sample_test.clj" with: 13 | """ clojure 14 | (ns sample-test 15 | (:require [clojure.test :refer :all])) 16 | 17 | (deftest my-test 18 | (= 4 5)) 19 | """ 20 | When I run `bin/kaocha` 21 | Then the output should contain: 22 | """ text 23 | FAIL in sample-test/my-test (sample_test.clj:4) 24 | Test ran without assertions. Did you forget an (is ...)? 25 | """ 26 | 27 | Scenario: Detecting single argument `=` 28 | Given a file named "test/sample_test.clj" with: 29 | """ clojure 30 | (ns sample-test 31 | (:require [clojure.test :refer :all])) 32 | 33 | (deftest my-test 34 | (is (= 4) 5)) 35 | """ 36 | When I run `bin/kaocha` 37 | Then the output should contain: 38 | """ text 39 | FAIL in sample-test/my-test (sample_test.clj:5) 40 | Equality assertion expects 2 or more values to compare, but only 1 arguments given. 41 | Expected: 42 | (= 4 arg2) 43 | Actual: 44 | (= 4) 45 | 1 tests, 1 assertions, 1 failures. 46 | """ 47 | 48 | Scenario: Pretty printed diffs 49 | Given a file named "test/sample_test.clj" with: 50 | """ clojure 51 | (ns sample-test 52 | (:require [clojure.test :refer :all])) 53 | 54 | (defn my-fn [] 55 | {:xxx [1 2 3] 56 | :blue :red 57 | "hello" {:world :!}}) 58 | 59 | (deftest my-test 60 | (is (= {:xxx [1 3 4] 61 | "hello" {:world :?}} 62 | {:xxx [1 2 3] 63 | :blue :red 64 | "hello" {:world :!}}))) 65 | """ 66 | When I run `bin/kaocha` 67 | Then the output should contain: 68 | """ text 69 | FAIL in sample-test/my-test (sample_test.clj:10) 70 | Expected: 71 | {"hello" {:world :?}, :xxx [1 3 4]} 72 | Actual: 73 | {"hello" {:world -:? +:!}, :xxx [1 +2 3 -4], +:blue :red} 74 | 1 tests, 1 assertions, 1 failures. 75 | """ 76 | -------------------------------------------------------------------------------- /test/features/command_line/capability_check.feature: -------------------------------------------------------------------------------- 1 | Feature: Capability check for org.clojure/tools.cli 2 | 3 | If a project's dependency pulls in an old version of tools.cli, then this may 4 | break command line flags of the form `--[no-]xxx`. Before starting the main 5 | command line runner, Kaocha verifies that tools.cli has the necessary 6 | capabilities. 7 | 8 | Scenario: With an outdated tools.cli 9 | When I run `clojure -Sdeps '{:deps {org.clojure/tools.cli {:mvn/version "0.3.5"}}}' --main kaocha.runner` 10 | Then stderr should contain: 11 | """ 12 | org.clojure/tools.cli does not have all the capabilities that Kaocha needs. Make sure you are using version 0.3.6 or greater. 13 | """ 14 | -------------------------------------------------------------------------------- /test/features/command_line/fail_fast.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: `--fail-fast` option 2 | 3 | Kaocha by default runs all tests it can find, providing a final summary on 4 | failures and errors when all tests have finished. With the `--fail-fast` 5 | option the test run will be interrupted as soon as a single failure or error 6 | has occured. Afterwards a summary of the test run so far is printed. 7 | 8 | Scenario: Failing fast 9 | Given a file named "test/my/project/fail_fast_test.clj" with: 10 | """clojure 11 | (ns my.project.fail-fast-test 12 | (:require [clojure.test :refer :all])) 13 | 14 | (deftest test-1 15 | (is true)) 16 | 17 | (deftest test-2 18 | (is true) 19 | (is false) 20 | (is true)) 21 | 22 | (deftest test-3 23 | (is true)) 24 | """ 25 | When I run `bin/kaocha --fail-fast` 26 | Then the exit-code should be 1 27 | And the output should contain: 28 | """ 29 | [(..F)] 30 | 31 | FAIL in my.project.fail-fast-test/test-2 (fail_fast_test.clj:9) 32 | expected: false 33 | actual: false 34 | 2 tests, 3 assertions, 1 failures. 35 | """ 36 | -------------------------------------------------------------------------------- /test/features/command_line/print_config.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: Print the Kaocha configuration 2 | 3 | A Kaocha test run starts with building up a Kaocha configuration map, based on 4 | default values, the contents of `tests.edn`, command line flags, and active 5 | plugins. 6 | 7 | Debugging issues with Kaocha often starts with inspecting the configuration, 8 | which is why a `--print-config` flag is provided. This builds up the 9 | configuration from any available sources, runs it through any active plugins, 10 | and then pretty prints the result, an EDN map. 11 | 12 | Note that the ordering, while not expected to change, is not guaranteed. We 13 | recommend parsing the configuration as EDN and not relying on order. If you 14 | are manipulating the output as text (say, on the command line) and can't 15 | avoid relying on the order, run it through a tool like 16 | [puget](https://github.com/greglook/puget) or 17 | [zprint](https://github.com/kkinnear/zprint) that sorts the keys 18 | alphabetically first. 19 | 20 | Scenario: Using `--print-config` 21 | When I run `bin/kaocha --print-config` 22 | Then the EDN output should contain: 23 | """ clojure 24 | {:kaocha.plugin.randomize/randomize? false, 25 | :kaocha/reporter [kaocha.report/dots], 26 | :kaocha/color? false, 27 | :kaocha/fail-fast? false} 28 | """ 29 | And the EDN output should contain: 30 | """ clojure 31 | {:kaocha/tests 32 | [{:kaocha.testable/type :kaocha.type/clojure.test, 33 | :kaocha.testable/id :unit, 34 | :kaocha/ns-patterns ["-test$"], 35 | :kaocha/source-paths ["src"], 36 | :kaocha/test-paths ["test"], 37 | :kaocha.filter/skip-meta [:kaocha/skip]}]} 38 | """ 39 | -------------------------------------------------------------------------------- /test/features/command_line/profile.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: `--profile` option 2 | 3 | The `--profile KEYWORD` flags sets the profile that is used to read the 4 | `tests.edn` configuration file. By using the `#profile {}` tagged reader 5 | literal you can provide different configuration values for different 6 | scenarios. 7 | 8 | If the `CI` environment value is set to `"true"`, as is the case on most CI 9 | platforms, then the profile will default to `:ci`. Otherwise it defaults to 10 | `:default`. 11 | 12 | Scenario: Specifying profile on the command line 13 | Given a file named "tests.edn" with: 14 | """ clojure 15 | #kaocha/v1 16 | {:reporter #profile {:ci kaocha.report/documentation 17 | :default kaocha.report/dots}} 18 | """ 19 | And a file named "test/my/project/my_test.clj" with: 20 | """clojure 21 | (ns my.project.my-test 22 | (:require [clojure.test :refer :all])) 23 | 24 | (deftest test-1 25 | (is true)) 26 | """ 27 | When I run `bin/kaocha --profile :ci` 28 | And the output should contain: 29 | """ 30 | --- unit (clojure.test) --------------------------- 31 | my.project.my-test 32 | test-1 33 | """ 34 | -------------------------------------------------------------------------------- /test/features/command_line/reporter.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: `--reporter` option 2 | 3 | The progress and summary printed by Kaocha are done by one or more "reporter" 4 | functions. A reporter can be specified with the `--reporter` option followed 5 | by a fully qualified function name. 6 | 7 | Reporters in the `kaocha.report` namespace can be specified without a 8 | namespace prefix. 9 | 10 | See the 11 | [`kaocha.report`](https://github.com/lambdaisland/kaocha/blob/master/src/kaocha/report.clj) 12 | namespace for built-in reporters. 13 | 14 | Background: An example test 15 | Given a file named "test/my/project/reporter_test.clj" with: 16 | """clojure 17 | (ns my.project.reporter-test 18 | (:require [clojure.test :refer :all])) 19 | 20 | (deftest test-1 21 | (is (= 1 0))) 22 | 23 | (deftest test-2 24 | (is true) 25 | (is (throw (Exception. ""))) 26 | (is true)) 27 | 28 | (deftest test-3 29 | (is true)) 30 | """ 31 | 32 | Scenario: Using a fully qualified function as a reporter 33 | When I run `bin/kaocha --reporter kaocha.report/documentation` 34 | And the output should contain: 35 | """ 36 | my.project.reporter-test 37 | test-1 FAIL 38 | test-2 ERROR 39 | test-3 40 | """ 41 | 42 | Scenario: Specifying a reporter via shorthand 43 | When I run `bin/kaocha --reporter documentation` 44 | Then the exit-code should be 2 45 | And the output should contain: 46 | """ 47 | my.project.reporter-test 48 | test-1 FAIL 49 | test-2 ERROR 50 | test-3 51 | """ 52 | 53 | Scenario: Using a reporter which does not exist 54 | When I run `bin/kaocha --reporter does/not-exist` 55 | Then stderr should contain 56 | """ 57 | ERROR: Failed to resolve reporter var: does/not-exist 58 | """ 59 | -------------------------------------------------------------------------------- /test/features/command_line/suite_names.feature: -------------------------------------------------------------------------------- 1 | Feature: CLI: Selecting test suites 2 | 3 | Each test suite has a unique id, given as a keyword in the test configuration. 4 | You can supply one or more of these ids on the command line to run only those 5 | test suites. 6 | 7 | Background: Given two test suites, `:aaa` and `:bbb` 8 | Given a file named "tests.edn" with: 9 | """clojure 10 | #kaocha/v1 11 | {:tests [{:id :aaa 12 | :test-paths ["tests/aaa"]} 13 | {:id :bbb 14 | :test-paths ["tests/bbb"]}]} 15 | """ 16 | 17 | And a file named "tests/aaa/aaa_test.clj" with: 18 | """clojure 19 | (ns aaa-test (:require [clojure.test :refer :all])) 20 | (deftest foo-test (is true)) 21 | """ 22 | 23 | And a file named "tests/bbb/bbb_test.clj" with: 24 | """clojure 25 | (ns bbb-test (:require [clojure.test :refer :all])) 26 | (deftest bbb-test (is true)) 27 | """ 28 | 29 | Scenario: Specifying a test suite on the command line 30 | When I run `bin/kaocha aaa --reporter documentation` 31 | And the output should contain: 32 | """ 33 | aaa-test 34 | foo-test 35 | """ 36 | And the output should not contain 37 | """ 38 | bbb-test 39 | """ 40 | 41 | Scenario: Specifying a test suite using keyword syntax 42 | When I run `bin/kaocha :aaa --reporter documentation` 43 | And the output should contain: 44 | """ 45 | aaa-test 46 | foo-test 47 | """ 48 | And the output should not contain 49 | """ 50 | bbb-test 51 | """ 52 | 53 | Scenario: Specifying an unknown suite 54 | When I run `bin/kaocha suite-name` 55 | Then the output should contain: 56 | """ 57 | No such suite: :suite-name, valid options: :aaa, :bbb. 58 | """ 59 | -------------------------------------------------------------------------------- /test/features/config/warnings.feature: -------------------------------------------------------------------------------- 1 | 2 | Feature: Configuration: Warnings 3 | 4 | Kaocha will warn about common mistakes. 5 | 6 | 7 | Scenario: No config 8 | Given a file named "test/my/foo_test.clj" with: 9 | """ clojure 10 | (ns my.foo-test 11 | (:require [clojure.test :refer :all])) 12 | 13 | (deftest var-test 14 | (is (= 456 456))) 15 | """ 16 | When I run `bin/kaocha -c alt-tests.edn` 17 | Then stderr should contain: 18 | """ 19 | Did not load a configuration file and using the defaults. 20 | """ 21 | Scenario: Warn about bad configuration 22 | Given a file named "tests.edn" with: 23 | """ clojure 24 | #kaocha/v1 25 | {:plugins notifier} 26 | """ 27 | And a file named "test/my/foo_test.clj" with: 28 | """ clojure 29 | (ns my.foo-test 30 | (:require [clojure.test :refer :all])) 31 | 32 | (deftest var-test 33 | (is (= 456 456))) 34 | """ 35 | When I run `bin/kaocha` 36 | Then stderr should contain: 37 | """ 38 | Invalid configuration file: 39 | """ 40 | -------------------------------------------------------------------------------- /test/features/filtering/focus_meta.feature: -------------------------------------------------------------------------------- 1 | Feature: Focusing based on metadata 2 | 3 | You can limit the test run based on test's metadata. How to associate metadata 4 | with a test depends on the test type, for `clojure.test` type tests metadata 5 | can be associated with a test var or test namespace. 6 | 7 | Using the `--focus-meta` command line flag, or `:kaocha.filter/focus-meta` key 8 | in test suite configuration, you can limit the tests being run to only those 9 | where the given metadata key has a truthy value associated with it. 10 | 11 | Background: Some tests with metadata 12 | Given a file named "test/my/project/sample_test.clj" with: 13 | """clojure 14 | (ns ^:xxx my.project.sample-test 15 | (:require [clojure.test :refer :all])) 16 | 17 | (deftest some-test 18 | (is (= 1 1))) 19 | 20 | (deftest other-test 21 | (is (= 2 2))) 22 | """ 23 | And a file named "test/my/project/other_sample_test.clj" with: 24 | """clojure 25 | (ns my.project.other-sample-test 26 | (:require [clojure.test :refer :all])) 27 | 28 | (deftest ^:yyy other-test 29 | (is (= 3 3))) 30 | """ 31 | 32 | Scenario: Focusing by metadata from the command line 33 | When I run `bin/kaocha --focus-meta :xxx --reporter documentation` 34 | Then the output should contain: 35 | """ 36 | --- unit (clojure.test) --------------------------- 37 | my.project.sample-test 38 | other-test 39 | some-test 40 | 41 | 2 tests, 2 assertions, 0 failures. 42 | """ 43 | 44 | Scenario: Focusing on a test group by metadata from the command line 45 | When I run `bin/kaocha --focus-meta :yyy --reporter documentation` 46 | Then the output should contain: 47 | """ 48 | --- unit (clojure.test) --------------------------- 49 | my.project.other-sample-test 50 | other-test 51 | 52 | 1 tests, 1 assertions, 0 failures. 53 | """ 54 | 55 | Scenario: Focusing based on metadata via configuration 56 | Given a file named "tests.edn" with: 57 | """ edn 58 | #kaocha/v1 59 | {:tests [{:kaocha.filter/focus-meta [:yyy]}] 60 | :color? false 61 | :randomize? false} 62 | """ 63 | When I run `bin/kaocha --reporter documentation` 64 | Then the output should contain: 65 | """ 66 | --- unit (clojure.test) --------------------------- 67 | my.project.other-sample-test 68 | other-test 69 | 70 | 1 tests, 1 assertions, 0 failures. 71 | """ 72 | -------------------------------------------------------------------------------- /test/features/filtering/focusing.feature: -------------------------------------------------------------------------------- 1 | Feature: Focusing on specific tests 2 | 3 | You can limit the test run to only specific tests or test groups (e.g. 4 | namespaces) using the `--focus` command line flag, or `:kaocha.filter/focus` 5 | key in test suite configuration. 6 | 7 | Background: A simple test suite 8 | Given a file named "test/my/project/sample_test.clj" with: 9 | """clojure 10 | (ns my.project.sample-test 11 | (:require [clojure.test :refer :all])) 12 | 13 | (deftest some-test 14 | (is (= 1 1))) 15 | 16 | (deftest other-test 17 | (is (= 2 2))) 18 | """ 19 | And a file named "test/my/project/other_sample_test.clj" with: 20 | """clojure 21 | (ns my.project.other-sample-test 22 | (:require [clojure.test :refer :all])) 23 | 24 | (deftest other-test 25 | (is (= 1 2))) 26 | """ 27 | 28 | Scenario: Focusing on test id from the command line 29 | When I run `bin/kaocha --focus my.project.sample-test/some-test --reporter documentation` 30 | Then the output should contain: 31 | """ 32 | --- unit (clojure.test) --------------------------- 33 | my.project.sample-test 34 | some-test 35 | 36 | 1 tests, 1 assertions, 0 failures. 37 | """ 38 | 39 | Scenario: Focusing on test group id from the command line 40 | When I run `bin/kaocha --focus my.project.sample-test --reporter documentation` 41 | Then the output should contain: 42 | """ 43 | --- unit (clojure.test) --------------------------- 44 | my.project.sample-test 45 | other-test 46 | some-test 47 | 48 | 2 tests, 2 assertions, 0 failures. 49 | """ 50 | 51 | Scenario: Focusing via configuration 52 | Given a file named "tests.edn" with: 53 | """ edn 54 | #kaocha/v1 55 | {:tests [{:kaocha.filter/focus [my.project.sample-test/other-test]}] 56 | :color? false 57 | :randomize? false} 58 | """ 59 | When I run `bin/kaocha --reporter documentation` 60 | Then the output should contain: 61 | """ 62 | --- unit (clojure.test) --------------------------- 63 | my.project.sample-test 64 | other-test 65 | 66 | 1 tests, 1 assertions, 0 failures. 67 | """ 68 | -------------------------------------------------------------------------------- /test/features/filtering/skipping.feature: -------------------------------------------------------------------------------- 1 | Feature: Skipping test based on ids 2 | 3 | You can tell Kaocha to completely ignore certain tests or test groups, either 4 | with the `--skip` command line flag, or the `:kaocha.filter/skip` test suite 5 | configuration key. 6 | 7 | Both of these take test ids or test group ids (e.g. the fully qualified name 8 | of a test var, or the name of a test namespace). 9 | 10 | Background: A simple test suite 11 | Given a file named "test/my/project/sample_test.clj" with: 12 | """clojure 13 | (ns my.project.sample-test 14 | (:require [clojure.test :refer :all])) 15 | 16 | (deftest some-test 17 | (is (= 1 1))) 18 | 19 | (deftest other-test 20 | (is (= 2 2))) 21 | """ 22 | And a file named "test/my/project/other_sample_test.clj" with: 23 | """clojure 24 | (ns my.project.other-sample-test 25 | (:require [clojure.test :refer :all])) 26 | 27 | (deftest other-test 28 | (is (= 3 3))) 29 | """ 30 | 31 | Scenario: Skipping test id from the command line 32 | When I run `bin/kaocha --skip my.project.sample-test/some-test --reporter documentation` 33 | Then the output should contain: 34 | """ 35 | --- unit (clojure.test) --------------------------- 36 | my.project.other-sample-test 37 | other-test 38 | 39 | my.project.sample-test 40 | other-test 41 | 42 | 2 tests, 2 assertions, 0 failures. 43 | """ 44 | 45 | Scenario: Skipping a test group id from the command line 46 | When I run `bin/kaocha --skip my.project.sample-test --reporter documentation` 47 | Then the output should contain: 48 | """ 49 | --- unit (clojure.test) --------------------------- 50 | my.project.other-sample-test 51 | other-test 52 | 53 | 1 tests, 1 assertions, 0 failures. 54 | """ 55 | 56 | Scenario: Skipping via configuration 57 | Given a file named "tests.edn" with: 58 | """ edn 59 | #kaocha/v1 60 | {:tests [{:kaocha.filter/skip [my.project.sample-test]}] 61 | :color? false 62 | :randomize? false} 63 | """ 64 | When I run `bin/kaocha --reporter documentation` 65 | Then the output should contain: 66 | """ 67 | --- unit (clojure.test) --------------------------- 68 | my.project.other-sample-test 69 | other-test 70 | 71 | 1 tests, 1 assertions, 0 failures. 72 | """ 73 | -------------------------------------------------------------------------------- /test/features/pending.feature: -------------------------------------------------------------------------------- 1 | Feature: Marking tests as pending 2 | 3 | Pending tests are tests that are not yet implemented, or that need fixing, and 4 | that you don't want to forget about. Pending tests are similar to skipped 5 | tests (see the section on "Filtering"), in that the runner will skip over them 6 | without trying to run them. 7 | 8 | The difference is that pending tests are explicitly reported in the test 9 | result. At the end of each test run you get to see the number of pending 10 | tests, followed by a list of their test ids and file/line information. This 11 | constant reminder is there to make sure pending tests are not left 12 | unaddressed. 13 | 14 | Add the `^:kaocha/pending` metadata to a test to mark it as pending. The 15 | metadata check is done inside the Kaocha runner itself, not in the specific 16 | test type implementation, so this metadata is supported on any test type that 17 | allows setting metadata tags, including ClojureScript and Cucumber tests. 18 | 19 | Scenario: Marking a test as pending 20 | Given a file named "test/sample/sample_test.clj" with: 21 | """clojure 22 | (ns sample.sample-test 23 | (:require [clojure.test :refer :all])) 24 | 25 | (deftest ^:kaocha/pending my-test) 26 | """ 27 | When I run `bin/kaocha` 28 | Then the output should contain: 29 | """ 30 | [(P)] 31 | 1 tests, 0 assertions, 1 pending, 0 failures. 32 | 33 | PENDING sample.sample-test/my-test (sample/sample_test.clj:4) 34 | """ 35 | -------------------------------------------------------------------------------- /test/features/plugins/capture_output.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Capture output 2 | 3 | Kaocha has a plugin which will capture all output written to stdout or stderr 4 | during the test run. When tests pass this output is hidden, when they fail the 5 | output is made visible to help understand the problem. 6 | 7 | This plugin is loaded by default, but can be disabled with `--no-capture-output` 8 | 9 | Scenario: Show output of failing test 10 | Given a file named "test/sample_test.clj" with: 11 | """ clojure 12 | (ns sample-test 13 | (:require [clojure.test :refer :all])) 14 | 15 | (deftest stdout-pass-test 16 | (println "You peng zi yuan fang lai") 17 | (is (= :same :same))) 18 | 19 | (deftest stdout-fail-test 20 | (println "Bu yi le hu?") 21 | (is (= :same :not-same))) 22 | """ 23 | When I run `bin/kaocha` 24 | Then the output should contain: 25 | """ 26 | FAIL in sample-test/stdout-fail-test (sample_test.clj:10) 27 | Expected: 28 | :same 29 | Actual: 30 | -:same +:not-same 31 | ╭───── Test output ─────────────────────────────────────────────────────── 32 | │ Bu yi le hu? 33 | ╰───────────────────────────────────────────────────────────────────────── 34 | 2 tests, 2 assertions, 1 failures. 35 | """ 36 | 37 | Scenario: Bypass output capturing 38 | The `kaocha.plugin.capture-output/bypass` macro can be used to force output 39 | to STDOUT/STDERR. 40 | 41 | Given a file named "test/sample_test.clj" with: 42 | """ clojure 43 | (ns sample-test 44 | (:require [clojure.test :refer :all] 45 | [kaocha.plugin.capture-output :as capture])) 46 | 47 | (deftest stdout-pass-test 48 | (capture/bypass 49 | (println "You peng zi yuan fang lai")) 50 | (is (= :same :same))) 51 | """ 52 | When I run `bin/kaocha` 53 | Then the output should contain: 54 | """ 55 | [(You peng zi yuan fang lai 56 | .)] 57 | 1 tests, 1 assertions, 0 failures. 58 | """ -------------------------------------------------------------------------------- /test/features/plugins/hooks_plugin.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Hooks 2 | 3 | The hooks plugin allows hooking into Kaocha's process with arbitrary 4 | functions. This is very similar to using writing a plugin, but requires less 5 | boilerplate. 6 | 7 | See the documentation for extending Kaocha for a description of the different 8 | hooks. The supported hooks are: config, pre-load, post-load, pre-run, post-run, wrap-run, 9 | pre-test, post-test, pre-report, post-summary. 10 | 11 | The hooks plugin also provides hooks at the test suite level, which in order 12 | are `:kaocha.hooks/pre-load-test`, `:kaocha.hooks/post-load-test`, 13 | `:kaocha.hooks/pre-test`, `:kaocha.hooks/post-test`. 14 | 15 | Hooks can be specified as a fully qualified symbol referencing a function, or 16 | a collection thereof. The referenced namespaces will be loaded during the 17 | config phase. It's also possible to have functions directly inline, but due to 18 | limitations of the EDN reader this is of limited use, and you are generally 19 | better off sticking your hooks into a proper namespace. 20 | 21 | Scenario: Implementing a hook 22 | Given a file named "tests.edn" with: 23 | """ clojure 24 | #kaocha/v1 25 | {:plugins [:kaocha.plugin/hooks] 26 | :kaocha.hooks/pre-test [my.kaocha.hooks/sample-hook]} 27 | """ 28 | And a file named "src/my/kaocha/hooks.clj" with: 29 | """ clojure 30 | (ns my.kaocha.hooks) 31 | 32 | (println "ok") 33 | 34 | (defn sample-hook [test test-plan] 35 | (if (re-find #"fail" (str (:kaocha.testable/id test))) 36 | (assoc test :kaocha.testable/pending true) 37 | test)) 38 | """ 39 | And a file named "test/sample_test.clj" with: 40 | """ clojure 41 | (ns sample-test 42 | (:require [clojure.test :refer :all])) 43 | 44 | (deftest stdout-pass-test 45 | (println "You peng zi yuan fang lai") 46 | (is (= :same :same))) 47 | 48 | (deftest stdout-fail-test 49 | (println "Bu yi le hu?") 50 | (is (= :same :not-same))) 51 | """ 52 | When I run `bin/kaocha` 53 | Then the output should contain: 54 | """ 55 | PENDING sample-test/stdout-fail-test (sample_test.clj:8) 56 | """ 57 | 58 | Scenario: Implementing a test-suite specific hook 59 | Given a file named "tests.edn" with: 60 | """ clojure 61 | #kaocha/v1 62 | {:plugins [:kaocha.plugin/hooks] 63 | :tests [{:id :unit 64 | :kaocha.hooks/before [my.kaocha.hooks/sample-before-hook] 65 | :kaocha.hooks/after [my.kaocha.hooks/sample-after-hook]}]} 66 | """ 67 | And a file named "src/my/kaocha/hooks.clj" with: 68 | """ clojure 69 | (ns my.kaocha.hooks) 70 | 71 | (defn sample-before-hook [suite test-plan] 72 | (println "before suite:" (:kaocha.testable/id suite)) 73 | suite) 74 | 75 | (defn sample-after-hook [suite test-plan] 76 | (println "after suite:" (:kaocha.testable/id suite)) 77 | suite) 78 | """ 79 | And a file named "test/sample_test.clj" with: 80 | """ clojure 81 | (ns sample-test 82 | (:require [clojure.test :refer :all])) 83 | 84 | (deftest stdout-pass-test 85 | (println "You peng zi yuan fang lai") 86 | (is (= :same :same))) 87 | """ 88 | When I run `bin/kaocha` 89 | Then the output should contain: 90 | """ 91 | before suite: :unit 92 | [(.)]after suite: :unit 93 | """ 94 | -------------------------------------------------------------------------------- /test/features/plugins/notifier_plugin.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Notifier (desktop notifications) 2 | 3 | Desktop notifications can be enabled with the `:kaocha.plugin/notifier` 4 | plugin. This will pop up a fail/pass notification bubble including a summary 5 | of tests passed/errored/failed at the end of each test run. It's particularly 6 | useful in combination with `--watch`, e.g. `bin/kaocha --plugin notifier 7 | --watch`. 8 | 9 | It does this by invoking a shell command which can be configured, so it can be 10 | used to invoke an arbitrary command or script. By default it will try to 11 | detect which command to use, using either `notify-send` (Linux) or 12 | `terminal-notifier` (Mac OS X), either of which may need to be installed 13 | first. 14 | 15 | Several replacement patterns are available: 16 | 17 | - `%{title}` : The notification title, either `⛔️ Failing` or `✅ Passing` 18 | - `%{message}` : Test result summary, e.g. `5 tests, 12 assertions, 0 failures` 19 | - `%{icon}` : Full local path to an icon to use (currently uses the Clojure icon) 20 | - `%{failed?}` : `true` if any tests failed or errored, `false` otherwise 21 | - `%{count}` : the number of tests 22 | - `%{pass}` : the number of passing assertions 23 | - `%{fail}` : the number of failing assertions 24 | - `%{error}` : the number of errors 25 | - `%{pending}` : the number of pending tests 26 | - `%{urgency}` : `normal` if the tests pass, `critical` otherwise, meant for use with `notify-send` 27 | 28 | If no command is configured, and neither notification command is found, then 29 | the plugin will silently do nothing. You can explicitly inhibit its behaviour 30 | with `--no-notifications`. 31 | 32 | Scenario: Enabling Desktop Notifications 33 | Given a file named "tests.edn" with: 34 | """ clojure 35 | #kaocha/v1 36 | {:plugins [:kaocha.plugin/notifier] 37 | 38 | ;; Configuring a command is optional. Since CI does not have desktop 39 | ;; notifications we pipe to a file instead. 40 | :kaocha.plugin.notifier/command 41 | "sh -c 'echo \"%{title}\n%{message}\n%{failed?}\n%{count}\n%{urgency}\" > /tmp/kaocha.txt'" 42 | 43 | ;; Fallbacks: 44 | 45 | ;; :kaocha.plugin.notifier/command 46 | ;; "notify-send -a Kaocha %{title} %{message} -i %{icon} -u %{urgency}" 47 | 48 | ;; :kaocha.plugin.notifier/command 49 | ;; "terminal-notifier -message %{message} -title %{title} -appIcon %{icon}" 50 | } 51 | """ 52 | And a file named "test/sample_test.clj" with: 53 | """ clojure 54 | (ns sample-test 55 | (:require [clojure.test :refer :all])) 56 | 57 | (deftest simple-fail-test 58 | (is (= :same :not-same))) 59 | """ 60 | When I run `bin/kaocha` 61 | And I run `cat /tmp/kaocha.txt` 62 | Then the output should contain: 63 | """ 64 | ⛔️ Failing 65 | 1 tests, 1 failures. 66 | true 67 | 1 68 | critical 69 | """ 70 | -------------------------------------------------------------------------------- /test/features/plugins/orchestra_plugin.feature: -------------------------------------------------------------------------------- 1 | Feature: Orchestra (spec instrumentation) 2 | 3 | You can enable spec instrumentation of your functions before running 4 | tests with the `:kaocha.plugin/orchestra` plugin. This uses the 5 | [Orchestra](https://github.com/jeaye/orchestra) library to instrument 6 | `:args`, `:ret`, and `:fn` specs. 7 | 8 | You can use the `:kaocha.plugin/preloads` plugin to ensure namespaces 9 | are required (similar to ClojureScript's preloads feature). This is 10 | useful to ensure that your specs required before the orchestra plugin 11 | instruments your functions. 12 | 13 | Scenario: Enabling Orchestra 14 | Given a file named "tests.edn" with: 15 | """ clojure 16 | #kaocha/v1 17 | {:plugins [:orchestra 18 | :preloads] 19 | :kaocha.plugin.preloads/ns-names [my.specs] 20 | :color? false} 21 | """ 22 | And a file named "test/orchestra_test.clj" with: 23 | """ clojure 24 | (ns orchestra-test 25 | (:require [clojure.test :refer :all] 26 | [clojure.spec.alpha :as spec])) 27 | 28 | (defn simple-fn [] 29 | "x") 30 | 31 | (spec/fdef simple-fn :ret :simple/int) 32 | 33 | (deftest spec-fail-test 34 | (is (= "x" (simple-fn)) "Just testing simple-fn")) 35 | """ 36 | And a file named "src/my/specs.clj" with: 37 | """ clojure 38 | (ns my.specs 39 | (:require [clojure.spec.alpha :as spec])) 40 | 41 | (spec/def :simple/int int?) 42 | """ 43 | When I run `bin/kaocha` 44 | Then the output should contain: 45 | """ 46 | ERROR in orchestra-test/spec-fail-test (orchestra_test.clj:11) 47 | Just testing simple-fn 48 | Call to #'orchestra-test/simple-fn did not conform to spec. 49 | orchestra_test.clj:11 50 | 51 | -- Spec failed -------------------- 52 | 53 | Return value 54 | 55 | "x" 56 | 57 | should satisfy 58 | 59 | int? 60 | 61 | -- Relevant specs ------- 62 | 63 | :simple/int: 64 | clojure.core/int? 65 | 66 | ------------------------- 67 | Detected 1 error 68 | """ 69 | -------------------------------------------------------------------------------- /test/features/plugins/version_filter.feature: -------------------------------------------------------------------------------- 1 | Feature: Plugin: Clojure/Java Version filter 2 | 3 | The `version-filter` plugin will look for test metadata specifying the minimum 4 | or maximum version of Clojure or Java the test is designed to work with, and 5 | skip the test unless it falls within the range specified. 6 | 7 | The recognized metadata keys are `:min-clojure-version`, 8 | `:max-clojure-version`, `:min-java-version`, and `:max-java-version`. The 9 | associated value is a version string, such as `"1.10.0"`. 10 | 11 | You can set both a minimum and a maximum to limit to a certain range. The 12 | boundaries are always inclusive, so `^{:max-clojure-version "1.9"}` will run 13 | on Clojure `1.9.*` or earlier. 14 | 15 | Specificty matters, a test with a max version of `"1.10" will also run on 16 | version `"1.10.2"`, whereas if the max version is `"1.10.0"` it will not. 17 | 18 | Note that the Java version is based on the "java.runtime.version" system 19 | property. Before Java 9 this was the so called "developer version", which 20 | started with `1.`, e.g. `"1.8.0"`, so Java (JDK) versions effectivel jumped 21 | from `1.8` to `9`. 22 | [1](https://blogs.oracle.com/java-platform-group/a-new-jdk-9-version-string-scheme) 23 | [2](https://en.wikipedia.org/wiki/Java_version_history#Versioning_change) 24 | 25 | Scenario: Enabling in `tests.edn` 26 | Given a file named "tests.edn" with: 27 | """ clojure 28 | #kaocha/v1 29 | {:plugins [:kaocha.plugin/version-filter] 30 | :color? false} 31 | """ 32 | And a file named "test/my/sample_test.clj" with: 33 | """ clojure 34 | (ns my.sample-test 35 | (:require [clojure.test :refer :all])) 36 | 37 | (deftest ^{:max-java-version "1.7"} this-test-gets-skipped 38 | (is false)) 39 | 40 | (deftest ^{:min-clojure-version "1.6.0"} this-test-runs 41 | (is true)) 42 | """ 43 | When I run `bin/kaocha --reporter documentation` 44 | Then the output should contain: 45 | """ 46 | --- unit (clojure.test) --------------------------- 47 | my.sample-test 48 | this-test-runs 49 | 50 | 1 tests, 1 assertions, 0 failures. 51 | """ 52 | -------------------------------------------------------------------------------- /test/features/syntax_error.feature: -------------------------------------------------------------------------------- 1 | 2 | Feature: Syntax errors are preserved 3 | Syntax errors should be passed along. 4 | Scenario: Show output of failing test 5 | Given a file named "test/sample_test.clj" with: 6 | """ clojure 7 | (ns sample-test 8 | (:require [clojure.test :refer :all])) 9 | 10 | stray-symbol 11 | 12 | (deftest stdout-pass-test 13 | (is (= :same :same))) 14 | """ 15 | When I run `bin/kaocha` 16 | Then the output should contain: 17 | """ 18 | Exception: clojure.lang.Compiler$CompilerException 19 | """ 20 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_factories.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-factories 2 | (:require [kaocha.specs] 3 | [kaocha.api] 4 | [clojure.spec.alpha :as spec] 5 | [clojure.spec.gen.alpha :as gen] 6 | [kaocha.config :as config])) 7 | 8 | (defn var-testable [m] 9 | (let [testable (gen/generate (spec/gen :kaocha.type/var))] 10 | (merge testable 11 | {:kaocha.testable/type :kaocha.type/var 12 | :kaocha.testable/desc (name (:kaocha.testable/id testable))} 13 | m))) 14 | 15 | (defn test-plan [m] 16 | (merge 17 | (config/default-config) 18 | 19 | m)) 20 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_helper.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-helper 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :as t] 4 | [kaocha.core-ext :refer :all] 5 | [matcher-combinators.result :as mc.result] 6 | [matcher-combinators.core :as mc.core] 7 | [matcher-combinators.model :as mc.model] 8 | [clojure.spec.alpha :as spec] 9 | [expound.alpha :as expound] 10 | [orchestra.spec.test :as orchestra]) 11 | (:import (clojure.lang ExceptionInfo))) 12 | 13 | (require 'matcher-combinators.clj-test) 14 | 15 | (extend-protocol mc.core/Matcher 16 | clojure.lang.Var 17 | (match [this actual] 18 | (if (= this actual) 19 | {::mc.result/type :match 20 | ::mc.result/value actual 21 | ::mc.result/weight 0} 22 | {::mc.result/type :mismatch 23 | ::mc.result/value (if (and (keyword? actual) (= ::mc.core/missing actual)) 24 | (mc.model/->Missing this) 25 | (mc.model/->Mismatch this actual)) 26 | ::mc.result/weight 1}))) 27 | 28 | ;; TODO move to kaocha.assertions 29 | 30 | (defmacro thrown-ex-data? 31 | "Verifies that an expression throws an ExceptionInfo with specific data and 32 | message. Message can be string or regex. " 33 | [ex-msg ex-data & body] 34 | (assert nil "thrown-ex-data? used outside (is ) block")) 35 | 36 | (defmethod clojure.test/assert-expr 'thrown-ex-data? [msg form] 37 | (let [[_ ex-msg ex-data & body] form] 38 | `(try 39 | ~@body 40 | (t/do-report {:type :fail 41 | :message ~msg 42 | :expected '~form 43 | :actual nil}) 44 | (catch ExceptionInfo e# 45 | (let [m# (.getMessage e#) 46 | d# (clojure.core/ex-data e#)] 47 | (cond 48 | (not (or (and (string? ~ex-msg) (= ~ex-msg m#)) 49 | (and (regex? ~ex-msg) (re-find ~ex-msg m#)))) 50 | (t/do-report {:type :fail 51 | :message ~msg 52 | :expected '~ex-msg 53 | :actual m#}) 54 | 55 | (not= ~ex-data d#) 56 | (t/do-report {:type :fail 57 | :message ~msg 58 | :expected ~ex-data 59 | :actual d#}) 60 | 61 | :else 62 | (t/do-report {:type :pass 63 | :message ~msg 64 | :expected '~form 65 | :actual e#}))) 66 | true)))) 67 | 68 | (defn spec-valid? 69 | "Asserts that the value matches the spec." 70 | [spec value & [msg]] 71 | (spec/valid? spec value)) 72 | 73 | (defmethod t/assert-expr 'spec-valid? [msg form] 74 | `(let [[spec# value#] (list ~@(rest form))] 75 | (t/do-report 76 | (if (spec/valid? spec# value#) 77 | {:type :pass 78 | :message ~msg 79 | :expected '~form 80 | :actual '~form} 81 | {:type :fail 82 | :message (or ~msg (expound/expound-str spec# value#)) 83 | :expected '~form 84 | :actual (list '~'not '~form)})))) 85 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_plugins.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-plugins 2 | (:require [kaocha.plugin :as plugin] 3 | [kaocha.output :as output] 4 | [clojure.test :as t] 5 | [clojure.string :as str] 6 | [kaocha.hierarchy :as hierarchy])) 7 | 8 | (hierarchy/derive! ::unexpected-error :kaocha/known-key) 9 | (hierarchy/derive! ::unexpected-error :kaocha/fail-type) 10 | 11 | (hierarchy/derive! ::unexpected-warning :kaocha/known-key) 12 | (hierarchy/derive! ::unexpected-warning :kaocha/fail-type) 13 | 14 | (plugin/defplugin ::capture-warn+error 15 | "Turn errors and warning messages into test failures." 16 | (wrap-run [run test-plan] 17 | (fn [& args] 18 | (let [warnings (atom []) 19 | errors (atom [])] 20 | (with-redefs [output/warn (fn [& xs] (swap! warnings conj xs)) 21 | output/error (fn [& xs] (swap! errors conj xs))] 22 | (let [result (apply run args)] 23 | (when (seq @errors) 24 | (#'t/do-report {:type ::unexpected-error 25 | :message (str "Unexpected Error:\n" 26 | (str/join "\n" (map #(apply str " ERROR: " %) @errors)))})) 27 | (when (seq @warnings) 28 | (#'t/do-report {:type ::unexpected-warning 29 | :message (str "Unexpected Warning:\n" 30 | (str/join "\n" (map #(apply str "WARNING: " %) @warnings)))})) 31 | result)))))) 32 | -------------------------------------------------------------------------------- /test/shared/kaocha/test_util.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.test-util 2 | (:require [clojure.test :as t] 3 | [clojure.spec.alpha :as spec] 4 | [expound.alpha :as expound] 5 | [kaocha.report :as report] 6 | [kaocha.testable :as testable] 7 | [kaocha.output :as output] 8 | [kaocha.type :as type])) 9 | 10 | (def ^:dynamic *report-history* nil) 11 | 12 | (defmacro with-test-ctx 13 | "When testing lower level functions, make sure the necessary shared state is set 14 | up. This also helps to isolate us from the outer test runner state." 15 | [opts & body] 16 | `(binding [t/*report-counters* (ref type/initial-counters) 17 | t/*testing-vars* (list) 18 | *report-history* (atom []) 19 | testable/*fail-fast?* (:fail-fast? ~opts)] 20 | (with-redefs [t/report (fn [m#] 21 | (swap! *report-history* conj m#) 22 | (report/report-counters m#) 23 | (when (:fail-fast? ~opts) (report/fail-fast m#)))] 24 | (let [result# (do ~@body)] 25 | {:result result# 26 | :report @*report-history*})))) 27 | 28 | (defmacro with-out-err 29 | "Captures the return value of the expression, as well as anything written on 30 | stdout or stderr." 31 | [& body] 32 | `(let [o# (java.io.StringWriter.) 33 | e# (java.io.StringWriter.)] 34 | (binding [*out* o# 35 | *err* e#] 36 | (let [r# (do ~@body)] 37 | {:out (str o#) 38 | :err (str e#) 39 | :result r#})))) 40 | 41 | (defmacro expect-warning 42 | {:style/indent [1]} 43 | [pattern & body] 44 | `(let [warnings# (atom [])] 45 | (with-redefs [output/warn (fn [& xs#] (swap! warnings# conj xs#))] 46 | (let [result# (do ~@body)] 47 | (when-not (seq (filter #(re-find ~pattern (apply str %)) @warnings#)) 48 | (#'t/do-report {:type :fail 49 | :message (str "Expected test to generate a warning (" 50 | ~pattern 51 | ") but no warning occured.")})) 52 | result#)))) 53 | 54 | (defmacro with-test-out-str 55 | "Evaluates exprs in a context in which [[clojure.test/*test-out*]] is bound to a 56 | fresh StringWriter. Returns the string created by any nested printing calls 57 | that use [[clojure.test/with-test-out]]." 58 | [& body] 59 | `(let [s# (new java.io.StringWriter)] 60 | (binding [clojure.test/*test-out* s#] 61 | ~@body 62 | (str s#)))) 63 | -------------------------------------------------------------------------------- /test/step_definitions/kaocha_integration.clj: -------------------------------------------------------------------------------- 1 | (ns features.steps.kaocha-integration 2 | ^{:clojure.tools.namespace.repl/load false 3 | :clojure.tools.namespace.repl/unload false} 4 | (:require [clojure.edn :as edn] 5 | [clojure.java.io :as io] 6 | [clojure.java.shell :as shell] 7 | [clojure.string :as str] 8 | [clojure.test :as t :refer :all] 9 | [kaocha.integration-helpers :refer :all] 10 | [kaocha.output :as output] 11 | [kaocha.shellwords :refer [shellwords]] 12 | [lambdaisland.cucumber.dsl :refer :all] 13 | [me.raynes.fs :as fs])) 14 | 15 | (require 'kaocha.assertions) 16 | 17 | (Given "a file named {string} with:" [m path contents] 18 | (spit-file m path contents)) 19 | 20 | (def last-cpcache-dir (atom nil)) 21 | 22 | (When "I run `(.*)`" [m args] 23 | (let [{:keys [config-file dir] :as m} (test-dir-setup m)] 24 | 25 | (when-let [cache @last-cpcache-dir] 26 | (let [target (join dir ".cpcache")] 27 | (when-not (.isDirectory (io/file target)) 28 | (mkdir target) 29 | (run! #(fs/copy % (io/file (join target (.getName %)))) (fs/glob cache "*"))))) 30 | 31 | (let [result (apply shell/sh (conj (shellwords args) 32 | :dir dir))] 33 | ;; By default these are hidden unless the test fails 34 | (when (seq (:out result)) 35 | (println (str dir) "$" args) 36 | (println (str (output/colored :underline "stdout") ":\n" (:out result)))) 37 | (when (seq (:err result)) 38 | (println (str (output/colored :underline "stderr") ":\n" (:err result)))) 39 | (let [cpcache (io/file (join dir ".cpcache"))] 40 | (when (.exists cpcache) 41 | (reset! last-cpcache-dir cpcache))) 42 | (merge m result)))) 43 | 44 | (Then "the exit-code is non-zero" [{:keys [exit] :as m}] 45 | (is (not= "0" exit)) 46 | m) 47 | 48 | (Then "the exit-code should be {int}" [{:keys [exit] :as m} code] 49 | (is (= code (Integer. exit))) 50 | m) 51 | 52 | (Then "the output should contain:" [m output] 53 | (is (substring? output (:out m))) 54 | m) 55 | 56 | (Then "stderr should contain:" [m output] 57 | (is (substring? output (:err m))) 58 | m) 59 | 60 | (Then "the output should be" [m output] 61 | (is (= (str/trim output) (str/trim (:out m)))) 62 | m) 63 | 64 | (Then "the output should not contain" [m output] 65 | (is (not (str/includes? (:out m) output))) 66 | m) 67 | 68 | (Then "the EDN output should contain:" [m output] 69 | (let [actual (edn/read-string (:out m)) 70 | expected (edn/read-string output)] 71 | (is (= (select-keys actual (keys expected)) expected))) 72 | m) 73 | 74 | 75 | (Then "stderr should contain" [m output] 76 | (is (substring? output (:err m))) 77 | m) 78 | 79 | (Then "print output" [m] 80 | (t/with-test-out 81 | (println "----out---------------------------------------") 82 | (println (:out m)) 83 | (println "----err---------------------------------------") 84 | (println (:err m))) 85 | m) 86 | 87 | #_ 88 | (do 89 | (require 'kaocha.repl) 90 | (kaocha.repl/run :plugins.version-filter {:kaocha.plugin.capture-output/capture-output? false 91 | } 92 | )) 93 | -------------------------------------------------------------------------------- /test/unit/kaocha/api_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.api-test 2 | (:require [clojure.test :refer [testing is deftest]] 3 | [kaocha.api :refer :all] 4 | [kaocha.test-util :refer [with-out-err]] 5 | [slingshot.slingshot :refer [try+]])) 6 | 7 | (deftest run-test 8 | (testing "allows API usage" 9 | (let [config {:kaocha/tests [{:kaocha.testable/id :unit 10 | :kaocha.testable/desc "unit (clojure.test)" 11 | :kaocha.testable/type :kaocha.type/clojure.test 12 | :kaocha/test-paths ["fixtures/a-tests"] 13 | :kaocha/source-paths ["src"] 14 | :kaocha/ns-patterns ["-test$"]}]}] 15 | (is (match? 16 | {:kaocha.result/tests 17 | [{:kaocha.testable/id :unit 18 | :kaocha.testable/type :kaocha.type/clojure.test 19 | :kaocha/test-paths ["fixtures/a-tests"] 20 | :kaocha/source-paths ["src"] 21 | :kaocha/ns-patterns ["-test$"] 22 | :kaocha.result/tests 23 | [{:kaocha.testable/type :kaocha.type/ns 24 | :kaocha.testable/id :baz.qux-test 25 | :kaocha.result/tests 26 | [{:kaocha.testable/type :kaocha.type/var 27 | :kaocha.testable/id :baz.qux-test/nested-test 28 | :kaocha.testable/desc "nested-test" 29 | :kaocha.var/name 'baz.qux-test/nested-test 30 | :kaocha.result/count 1 31 | :kaocha.result/pass 1 32 | :kaocha.result/error 1 33 | :kaocha.result/fail 0}]} 34 | {:kaocha.testable/type :kaocha.type/ns 35 | :kaocha.testable/id :foo.bar-test 36 | :kaocha.result/tests 37 | [{:kaocha.testable/type :kaocha.type/var 38 | :kaocha.testable/id :foo.bar-test/a-test 39 | :kaocha.testable/desc "a-test" 40 | :kaocha.var/name 'foo.bar-test/a-test 41 | :kaocha.result/count 1 42 | :kaocha.result/pass 1 43 | :kaocha.result/error 0 44 | :kaocha.result/fail 0}]}]}]} 45 | (:result (with-out-err (run config)))))))) 46 | 47 | (deftest no-tests 48 | (testing "no tests are found!") 49 | (is (= :caught (try+ 50 | (run {}) 51 | (catch :kaocha/early-exit e 52 | :caught))))) 53 | -------------------------------------------------------------------------------- /test/unit/kaocha/config/included-test.edn: -------------------------------------------------------------------------------- 1 | {:reporter [kaocha.report.progress/report] 2 | :plugins [:other.kaocha.plugin/bar]} 3 | -------------------------------------------------------------------------------- /test/unit/kaocha/config/loaded-test-profile.edn: -------------------------------------------------------------------------------- 1 | 2 | #kaocha/v1 3 | {:reporter #profile {:test kaocha.report.progress/report 4 | :ci kaocha.report/documentation 5 | :default kaocha.report/documentation}} 6 | -------------------------------------------------------------------------------- /test/unit/kaocha/config/loaded-test-resource.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 #meta-merge 2 | [#include "kaocha/config/included-test.edn" 3 | {:fail-fast? true 4 | :plugins [:some.kaocha.plugin/qux]}] 5 | -------------------------------------------------------------------------------- /test/unit/kaocha/config/loaded-test-spec-mismatch.edn: -------------------------------------------------------------------------------- 1 | 2 | #kaocha/v1 #meta-merge 3 | [#include "./included-test.edn" 4 | {:plugins :some.kaocha.plugin/foo}] 5 | -------------------------------------------------------------------------------- /test/unit/kaocha/config/loaded-test.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 #meta-merge 2 | [#include "./included-test.edn" 3 | {:plugins [:some.kaocha.plugin/foo]}] 4 | -------------------------------------------------------------------------------- /test/unit/kaocha/core_ext_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.core-ext-test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :refer :all] 4 | [kaocha.core-ext :refer :all])) 5 | 6 | (deftest regex?-test 7 | (is (regex? #"foo")) 8 | (is (not (regex? "foo")))) 9 | 10 | (deftest exception?-test 11 | (is (exception? (Exception. "oh no"))) 12 | (is (not (exception? (Throwable. "oh no"))))) 13 | 14 | (deftest error?-test 15 | (is (error? (Error. "oh no"))) 16 | (is (not (error? (Exception. "oh no")))) 17 | (is (not (error? (Throwable. "oh no"))))) 18 | 19 | (deftest throwable?-test 20 | (is (throwable? (Error. "oh no"))) 21 | (is (throwable? (Exception. "oh no"))) 22 | (is (throwable? (Throwable. "oh no")))) 23 | 24 | (deftest ns?-test 25 | (is (ns? *ns*)) 26 | (is (not (ns? {})))) 27 | 28 | (deftest regex-test 29 | (is (= "ok" (re-find (regex "[ko]+") "--ok--"))) 30 | (is (= "ok" (re-find (regex #"[ko]+") "--ok--"))) 31 | (is (= "ok" (re-find (regex "o" "k") "--ok--"))) 32 | (is (thrown? clojure.lang.ExceptionInfo (regex 123) ""))) 33 | 34 | (deftest mapply-test 35 | (let [f (fn [& {:as opts}] 36 | {:opts opts})] 37 | (is (= {:opts {:foo :bar :abc :xyz}} 38 | (mapply f {:foo :bar :abc :xyz}))))) 39 | -------------------------------------------------------------------------------- /test/unit/kaocha/fixtures_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.fixtures-test 2 | (:refer-clojure :exclude [symbol]) 3 | (:require [clojure.test :as t :refer [testing is deftest]] 4 | [kaocha.test-factories :as f] 5 | [kaocha.testable :as testable] 6 | [kaocha.classpath :as classpath] 7 | [kaocha.test-helper] 8 | [kaocha.core-ext :refer :all] 9 | [kaocha.test-util :refer [with-test-ctx]] 10 | [kaocha.type.var] 11 | [matcher-combinators.test :refer [match?]])) 12 | 13 | (deftest once-fixtures-test 14 | (classpath/add-classpath "fixtures/d-tests") 15 | (testing "once fixture calling f twice" 16 | (require 'ddd.double-once-fixture-test) 17 | (let [{:keys [result report]} 18 | (with-test-ctx {:fail-fast? false} 19 | (testable/run (testable/load {:kaocha.testable/type :kaocha.type/ns 20 | :kaocha.testable/id :ddd.double-once-fixture-test 21 | :kaocha.testable/desc "ddd.double-once-fixture-test" 22 | :kaocha.ns/name 'ddd.double-once-fixture-test}) 23 | (f/test-plan {})))] 24 | 25 | (is (match? {:kaocha.testable/type :kaocha.type/ns 26 | :kaocha.testable/id :ddd.double-once-fixture-test 27 | :kaocha.testable/desc "ddd.double-once-fixture-test" 28 | :kaocha.result/tests 29 | [{:kaocha.testable/type :kaocha.type/var 30 | :kaocha.testable/id :ddd.double-once-fixture-test/example-fail-test 31 | :kaocha.testable/desc "example-fail-test" 32 | :kaocha.var/name 'ddd.double-once-fixture-test/example-fail-test 33 | :kaocha.var/var (resolve 'ddd.double-once-fixture-test/example-fail-test) 34 | :kaocha.var/test fn? 35 | :kaocha.result/count 1 36 | :kaocha.result/pass 0 37 | :kaocha.result/error 0 38 | :kaocha.result/fail 1} 39 | {:kaocha.testable/type :kaocha.type/var 40 | :kaocha.testable/id :ddd.double-once-fixture-test/example-fail-test 41 | :kaocha.testable/desc "example-fail-test" 42 | :kaocha.var/name 'ddd.double-once-fixture-test/example-fail-test 43 | :kaocha.var/var (resolve 'ddd.double-once-fixture-test/example-fail-test) 44 | :kaocha.var/test fn? 45 | :kaocha.result/count 1 46 | :kaocha.result/pass 1 47 | :kaocha.result/error 0 48 | :kaocha.result/fail 0}]} 49 | result)) 50 | 51 | (is (match? [{:type :begin-test-ns} 52 | {:type :begin-test-var} 53 | {:type :fail 54 | :expected '(= 1 2) 55 | :actual '(not (= 1 2)) 56 | :message nil} 57 | {:type :end-test-var} 58 | {:type :begin-test-var} 59 | {:type :pass 60 | :expected '(= 2 2) 61 | :actual (list = 2 2) 62 | :message nil} 63 | {:type :end-test-var} 64 | {:type :end-test-ns}] 65 | (mapv #(select-keys % [:type :expected :actual :message]) report)))))) 66 | -------------------------------------------------------------------------------- /test/unit/kaocha/hierarchy_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.hierarchy-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.hierarchy :as hierarchy])) 4 | 5 | (require 'kaocha.type.clojure.test) 6 | 7 | (derive ::global-leaf :kaocha.testable.type/leaf) 8 | (derive ::global-group :kaocha.testable.type/group) 9 | (derive ::global-suite :kaocha.testable.type/suite) 10 | (hierarchy/derive! ::local-leaf :kaocha.testable.type/leaf) 11 | (hierarchy/derive! ::local-group :kaocha.testable.type/group) 12 | (hierarchy/derive! ::local-suite :kaocha.testable.type/suite) 13 | 14 | (deftest fail-type?-test 15 | (is (hierarchy/fail-type? {:type :fail})) 16 | (is (hierarchy/fail-type? {:type :error}))) 17 | 18 | (deftest error-type?-test 19 | (is (hierarchy/error-type? {:type :error}))) 20 | 21 | (deftest pass-type?-test 22 | (is (hierarchy/pass-type? {:type :pass}))) 23 | 24 | (deftest known-key?-test 25 | (is (hierarchy/known-key? {:type :pass})) 26 | (is (hierarchy/known-key? {:type :fail})) 27 | (is (hierarchy/known-key? {:type :error})) 28 | (is (hierarchy/known-key? {:type :kaocha/known-key})) 29 | (is (hierarchy/known-key? {:type :kaocha/deferred})) 30 | (is (not (hierarchy/known-key? {:type :kaocha/foo})))) 31 | 32 | (derive ::global-deferred :kaocha/deferred) 33 | (hierarchy/derive! ::local-deferred :kaocha/deferred) 34 | 35 | (deftest deferred?-test 36 | (is (hierarchy/deferred? {:type ::global-deferred})) 37 | (is (hierarchy/deferred? {:type ::local-deferred}))) 38 | 39 | (deftest pending?-test 40 | (is (hierarchy/pending? {:type :kaocha/pending}))) 41 | 42 | (deftest suite-test 43 | (is (hierarchy/suite? {:kaocha.testable/type :kaocha.testable.type/suite})) 44 | (is (hierarchy/suite? {:kaocha.testable/type :kaocha.type/clojure.test})) 45 | (is (hierarchy/suite? {:kaocha.testable/type ::global-suite})) 46 | (is (hierarchy/suite? {:kaocha.testable/type ::local-suite}))) 47 | 48 | (deftest group-test 49 | (is (hierarchy/group? {:kaocha.testable/type :kaocha.testable.type/group})) 50 | (is (hierarchy/group? {:kaocha.testable/type :kaocha.type/ns})) 51 | (is (hierarchy/group? {:kaocha.testable/type ::global-group})) 52 | (is (hierarchy/group? {:kaocha.testable/type ::local-group}))) 53 | 54 | (deftest leaf-test 55 | (is (hierarchy/leaf? {:kaocha.testable/type :kaocha.testable.type/leaf})) 56 | (is (hierarchy/leaf? {:kaocha.testable/type :kaocha.type/var})) 57 | (is (hierarchy/leaf? {:kaocha.testable/type ::global-leaf})) 58 | (is (hierarchy/leaf? {:kaocha.testable/type ::local-leaf}))) 59 | -------------------------------------------------------------------------------- /test/unit/kaocha/history_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.history-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.history :refer :all])) 4 | 5 | (deftest clojure-test-summary-test 6 | (is (= (clojure-test-summary []) 7 | {:type :summary :test 0, :pass 0, :fail 0, :error 0})) 8 | 9 | (is (= (clojure-test-summary [{:type :pass}]) 10 | {:type :summary :test 0, :pass 1, :fail 0, :error 0})) 11 | 12 | (is (= (clojure-test-summary [{:type :begin-test-var} 13 | {:type :pass} 14 | {:type :fail} 15 | {:type :pass} 16 | {:type :begin-test-ns} 17 | {:type :error}]) 18 | {:type :summary :test 1, :pass 2, :fail 1, :error 1}))) 19 | -------------------------------------------------------------------------------- /test/unit/kaocha/monkey_patch_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.monkey-patch-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.monkey-patch :as monkey-patch] 4 | [kaocha.plugin :as plugin])) 5 | 6 | ;; Note: this test is doing some somersaults to avoid crashing into clojure.test 7 | ;; while it is itself running. 8 | (deftest pre-report-hook-is-used 9 | (let [result (atom nil)] 10 | (binding [plugin/*current-chain* [{:kaocha.hooks/pre-report (fn [event] (assoc event :been-here true))}]] 11 | (with-redefs [monkey-patch/report (fn [event] (reset! result event))] 12 | (monkey-patch/do-report {:type :pass}))) 13 | (is (:been-here @result)))) 14 | -------------------------------------------------------------------------------- /test/unit/kaocha/output_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.output-test 2 | (:require [kaocha.output :as output] 3 | [clojure.test :refer :all] 4 | [kaocha.test-util :as util])) 5 | 6 | (deftest colored-test 7 | (is (= "foo" (output/colored :green "foo"))) 8 | 9 | (is (= "foo" 10 | (binding [output/*colored-output* false] 11 | (output/colored :green "foo"))))) 12 | 13 | (deftest warn-test 14 | (testing "without color" 15 | (is (= {:err "WARNING: Oh no!\n", :out "", :result nil} 16 | (binding [output/*colored-output* false] 17 | (util/with-out-err 18 | (output/warn "Oh no!")))))) 19 | 20 | (testing "with color" 21 | (is (= {:err "WARNING: Oh no!\n", :out "", :result nil} 22 | (util/with-out-err 23 | (output/warn "Oh no!"))))) 24 | 25 | (testing "multiple arguments" 26 | (is (= {:err "WARNING: one mississippi, two mississippi\n", :out "", :result nil} 27 | (util/with-out-err 28 | (output/warn "one mississippi" ", " "two mississippi")))))) 29 | 30 | (deftest error-test 31 | (testing "without color" 32 | (is (= {:err "ERROR: Oh no!\n", :out "", :result nil} 33 | (binding [output/*colored-output* false] 34 | (util/with-out-err 35 | (output/error "Oh no!")))))) 36 | 37 | (testing "with color" 38 | (is (= {:err "ERROR: Oh no!\n", :out "", :result nil} 39 | (util/with-out-err 40 | (output/error "Oh no!"))))) 41 | 42 | (testing "multiple arguments" 43 | (is (= {:err "ERROR: one mississippi, two mississippi\n", :out "", :result nil} 44 | (util/with-out-err 45 | (output/error "one mississippi" ", " "two mississippi")))))) 46 | 47 | (deftest format-doc-test 48 | (testing "without color" 49 | (is (= '[:group "[" [:align ([:group "{" [:align ([:span ":x" " " ":y"])] "}"])] "]"] 50 | (binding [output/*colored-output* false] 51 | (output/format-doc [{:x :y}]))))) 52 | 53 | (testing "with color" 54 | (is (= '[:group 55 | [:span [:pass ""] "[" [:pass ""]] 56 | [:align 57 | ([:group 58 | [:span [:pass ""] "{" [:pass ""]] 59 | [:align 60 | ([:span 61 | [:span [:pass ""] ":x" [:pass ""]] 62 | " " 63 | [:span [:pass ""] ":y" [:pass ""]]])] 64 | [:span [:pass ""] "}" [:pass ""]]])] 65 | [:span [:pass ""] "]" [:pass ""]]] 66 | (output/format-doc [{:x :y}]))))) 67 | 68 | 69 | (deftest print-doc-test 70 | (testing "prints with fipp" 71 | (is (= {:err "" 72 | :out "[:aaa :bbb :ccc]\n", 73 | :result nil} 74 | (util/with-out-err 75 | (-> (output/format-doc [:aaa :bbb :ccc]) 76 | (output/print-doc)))))) 77 | 78 | (testing "respects *print-length*" 79 | (is (= {:err "", 80 | :out "[:aaa\n :bbb\n :ccc]\n", 81 | :result nil} 82 | (util/with-out-err 83 | (binding [*print-length* 1] 84 | (-> (output/format-doc [:aaa :bbb :ccc]) 85 | (output/print-doc)))))))) 86 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/gc_profiling_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.gc-profiling-test 2 | (:require [clojure.test :refer [deftest is testing]] 3 | [clojure.set] 4 | [kaocha.test-helper :refer :all] 5 | [kaocha.testable :as testable] 6 | [kaocha.plugin :as plugin] 7 | [kaocha.test-util :refer [with-test-ctx]] 8 | [kaocha.plugin.gc-profiling :as gc])) 9 | 10 | 11 | (def plugin-chain (plugin/register :kaocha.plugin/gc-profiling [])) 12 | 13 | 14 | (def test-suite {:kaocha.testable/type :kaocha.type/clojure.test 15 | :kaocha.testable/id :a 16 | :kaocha/source-paths [] 17 | :kaocha/test-paths ["fixtures/a-tests"] 18 | :kaocha/ns-patterns [".*"]}) 19 | 20 | (deftest convert-bytes 21 | (testing "Basic values" 22 | (is 23 | (= "1.00GB" (gc/convert-bytes (+ 1 1e9)))) 24 | (is 25 | (= "1.00MB" (gc/convert-bytes (+ 1 1e6)))) 26 | (is 27 | (= "1.00kB" (gc/convert-bytes 1001))) 28 | (is 29 | (= "11B" (gc/convert-bytes 11))) 30 | (is 31 | (= "0B" (gc/convert-bytes 0))) 32 | ) 33 | (testing "Negative values" 34 | (is 35 | (= "-1.00GB" (gc/convert-bytes (+ -1 -1e9)))) 36 | (is 37 | (= "-1.00MB" (gc/convert-bytes (+ -1 -1e6)))) 38 | (is 39 | (= "-1.00kB" (gc/convert-bytes -1001))) 40 | (is 41 | (= "-11B" (gc/convert-bytes -11))))) 42 | 43 | (deftest rounding-divide 44 | (testing "Yielding whole numbers." 45 | (is 46 | (= 2 (gc/rounding-divide 4 2))) 47 | (is 48 | (= 2 (gc/rounding-divide 4.0 2))) 49 | (is 50 | (= 2 (gc/rounding-divide 4 2.0))) 51 | (is 52 | (= 2 (gc/rounding-divide 4.0 2.0)))) 53 | (testing "Yielding rational numbers as decimals." 54 | (is 55 | (= 3 (gc/rounding-divide 5 2))) 56 | (is 57 | (= 3 (gc/rounding-divide 5 2.0))) 58 | (is 59 | (= 3 (gc/rounding-divide 5.0 2))) 60 | (is 61 | (= 3 (gc/rounding-divide 5.0 2.0)))) 62 | (testing "Yielding rational numbers as decimals." 63 | (is 64 | (= -2 (gc/rounding-divide -5 2))) 65 | (is 66 | (= -2 (gc/rounding-divide -5 2.0))) 67 | (is 68 | (= -2 (gc/rounding-divide -5.0 2))) 69 | (is 70 | (= -2 (gc/rounding-divide -5.0 2.0))))) 71 | 72 | 73 | (deftest gc-profiling-test 74 | (plugin/with-plugins plugin-chain 75 | (is 76 | (match? {:kaocha.plugin.gc-profiling/gc-profiling? true 77 | :kaocha.plugin.gc-profiling/gc-profiling-individual false} 78 | (plugin/run-hook :kaocha.hooks/config {}))) 79 | (let [result ( plugin/run-hook :kaocha.hooks/cli-options [])] 80 | (is 81 | (clojure.set/subset? #{"--[no-]gc-profiling" 82 | "--[no-]gc-profiling-individual"} 83 | (set (map second result))))) 84 | (is 85 | 86 | (let [test-plan (testable/load test-suite) 87 | test-results (->> (testable/run test-plan test-plan) 88 | (with-test-ctx {}) 89 | :result 90 | :kaocha.result/tests)] 91 | (every? :kaocha.plugin.gc-profiling/delta test-results))))) 92 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/hooks_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.hooks-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.plugin.hooks :as hooks] 4 | [kaocha.testable :as testable] 5 | [kaocha.test-util :as util] 6 | [kaocha.plugin :as plugin] 7 | [clojure.test :as t])) 8 | 9 | (def ^:dynamic *inside-wrap-run?* false) 10 | 11 | (deftest wrap-run-test 12 | (let [run (fn [] 13 | (print "inside-wrap-run?:" *inside-wrap-run?*)) 14 | wrap-run (fn [run] 15 | (fn [& args] 16 | (binding [*inside-wrap-run?* true] 17 | (apply run args)))) 18 | test-plan {:kaocha.hooks/wrap-run [wrap-run]}] 19 | (binding [plugin/*current-chain* [hooks/hooks-hooks]] 20 | (let [run' (plugin/run-hook :kaocha.hooks/wrap-run run test-plan)] 21 | (is (= "inside-wrap-run?: true" 22 | (with-out-str (run')))))))) 23 | 24 | (deftest pre-report-test 25 | (is (match? {:report [{:type :fail 26 | :went-through-hook? true}]} 27 | (let [test-plan {:kaocha.hooks/pre-report 28 | [(fn [m] (assoc m :went-through-hook? true))]}] 29 | (util/with-test-ctx {} 30 | (binding [plugin/*current-chain* [hooks/hooks-hooks] 31 | testable/*test-plan* test-plan] 32 | (t/do-report {:type :fail}))))))) 33 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/randomize_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.randomize-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.test-helper :refer :all] 4 | [kaocha.plugin :as plugin] 5 | [kaocha.testable :as testable])) 6 | 7 | 8 | (def plugin-chain (plugin/register :kaocha.plugin/randomize [])) 9 | 10 | (def test-suite {:kaocha.testable/type :kaocha.type/clojure.test 11 | :kaocha.testable/id :c 12 | :kaocha.testable/desc "c (clojure.test)" 13 | :kaocha/source-paths [] 14 | :kaocha/test-paths ["fixtures/c-tests"] 15 | :kaocha/ns-patterns [".*"]}) 16 | 17 | (deftest randomize-test 18 | (plugin/with-plugins plugin-chain 19 | (is (match? {:kaocha.plugin.randomize/randomize? true 20 | :kaocha.plugin.randomize/seed number?} 21 | (plugin/run-hook :kaocha.hooks/config {}))) 22 | 23 | (is (match? {:kaocha.testable/type :kaocha.type/clojure.test 24 | :kaocha.test-plan/tests [{:kaocha.testable/type :kaocha.type/ns 25 | :kaocha.testable/id :foo.hello-test 26 | :kaocha.testable/desc "foo.hello-test" 27 | :kaocha.test-plan/tests 28 | [{:kaocha.testable/id :foo.hello-test/pass-1} 29 | {:kaocha.testable/id :foo.hello-test/fail-1} 30 | {:kaocha.testable/id :foo.hello-test/pass-3} 31 | {:kaocha.testable/id :foo.hello-test/pass-2}]}]} 32 | 33 | (plugin/run-hook :kaocha.hooks/post-load 34 | (-> test-suite 35 | (assoc :kaocha.plugin.randomize/seed 123 36 | :kaocha.plugin.randomize/randomize? true) 37 | testable/load)))) 38 | 39 | (is (match? {:kaocha.testable/type :kaocha.type/clojure.test 40 | :kaocha.test-plan/tests [{:kaocha.testable/type :kaocha.type/ns 41 | :kaocha.testable/id :foo.hello-test 42 | :kaocha.testable/desc "foo.hello-test" 43 | :kaocha.test-plan/tests 44 | [{:kaocha.testable/id :foo.hello-test/pass-2} 45 | {:kaocha.testable/id :foo.hello-test/pass-3} 46 | {:kaocha.testable/id :foo.hello-test/fail-1} 47 | {:kaocha.testable/id :foo.hello-test/pass-1}]}]} 48 | 49 | (plugin/run-hook :kaocha.hooks/post-load 50 | (-> test-suite 51 | (assoc :kaocha.plugin.randomize/seed 456 52 | :kaocha.plugin.randomize/randomize? true) 53 | testable/load)))))) 54 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin/version_filter_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin.version-filter-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.plugin.version-filter :as v])) 4 | 5 | (defmacro with-java-version 6 | {:style/indent [1]} 7 | [version & body] 8 | `(let [original# (System/getProperty "java.runtime.version")] 9 | (try 10 | (System/setProperty "java.runtime.version" ~version) 11 | ~@body 12 | (finally 13 | (System/setProperty "java.runtime.version" original#))))) 14 | 15 | (defmacro with-clojure-version 16 | {:style/indent [1]} 17 | [version & body] 18 | `(binding [*clojure-version* (zipmap [:major :minor :incremental] (v/version-vector ~version))] 19 | ~@body)) 20 | 21 | (deftest version-vector-test 22 | (is (= (v/version-vector "1.10.0") [1 10 0])) 23 | (is (= (v/version-vector "1.10.0-beta5") [1 10 0])) 24 | (is (= (v/version-vector "10.0.2+13-Ubuntu-1ubuntu0.18.04.4") [10 0 2]))) 25 | 26 | (deftest compare-versions-test 27 | (is (= 0 (v/compare-versions "1.10.0" "1.10.0"))) 28 | (is (= 0 (v/compare-versions "1.10.0" "1.10"))) 29 | (is (= 0 (v/compare-versions "1.10" "1.10.0"))) 30 | (is (= -1 (v/compare-versions "1.9" "1.10"))) 31 | (is (= 1 (v/compare-versions "1.10" "1.9"))) 32 | (is (= 0 (v/compare-versions "1.10" "1.10.0"))) 33 | (is (= 1 (v/compare-versions "1.10.1" "1.10.0")))) 34 | 35 | (deftest version>=?-test 36 | (is (v/version>=? "1.10.0" "1.9")) 37 | (is (v/version>=? "1.10.0" "1.10")) 38 | (is (not (v/version>=? "1.10.0" "1.10.1"))) 39 | (is (not (v/version>=? "1.9" "1.10.0"))) 40 | (is (v/version>=? "1.10" "1.10.0")) 41 | (is (v/version>=? "1.10.1" "1.10.0")) 42 | (is (v/version>=? "1" nil)) 43 | (is (v/version>=? nil "1")) 44 | (is (v/version>=? nil nil))) 45 | 46 | (deftest skip?-test 47 | (with-clojure-version "1.10.0" 48 | (with-java-version "9" 49 | (is (not (v/skip? {:kaocha.testable/meta {:min-clojure-version "1.10" 50 | :max-java-version "10"}}))))) 51 | 52 | (with-clojure-version "1.9.0" 53 | (with-java-version "9" 54 | (is (v/skip? {:kaocha.testable/meta {:min-clojure-version "1.10" 55 | :max-java-version "10"}})))) 56 | 57 | (with-clojure-version "1.10.0" 58 | (with-java-version "11" 59 | (is (v/skip? {:kaocha.testable/meta {:min-clojure-version "1.10" 60 | :max-java-version "10"}}))))) 61 | -------------------------------------------------------------------------------- /test/unit/kaocha/plugin_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.plugin-test 2 | (:require [kaocha.plugin :as plugin] 3 | [clojure.test :refer :all] 4 | [kaocha.test-util :as util] 5 | [kaocha.output :as output]) 6 | (:import (clojure.lang ExceptionInfo))) 7 | 8 | (deftest missing-plugin-test 9 | (let [expected-message "Couldn't load plugin :kaocha.missing.plugin/gone. Failed to load namespaces kaocha.missing.plugin.gone and kaocha.missing.plugin. This could be caused by a misspelling or a missing dependency."] 10 | (is (thrown-with-msg? ExceptionInfo 11 | (re-pattern expected-message) 12 | (plugin/load-all [:kaocha.missing.plugin/gone]))) 13 | (is (= {:err (str "ERROR: " expected-message "\n") :out "" :result nil} 14 | (binding [output/*colored-output* false] 15 | (util/with-out-err 16 | (try 17 | (plugin/load-all [:kaocha.missing.plugin/gone]) 18 | (catch ExceptionInfo e 19 | nil)))))))) 20 | 21 | (deftest missing-unnamespaced-plugin-test 22 | (let [expected-message "Couldn't load plugin :kaocha.plugin/gone. Failed to load namespace kaocha.plugin.gone. This could be caused by a misspelling or a missing dependency."] 23 | (is (thrown-with-msg? ExceptionInfo 24 | (re-pattern expected-message) 25 | (plugin/load-all [:gone]))) 26 | (is (= {:err (str "ERROR: " expected-message "\n") :out "" :result nil} 27 | (binding [output/*colored-output* false] 28 | (util/with-out-err 29 | (try 30 | (plugin/load-all [:gone]) 31 | (catch ExceptionInfo e 32 | nil)))))))) 33 | 34 | 35 | (deftest missing-plugin-valid-ns-test 36 | (let [expected-message "Couldn't load plugin :kaocha/plugin. The plugin was not defined after loading namespace kaocha.plugin. Is the file missing a defplugin?"] 37 | (is (thrown-with-msg? ExceptionInfo 38 | (re-pattern expected-message) 39 | (plugin/load-all [:kaocha/plugin]))) 40 | (is (= {:err (str "ERROR: " expected-message "\n") :out "" :result nil} 41 | (binding [output/*colored-output* false] 42 | (util/with-out-err 43 | (try 44 | (plugin/load-all [:kaocha/plugin]) 45 | (catch ExceptionInfo e 46 | nil)))))))) 47 | 48 | (deftest normalize-name-test 49 | (are [input expected] (= expected (plugin/normalize-name input)) 50 | :abc :kaocha.plugin/abc 51 | :kaocha.plugin/abc :kaocha.plugin/abc 52 | :custom-ns/abc :custom-ns/abc 53 | :custom.ns.abc :custom.ns.abc)) 54 | -------------------------------------------------------------------------------- /test/unit/kaocha/post_assertion_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.post-assertion-test 2 | (:require [clojure.test :refer [deftest is use-fixtures]])) 3 | 4 | (defn- post-assertion-fixture 5 | "Verify that an :each fixture that makes its own assertions doesn't 6 | break join-fixtures wrapping." 7 | [f] 8 | (f) 9 | (is (= true true))) 10 | 11 | (use-fixtures :each #'post-assertion-fixture) 12 | 13 | (deftest post-assertion-simple-test 14 | (is (= 2 (+ 1 1)))) 15 | -------------------------------------------------------------------------------- /test/unit/kaocha/private_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.private-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.testable :as testable])) 4 | 5 | (deftest- private-test 6 | (is true)) 7 | 8 | (deftest public-test 9 | (is true)) 10 | 11 | (deftest test-load 12 | (let [testable (testable/load {:kaocha.testable/type :kaocha.type/ns 13 | :kaocha.testable/id :kaocha.private-test 14 | :kaocha.testable/desc "kaocha.private-test" 15 | :kaocha.ns/name 'kaocha.private-test})] 16 | (is (= 3 (count (:kaocha.test-plan/tests testable)))))) 17 | -------------------------------------------------------------------------------- /test/unit/kaocha/random_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.random-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.random :refer :all])) 4 | 5 | (deftest rand-ints-test 6 | (is (= (take 10 (repeatedly (rng 321098))) 7 | [-141996321 1985580023 -305308934 1158906095 -1212597759 -42859192 -98240991 1350981438 31847656 -294128715])) 8 | 9 | (is (= (take 10 (repeatedly (rng 1))) 10 | [-1155869325 431529176 1761283695 1749940626 892128508 155629808 1429008869 -1465154083 -138487339 -1242363800]))) 11 | 12 | (deftest randomize-tests-test 13 | (is (= (randomize-tests 1 '{first.ns [first.var1 first.var2 first.var3] 14 | other.ns [other.var1 other.var2 other.var3] 15 | third.ns [third.var1 third.var2 third.var3]}) 16 | '([first.ns (first.var2 first.var1 first.var3)] 17 | [other.ns (other.var1 other.var2 other.var3)] 18 | [third.ns (third.var3 third.var2 third.var1)]))) 19 | 20 | (is (= (randomize-tests 431 '{first.ns [first.var1 first.var2 first.var3] 21 | other.ns [other.var1 other.var2 other.var3] 22 | third.ns [third.var1 third.var2 third.var3]}) 23 | '([other.ns (other.var3 other.var2 other.var1)] 24 | [third.ns (third.var1 third.var3 third.var2)] 25 | [first.ns (first.var3 first.var2 first.var1)])))) 26 | -------------------------------------------------------------------------------- /test/unit/kaocha/repl_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.repl-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.repl :as repl] 4 | [kaocha.config :as config])) 5 | 6 | (deftest config-test 7 | (is (match? 8 | '{:kaocha/tests [{:kaocha.testable/id :foo 9 | :kaocha/test-paths ["test/foo"]}] 10 | :kaocha/reporter [kaocha.report.progress/report] 11 | :kaocha/color? false 12 | :kaocha/fail-fast? true 13 | :kaocha/plugins [:kaocha.plugin/randomize 14 | :kaocha.plugin/filter 15 | :kaocha.plugin/capture-output 16 | :kaocha.plugin.alpha/xfail]} 17 | 18 | (repl/config {:config-file "fixtures/custom_config.edn"})))) 19 | 20 | (deftest extra-config-test 21 | (is (match? 22 | '{:kaocha/tests [{:kaocha.testable/id :foo 23 | :kaocha/test-paths ["test/foo"]}] 24 | :kaocha/reporter [kaocha.report.progress/report] 25 | :kaocha/color? true 26 | :kaocha/fail-fast? true 27 | :kaocha/plugins [:kaocha.plugin/randomize 28 | :kaocha.plugin/filter 29 | :kaocha.plugin/capture-output 30 | :kaocha.plugin.alpha/xfail]} 31 | (repl/config {:color? true :config-file "fixtures/custom_config.edn"})))) 32 | 33 | 34 | (deftest config-with-profile-test 35 | (testing "specifying a profile" 36 | (is (match? 37 | '{:kaocha/tests [{:kaocha.testable/id :unit 38 | :kaocha/test-paths ["test"]}] 39 | :kaocha/reporter kaocha.report.progress/report } 40 | (repl/config {:profile :test :config-file "test/unit/kaocha/config/loaded-test-profile.edn"})))) 41 | (testing "not specifying a profile" 42 | (is (match? 43 | '{:kaocha/tests [{:kaocha.testable/id :unit 44 | :kaocha/test-paths ["test"]}] 45 | :kaocha/reporter kaocha.report/documentation } 46 | (repl/config {:config-file "test/unit/kaocha/config/loaded-test-profile.edn"}))))) 47 | 48 | 49 | -------------------------------------------------------------------------------- /test/unit/kaocha/runner_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.runner-test 2 | (:require [clojure.test :as t :refer :all] 3 | [kaocha.runner :as runner] 4 | [kaocha.test-util :refer [with-out-err]])) 5 | 6 | (defn -main [& args] 7 | (with-out-err 8 | (apply #'runner/-main* args))) 9 | 10 | (deftest working-tools-cli?-test 11 | (is (#'runner/working-tools-cli?))) 12 | 13 | (deftest main-test 14 | (testing "--test-help" 15 | (let [{:keys [out err result]} (-main "--test-help")] 16 | (is (re-find #"USAGE:" out)) 17 | (is (= 0 result)))) 18 | 19 | (testing "unknown command line options" 20 | (let [{:keys [out err result]} (-main "--foo")] 21 | (is (= -1 result)) 22 | (is (re-find #"Unknown option: \"--foo\"\n" out)) 23 | (is (re-find #"USAGE:" out))))) 24 | -------------------------------------------------------------------------------- /test/unit/kaocha/testable_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.testable-test 2 | (:require [clojure.spec.alpha :as spec] 3 | [clojure.test :as t :refer :all] 4 | [kaocha.report :as report] 5 | [kaocha.testable :as testable] 6 | [kaocha.test-helper] 7 | [kaocha.test-factories :as f])) 8 | 9 | (spec/def :kaocha.type/unknown map?) 10 | 11 | (deftest load--default 12 | (is (thrown-ex-data? "No implementation of kaocha.testable/load for :kaocha.type/unknown" 13 | {:kaocha.error/reason :kaocha.error/missing-method, 14 | :kaocha.error/missing-method 'kaocha.testable/load, 15 | :kaocha/testable {:kaocha.testable/type :kaocha.type/unknown 16 | :kaocha.testable/id :foo 17 | :kaocha.testable/desc "foo"}} 18 | (testable/load {:kaocha.testable/type :kaocha.type/unknown 19 | :kaocha.testable/id :foo 20 | :kaocha.testable/desc "foo"})))) 21 | 22 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 23 | 24 | (deftest run--default 25 | (is (thrown-ex-data? "No implementation of kaocha.testable/run for :kaocha.type/unknown" 26 | {:kaocha.error/reason :kaocha.error/missing-method, 27 | :kaocha.error/missing-method 'kaocha.testable/run, 28 | :kaocha/testable #:kaocha.testable{:type :kaocha.type/unknown 29 | :id :foo 30 | :desc "foo"}} 31 | 32 | (testable/run {:kaocha.testable/type :kaocha.type/unknown 33 | :kaocha.testable/id :foo 34 | :kaocha.testable/desc "foo"} 35 | (f/test-plan {}))))) 36 | 37 | 38 | (deftest test-seq-test 39 | (is (= (testable/test-seq 40 | {:kaocha.testable/id :x/_1 41 | :kaocha/tests [{:kaocha.testable/id :y/_1} 42 | {:kaocha.testable/id :z/_1}]}) 43 | [{:kaocha.testable/id :x/_1, 44 | :kaocha/tests [#:kaocha.testable{:id :y/_1} 45 | #:kaocha.testable{:id :z/_1}]} 46 | #:kaocha.testable{:id :y/_1} 47 | #:kaocha.testable{:id :z/_1}]))) 48 | -------------------------------------------------------------------------------- /test/unit/kaocha/version_check_test.clj: -------------------------------------------------------------------------------- 1 | (ns kaocha.version-check-test 2 | (:require [clojure.test :refer :all] 3 | [kaocha.test-helper] 4 | [kaocha.version-check :as v])) 5 | 6 | (deftest earlier-version-throws 7 | (is (thrown-ex-data? 8 | "Kaocha requires Clojure 1.9 or later." 9 | {:kaocha/early-exit 251} 10 | (binding [*clojure-version* {:major 1 :minor 8}] 11 | (v/check-version-minimum 1 9))))) 12 | 13 | (deftest current-version-does-not-throw 14 | (is (nil? (binding [*clojure-version* {:major 1 :minor 9}] 15 | (v/check-version-minimum 1 9))))) 16 | 17 | (deftest version-2-does-not-throw 18 | (is (nil? (binding [*clojure-version* {:major 2 :minor 0}] 19 | (v/check-version-minimum 1 9))))) 20 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:plugins [:kaocha.plugin.alpha/info 3 | :profiling 4 | :print-invocations 5 | :hooks 6 | :notifier 7 | :kaocha.plugin/version-filter] 8 | 9 | :tests [{:id :unit 10 | :test-paths ["test/shared" 11 | "test/unit"]} 12 | {:id :integration 13 | :type :kaocha.type/cucumber 14 | :test-paths ["test/shared" 15 | "test/features"] 16 | :cucumber/glue-paths ["test/step_definitions"] 17 | :kaocha.filter/skip [plugins.notifier-plugin/enabling-desktop-notifications]}] 18 | 19 | :kaocha.hooks/pre-load [kaocha.assertions/load-assertions] 20 | 21 | :kaocha/bindings {kaocha.stacktrace/*stacktrace-filters* [] 22 | kaocha.stacktrace/*stacktrace-stop-list* []} 23 | 24 | :reporter kaocha.report/documentation} 25 | --------------------------------------------------------------------------------