├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── benchmarks ├── Main.elm ├── README.md ├── Snippets.elm └── elm-package.json ├── elm-package.json ├── src ├── Expect.elm ├── Float.elm ├── Fuzz.elm ├── Fuzz │ └── Internal.elm ├── RoseTree.elm ├── Test.elm ├── Test │ ├── Expectation.elm │ ├── Fuzz.elm │ ├── Internal.elm │ ├── Runner.elm │ └── Runner │ │ └── Failure.elm └── Util.elm └── tests ├── FloatWithinTests.elm ├── FuzzerTests.elm ├── Helpers.elm ├── Main.elm ├── README.md ├── Runner ├── Log.elm ├── String.elm └── String │ └── Format.elm ├── RunnerTests.elm ├── SeedTests.elm ├── Tests.elm ├── elm-package.json ├── package.json └── run-tests.js /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | node_modules/ 3 | elm-stuff/ 4 | docs/ 5 | *.html 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | cache: 4 | directories: 5 | - test/elm-stuff/build-artifacts 6 | - sysconfcpus 7 | 8 | os: 9 | - osx 10 | - linux 11 | 12 | env: 13 | matrix: 14 | - ELM_VERSION=0.18.0 TARGET_NODE_VERSION=node 15 | - ELM_VERSION=0.18.0 TARGET_NODE_VERSION=4.0 16 | 17 | before_install: 18 | - if [ ${TRAVIS_OS_NAME} == "osx" ]; 19 | then brew update; brew install nvm; mkdir ~/.nvm; export NVM_DIR=~/.nvm; source $(brew --prefix nvm)/nvm.sh; 20 | fi 21 | - echo -e "Host github.com\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config 22 | - | # epic build time improvement - see https://github.com/elm-lang/elm-compiler/issues/1473#issuecomment-245704142 23 | if [ ! -d sysconfcpus/bin ]; 24 | then 25 | git clone https://github.com/obmarg/libsysconfcpus.git; 26 | cd libsysconfcpus; 27 | ./configure --prefix=$TRAVIS_BUILD_DIR/sysconfcpus; 28 | make && make install; 29 | cd ..; 30 | fi 31 | 32 | install: 33 | - nvm install $TARGET_NODE_VERSION 34 | - nvm use $TARGET_NODE_VERSION 35 | - node --version 36 | - npm --version 37 | - cd tests 38 | - npm install -g elm@$ELM_VERSION elm-test 39 | - mv $(npm config get prefix)/bin/elm-make $(npm config get prefix)/bin/elm-make-old 40 | - printf '%s\n\n' '#!/bin/bash' 'echo "Running elm-make with sysconfcpus -n 2"' '$TRAVIS_BUILD_DIR/sysconfcpus/bin/sysconfcpus -n 2 elm-make-old "$@"' > $(npm config get prefix)/bin/elm-make 41 | - chmod +x $(npm config get prefix)/bin/elm-make 42 | - npm install 43 | - elm package install --yes 44 | 45 | script: 46 | - npm test 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2017 the Elm-test contributors 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of elm-test nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Moved to [elm-explorations/test](https://github.com/elm-explorations/test) 2 | -------------------------------------------------------------------------------- /benchmarks/Main.elm: -------------------------------------------------------------------------------- 1 | module Main exposing (..) 2 | 3 | import Benchmark exposing (..) 4 | import Benchmark.Runner as Runner 5 | import Expect exposing (Expectation) 6 | import Random.Pcg 7 | import Snippets 8 | import Test.Internal exposing (Test(Labeled, Test)) 9 | 10 | 11 | main : Runner.BenchmarkProgram 12 | main = 13 | Runner.program suite 14 | 15 | 16 | suite : Benchmark 17 | suite = 18 | describe "Fuzz" 19 | [ describe "int" 20 | [ benchmark "generating" (benchTest Snippets.intPass) 21 | , benchmark "shrinking" (benchTest Snippets.intFail) 22 | ] 23 | , describe "intRange" 24 | [ benchmark "generating" (benchTest Snippets.intRangePass) 25 | , benchmark "shrinking" (benchTest Snippets.intRangeFail) 26 | ] 27 | , describe "string" 28 | [ benchmark "generating" (benchTest Snippets.stringPass) 29 | , benchmark "shrinking" (benchTest Snippets.stringFail) 30 | ] 31 | , describe "float" 32 | [ benchmark "generating" (benchTest Snippets.floatPass) 33 | , benchmark "shrinking" (benchTest Snippets.floatFail) 34 | ] 35 | , describe "bool" 36 | [ benchmark "generating" (benchTest Snippets.boolPass) 37 | , benchmark "shrinking" (benchTest Snippets.boolFail) 38 | ] 39 | , describe "char" 40 | [ benchmark "generating" (benchTest Snippets.charPass) 41 | , benchmark "shrinking" (benchTest Snippets.charFail) 42 | ] 43 | , describe "list of int" 44 | [ benchmark "generating" (benchTest Snippets.listIntPass) 45 | , benchmark "shrinking" (benchTest Snippets.listIntFail) 46 | ] 47 | , describe "maybe of int" 48 | [ benchmark "generating" (benchTest Snippets.maybeIntPass) 49 | , benchmark "shrinking" (benchTest Snippets.maybeIntFail) 50 | ] 51 | , describe "result of string and int" 52 | [ benchmark "generating" (benchTest Snippets.resultPass) 53 | , benchmark "shrinking" (benchTest Snippets.resultFail) 54 | ] 55 | , describe "map" 56 | [ benchmark "generating" (benchTest Snippets.mapPass) 57 | , benchmark "shrinking" (benchTest Snippets.mapFail) 58 | ] 59 | , describe "andMap" 60 | [ benchmark "generating" (benchTest Snippets.andMapPass) 61 | , benchmark "shrinking" (benchTest Snippets.andMapFail) 62 | ] 63 | , describe "map5" 64 | [ benchmark "generating" (benchTest Snippets.map5Pass) 65 | , benchmark "shrinking" (benchTest Snippets.map5Fail) 66 | ] 67 | , describe "andThen" 68 | [ benchmark "generating" (benchTest Snippets.andThenPass) 69 | , benchmark "shrinking" (benchTest Snippets.andThenFail) 70 | ] 71 | , describe "conditional" 72 | [ benchmark "generating" (benchTest Snippets.conditionalPass) 73 | , benchmark "shrinking" (benchTest Snippets.conditionalFail) 74 | ] 75 | ] 76 | 77 | 78 | benchTest : Test -> (() -> List Expectation) 79 | benchTest test = 80 | case test of 81 | Test fn -> 82 | \_ -> fn (Random.Pcg.initialSeed 0) 10 83 | 84 | Labeled _ test -> 85 | benchTest test 86 | 87 | test -> 88 | Debug.crash <| "No support for benchmarking this type of test: " ++ toString test 89 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # Benchmarks for elm-test 2 | 3 | These are some benchmarks of the elm-test library built using the excellent [elm-benchmark](https://github.com/BrianHicks/elm-benchmark). 4 | 5 | ## How to run 6 | 7 | ```sh 8 | cd ./benchmarks 9 | elm-make Main.elm 10 | open index.html 11 | ``` 12 | 13 | ## How to use 14 | These benchmarks can help get an idea of the performance impact of a change in the elm-test code. 15 | Beware however that a fifty percent performance increase in these benchmarks will most likely not translate to a fifty percent faster tests for users. 16 | In real word scenario's the execution of the test body will have a significant impact on the running time of the test suite, an aspect we're not testing here because it's different for every test suite. 17 | To get a feeling for the impact your change has on actual test run times try running some real test suites with and without your changes. 18 | 19 | ## Benchmarking complete test suites 20 | These are some examples of test suites that contain a lot of fuzzer tests: 21 | - [elm-benchmark](https://github.com/BrianHicks/elm-benchmark) 22 | - [elm-nonempty-list](https://github.com/mgold/elm-nonempty-list) 23 | - [json-elm-schema](https://github.com/NoRedInk/json-elm-schema) 24 | 25 | A tool you can use for benchmarking the suite is [bench](https://github.com/Gabriel439/bench). 26 | 27 | To run the tests using your modified code (this only works if your modified version is backwards compatible with the version of elm-test currenlty in use by the test suite): 28 | - In your test suite directories `elm-package.json`: 29 | - Remove the dependency on `elm-test`. 30 | - Add dependecies of `elm-test` as dependencies of the test suite itself. 31 | - Add the path to your changed elm-test src directory to your `source-directories`. 32 | It will be something like `//elm-test/src`. 33 | - Run `elm-test` once to trigger compilation. 34 | - Now run `elm-test` with your benchmarking tool. 35 | -------------------------------------------------------------------------------- /benchmarks/Snippets.elm: -------------------------------------------------------------------------------- 1 | module Snippets exposing (..) 2 | 3 | import Expect exposing (Expectation) 4 | import Fuzz exposing (Fuzzer) 5 | import Test exposing (Test, fuzz) 6 | 7 | 8 | intPass : Test 9 | intPass = 10 | fuzz Fuzz.int "(passes) int" <| 11 | \_ -> 12 | Expect.pass 13 | 14 | 15 | intFail : Test 16 | intFail = 17 | fuzz Fuzz.int "(fails) int" <| 18 | \numbers -> 19 | Expect.fail "Failed" 20 | 21 | 22 | intRangePass : Test 23 | intRangePass = 24 | fuzz (Fuzz.intRange 10 100) "(passes) intRange" <| 25 | \_ -> 26 | Expect.pass 27 | 28 | 29 | intRangeFail : Test 30 | intRangeFail = 31 | fuzz (Fuzz.intRange 10 100) "(fails) intRange" <| 32 | \numbers -> 33 | Expect.fail "Failed" 34 | 35 | 36 | stringPass : Test 37 | stringPass = 38 | fuzz Fuzz.string "(passes) string" <| 39 | \_ -> 40 | Expect.pass 41 | 42 | 43 | stringFail : Test 44 | stringFail = 45 | fuzz Fuzz.string "(fails) string" <| 46 | \numbers -> 47 | Expect.fail "Failed" 48 | 49 | 50 | floatPass : Test 51 | floatPass = 52 | fuzz Fuzz.float "(passes) float" <| 53 | \_ -> 54 | Expect.pass 55 | 56 | 57 | floatFail : Test 58 | floatFail = 59 | fuzz Fuzz.float "(fails) float" <| 60 | \numbers -> 61 | Expect.fail "Failed" 62 | 63 | 64 | boolPass : Test 65 | boolPass = 66 | fuzz Fuzz.bool "(passes) bool" <| 67 | \_ -> 68 | Expect.pass 69 | 70 | 71 | boolFail : Test 72 | boolFail = 73 | fuzz Fuzz.bool "(fails) bool" <| 74 | \numbers -> 75 | Expect.fail "Failed" 76 | 77 | 78 | charPass : Test 79 | charPass = 80 | fuzz Fuzz.char "(passes) char" <| 81 | \_ -> 82 | Expect.pass 83 | 84 | 85 | charFail : Test 86 | charFail = 87 | fuzz Fuzz.char "(fails) char" <| 88 | \numbers -> 89 | Expect.fail "Failed" 90 | 91 | 92 | listIntPass : Test 93 | listIntPass = 94 | fuzz (Fuzz.list Fuzz.int) "(passes) list of int" <| 95 | \_ -> 96 | Expect.pass 97 | 98 | 99 | listIntFail : Test 100 | listIntFail = 101 | fuzz (Fuzz.list Fuzz.int) "(fails) list of int" <| 102 | {- The empty list is the first value the list shrinker will try. 103 | If we immediately fail on that example than we're not doing a lot of shrinking. 104 | -} 105 | Expect.notEqual [] 106 | 107 | 108 | maybeIntPass : Test 109 | maybeIntPass = 110 | fuzz (Fuzz.maybe Fuzz.int) "(passes) maybe of int" <| 111 | \_ -> 112 | Expect.pass 113 | 114 | 115 | maybeIntFail : Test 116 | maybeIntFail = 117 | fuzz (Fuzz.maybe Fuzz.int) "(fails) maybe of int" <| 118 | \numbers -> 119 | Expect.fail "Failed" 120 | 121 | 122 | resultPass : Test 123 | resultPass = 124 | fuzz (Fuzz.result Fuzz.string Fuzz.int) "(passes) result of string and int" <| 125 | \_ -> 126 | Expect.pass 127 | 128 | 129 | resultFail : Test 130 | resultFail = 131 | fuzz (Fuzz.result Fuzz.string Fuzz.int) "(fails) result of string and int" <| 132 | \numbers -> 133 | Expect.fail "Failed" 134 | 135 | 136 | mapPass : Test 137 | mapPass = 138 | fuzz even "(passes) map" <| 139 | \_ -> Expect.pass 140 | 141 | 142 | mapFail : Test 143 | mapFail = 144 | fuzz even "(fails) map" <| 145 | \_ -> Expect.fail "Failed" 146 | 147 | 148 | andMapPass : Test 149 | andMapPass = 150 | fuzz person "(passes) andMap" <| 151 | \_ -> Expect.pass 152 | 153 | 154 | andMapFail : Test 155 | andMapFail = 156 | fuzz person "(fails) andMap" <| 157 | \_ -> Expect.fail "Failed" 158 | 159 | 160 | map5Pass : Test 161 | map5Pass = 162 | fuzz person2 "(passes) map5" <| 163 | \_ -> Expect.pass 164 | 165 | 166 | map5Fail : Test 167 | map5Fail = 168 | fuzz person2 "(fails) map5" <| 169 | \_ -> Expect.fail "Failed" 170 | 171 | 172 | andThenPass : Test 173 | andThenPass = 174 | fuzz (variableList 2 5 Fuzz.int) "(passes) andThen" <| 175 | \_ -> Expect.pass 176 | 177 | 178 | andThenFail : Test 179 | andThenFail = 180 | fuzz (variableList 2 5 Fuzz.int) "(fails) andThen" <| 181 | \_ -> Expect.fail "Failed" 182 | 183 | 184 | conditionalPass : Test 185 | conditionalPass = 186 | fuzz evenWithConditional "(passes) conditional" <| 187 | \_ -> Expect.pass 188 | 189 | 190 | conditionalFail : Test 191 | conditionalFail = 192 | fuzz evenWithConditional "(fails) conditional" <| 193 | \_ -> Expect.fail "Failed" 194 | 195 | 196 | type alias Person = 197 | { firstName : String 198 | , lastName : String 199 | , age : Int 200 | , nationality : String 201 | , height : Float 202 | } 203 | 204 | 205 | person : Fuzzer Person 206 | person = 207 | Fuzz.map Person Fuzz.string 208 | |> Fuzz.andMap Fuzz.string 209 | |> Fuzz.andMap Fuzz.int 210 | |> Fuzz.andMap Fuzz.string 211 | |> Fuzz.andMap Fuzz.float 212 | 213 | 214 | person2 : Fuzzer Person 215 | person2 = 216 | Fuzz.map5 Person 217 | Fuzz.string 218 | Fuzz.string 219 | Fuzz.int 220 | Fuzz.string 221 | Fuzz.float 222 | 223 | 224 | even : Fuzzer Int 225 | even = 226 | Fuzz.map ((*) 2) Fuzz.int 227 | 228 | 229 | variableList : Int -> Int -> Fuzzer a -> Fuzzer (List a) 230 | variableList min max item = 231 | Fuzz.intRange min max 232 | |> Fuzz.andThen (\length -> List.repeat length item |> sequence) 233 | 234 | 235 | sequence : List (Fuzzer a) -> Fuzzer (List a) 236 | sequence fuzzers = 237 | List.foldl 238 | (Fuzz.map2 (::)) 239 | (Fuzz.constant []) 240 | fuzzers 241 | 242 | 243 | evenWithConditional : Fuzzer Int 244 | evenWithConditional = 245 | Fuzz.intRange 1 6 246 | |> Fuzz.conditional 247 | { retries = 3 248 | , fallback = (+) 1 249 | , condition = \n -> (n % 2) == 0 250 | } 251 | -------------------------------------------------------------------------------- /benchmarks/elm-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0.0", 3 | "summary": "helpful summary of your project, less than 80 characters", 4 | "repository": "https://github.com/user/project.git", 5 | "license": "BSD3", 6 | "source-directories": [ 7 | ".", 8 | "../src" 9 | ], 10 | "exposed-modules": [], 11 | "dependencies": { 12 | "BrianHicks/elm-benchmark": "1.0.2 <= v < 2.0.0", 13 | "eeue56/elm-lazy-list": "1.0.0 <= v < 2.0.0", 14 | "eeue56/elm-shrink": "1.0.0 <= v < 2.0.0", 15 | "elm-lang/core": "5.1.1 <= v < 6.0.0", 16 | "elm-lang/html": "2.0.0 <= v < 3.0.0", 17 | "eeue56/elm-lazy": "1.0.0 <= v < 2.0.0", 18 | "mgold/elm-random-pcg": "5.0.0 <= v < 6.0.0" 19 | }, 20 | "elm-version": "0.18.0 <= v < 0.19.0" 21 | } 22 | -------------------------------------------------------------------------------- /elm-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "4.2.0", 3 | "summary": "Unit and Fuzz testing support with Console/Html/String outputs.", 4 | "repository": "https://github.com/elm-community/elm-test.git", 5 | "license": "BSD-3-Clause", 6 | "source-directories": [ 7 | "src" 8 | ], 9 | "exposed-modules": [ 10 | "Test", 11 | "Test.Runner", 12 | "Test.Runner.Failure", 13 | "Expect", 14 | "Fuzz" 15 | ], 16 | "dependencies": { 17 | "eeue56/elm-lazy": "1.0.0 <= v < 2.0.0", 18 | "eeue56/elm-lazy-list": "1.0.0 <= v < 2.0.0", 19 | "eeue56/elm-shrink": "1.0.0 <= v < 2.0.0", 20 | "elm-lang/core": "5.0.0 <= v < 6.0.0", 21 | "mgold/elm-random-pcg": "4.0.2 <= v < 6.0.0" 22 | }, 23 | "elm-version": "0.18.0 <= v < 0.19.0" 24 | } 25 | -------------------------------------------------------------------------------- /src/Expect.elm: -------------------------------------------------------------------------------- 1 | module Expect 2 | exposing 3 | ( Expectation 4 | , FloatingPointTolerance(Absolute, AbsoluteOrRelative, Relative) 5 | , all 6 | , atLeast 7 | , atMost 8 | , equal 9 | , equalDicts 10 | , equalLists 11 | , equalSets 12 | , err 13 | , fail 14 | , greaterThan 15 | , lessThan 16 | , notEqual 17 | , notWithin 18 | , onFail 19 | , pass 20 | , within 21 | ) 22 | 23 | {-| A library to create `Expectation`s, which describe a claim to be tested. 24 | 25 | 26 | ## Quick Reference 27 | 28 | - [`equal`](#equal) `(arg2 == arg1)` 29 | - [`notEqual`](#notEqual) `(arg2 /= arg1)` 30 | - [`lessThan`](#lessThan) `(arg2 < arg1)` 31 | - [`atMost`](#atMost) `(arg2 <= arg1)` 32 | - [`greaterThan`](#greaterThan) `(arg2 > arg1)` 33 | - [`atLeast`](#atLeast) `(arg2 >= arg1)` 34 | - [Floating Point Comparisons](#floating-point-comparisons) 35 | 36 | 37 | ## Basic Expectations 38 | 39 | @docs Expectation, equal, notEqual, all 40 | 41 | 42 | ## Numeric Comparisons 43 | 44 | @docs lessThan, atMost, greaterThan, atLeast 45 | 46 | 47 | ## Floating Point Comparisons 48 | 49 | These functions allow you to compare `Float` values up to a specified rounding error, which may be relative, absolute, 50 | or both. For an in-depth look, see our [Guide to Floating Point Comparison](#guide-to-floating-point-comparison). 51 | 52 | @docs FloatingPointTolerance, within, notWithin 53 | 54 | 55 | ## Collections 56 | 57 | @docs err, equalLists, equalDicts, equalSets 58 | 59 | 60 | ## Customizing 61 | 62 | These functions will let you build your own expectations. 63 | 64 | @docs pass, fail, onFail 65 | 66 | 67 | ## Guide to Floating Point Comparison 68 | 69 | In general, if you are multiplying, you want relative tolerance, and if you're adding, 70 | you want absolute tolerance. If you are doing both, you want both kinds of tolerance, 71 | or to split the calculation into smaller parts for testing. 72 | 73 | 74 | ### Absolute Tolerance 75 | 76 | Let's say we want to figure out if our estimation of pi is precise enough. 77 | 78 | Is `3.14` within `0.01` of `pi`? Yes, because `3.13 < pi < 3.15`. 79 | 80 | test "3.14 approximates pi with absolute precision" <| \_ -> 81 | 3.14 |> Expect.within (Absolute 0.01) pi 82 | 83 | 84 | ### Relative Tolerance 85 | 86 | What if we also want to know if our circle circumference estimation is close enough? 87 | 88 | Let's say our circle has a radius of `r` meters. The formula for circle circumference is `C=2*r*pi`. 89 | To make the calculations a bit easier ([ahem](https://tauday.com/tau-manifesto)), we'll look at half the circumference; `C/2=r*pi`. 90 | Is `r * 3.14` within `0.01` of `r * pi`? 91 | That depends, what does `r` equal? If `r` is `0.01`mm, or `0.00001` meters, we're comparing 92 | `0.00001 * 3.14 - 0.01 < r * pi < 0.00001 * 3.14 + 0.01` or `-0.0099686 < 0.0000314159 < 0.0100314`. 93 | That's a huge tolerance! A circumference that is _a thousand times longer_ than we expected would pass that test! 94 | 95 | On the other hand, if `r` is very large, we're going to need many more digits of pi. 96 | For an absolute tolerance of `0.01` and a pi estimation of `3.14`, this expectation only passes if `r < 2*pi`. 97 | 98 | If we use a relative tolerance of `0.01` instead, the circle area comparison becomes much better. Is `r * 3.14` within 99 | `1%` of `r * pi`? Yes! In fact, three digits of pi approximation is always good enough for a 0.1% relative tolerance, 100 | as long as `r` isn't [too close to zero](https://en.wikipedia.org/wiki/Denormal_number). 101 | 102 | fuzz 103 | (floatRange 0.000001 100000) 104 | "Circle half-circumference with relative tolerance" 105 | (\r -> r * 3.14 |> Expect.within (Relative 0.001) (r * pi)) 106 | 107 | 108 | ### Trouble with Numbers Near Zero 109 | 110 | If you are adding things near zero, you probably want absolute tolerance. If you're comparing values between `-1` and `1`, you should consider using absolute tolerance. 111 | 112 | For example: Is `1 + 2 - 3` within `1%` of `0`? Well, if `1`, `2` and `3` have any amount of rounding error, you might not get exactly zero. What is `1%` above and below `0`? Zero. We just lost all tolerance. Even if we hard-code the numbers, we might not get exactly zero; `0.1 + 0.2` rounds to a value just above `0.3`, since computers, counting in binary, cannot write down any of those three numbers using a finite number of digits, just like we cannot write `0.333...` exactly in base 10. 113 | 114 | Another example is comparing values that are on either side of zero. `0.0001` is more than `100%` away from `-0.0001`. In fact, `infinity` is closer to `0.0001` than `0.0001` is to `-0.0001`, if you are using a relative tolerance. Twice as close, actually. So even though both `0.0001` and `-0.0001` could be considered very close to zero, they are very far apart relative to each other. The same argument applies for any number of zeroes. 115 | 116 | -} 117 | 118 | import Dict exposing (Dict) 119 | import Set exposing (Set) 120 | import Test.Expectation 121 | import Test.Runner.Failure exposing (InvalidReason(..), Reason(..)) 122 | 123 | 124 | {-| The result of a single test run: either a [`pass`](#pass) or a 125 | [`fail`](#fail). 126 | -} 127 | type alias Expectation = 128 | Test.Expectation.Expectation 129 | 130 | 131 | {-| Passes if the arguments are equal. 132 | 133 | Expect.equal 0 (List.length []) 134 | 135 | -- Passes because (0 == 0) is True 136 | 137 | Failures resemble code written in pipeline style, so you can tell 138 | which argument is which: 139 | 140 | -- Fails because the expected value didn't split the space in "Betty Botter" 141 | String.split " " "Betty Botter bought some butter" 142 | |> Expect.equal [ "Betty Botter", "bought", "some", "butter" ] 143 | 144 | {- 145 | 146 | [ "Betty", "Botter", "bought", "some", "butter" ] 147 | ╷ 148 | │ Expect.equal 149 | ╵ 150 | [ "Betty Botter", "bought", "some", "butter" ] 151 | 152 | -} 153 | 154 | -} 155 | equal : a -> a -> Expectation 156 | equal = 157 | equateWith "Expect.equal" (==) 158 | 159 | 160 | {-| Passes if the arguments are not equal. 161 | 162 | -- Passes because (11 /= 100) is True 163 | 90 + 10 164 | |> Expect.notEqual 11 165 | 166 | 167 | -- Fails because (100 /= 100) is False 168 | 90 + 10 169 | |> Expect.notEqual 100 170 | 171 | {- 172 | 173 | 100 174 | ╷ 175 | │ Expect.notEqual 176 | ╵ 177 | 100 178 | 179 | -} 180 | 181 | -} 182 | notEqual : a -> a -> Expectation 183 | notEqual = 184 | equateWith "Expect.notEqual" (/=) 185 | 186 | 187 | {-| Passes if the second argument is less than the first. 188 | 189 | Expect.lessThan 1 (List.length []) 190 | 191 | -- Passes because (0 < 1) is True 192 | 193 | Failures resemble code written in pipeline style, so you can tell 194 | which argument is which: 195 | 196 | -- Fails because (0 < -1) is False 197 | List.length [] 198 | |> Expect.lessThan -1 199 | 200 | 201 | {- 202 | 203 | 0 204 | ╷ 205 | │ Expect.lessThan 206 | ╵ 207 | -1 208 | 209 | -} 210 | 211 | -} 212 | lessThan : comparable -> comparable -> Expectation 213 | lessThan = 214 | compareWith "Expect.lessThan" (<) 215 | 216 | 217 | {-| Passes if the second argument is less than or equal to the first. 218 | 219 | Expect.atMost 1 (List.length []) 220 | 221 | -- Passes because (0 <= 1) is True 222 | 223 | Failures resemble code written in pipeline style, so you can tell 224 | which argument is which: 225 | 226 | -- Fails because (0 <= -3) is False 227 | List.length [] 228 | |> Expect.atMost -3 229 | 230 | {- 231 | 232 | 0 233 | ╷ 234 | │ Expect.atMost 235 | ╵ 236 | -3 237 | 238 | -} 239 | 240 | -} 241 | atMost : comparable -> comparable -> Expectation 242 | atMost = 243 | compareWith "Expect.atMost" (<=) 244 | 245 | 246 | {-| Passes if the second argument is greater than the first. 247 | 248 | Expect.greaterThan -2 List.length [] 249 | 250 | -- Passes because (0 > -2) is True 251 | 252 | Failures resemble code written in pipeline style, so you can tell 253 | which argument is which: 254 | 255 | -- Fails because (0 > 1) is False 256 | List.length [] 257 | |> Expect.greaterThan 1 258 | 259 | {- 260 | 261 | 0 262 | ╷ 263 | │ Expect.greaterThan 264 | ╵ 265 | 1 266 | 267 | -} 268 | 269 | -} 270 | greaterThan : comparable -> comparable -> Expectation 271 | greaterThan = 272 | compareWith "Expect.greaterThan" (>) 273 | 274 | 275 | {-| Passes if the second argument is greater than or equal to the first. 276 | 277 | Expect.atLeast -2 (List.length []) 278 | 279 | -- Passes because (0 >= -2) is True 280 | 281 | Failures resemble code written in pipeline style, so you can tell 282 | which argument is which: 283 | 284 | -- Fails because (0 >= 3) is False 285 | List.length [] 286 | |> Expect.atLeast 3 287 | 288 | {- 289 | 290 | 0 291 | ╷ 292 | │ Expect.atLeast 293 | ╵ 294 | 3 295 | 296 | -} 297 | 298 | -} 299 | atLeast : comparable -> comparable -> Expectation 300 | atLeast = 301 | compareWith "Expect.atLeast" (>=) 302 | 303 | 304 | {-| A type to describe how close a floating point number must be to the expected value for the test to pass. This may be 305 | specified as absolute or relative. 306 | 307 | `AbsoluteOrRelative` tolerance uses a logical OR between the absolute (specified first) and relative tolerance. If you 308 | want a logical AND, use [`Expect.all`](#all). 309 | 310 | -} 311 | type FloatingPointTolerance 312 | = Absolute Float 313 | | Relative Float 314 | | AbsoluteOrRelative Float Float 315 | 316 | 317 | {-| Passes if the second and third arguments are equal within a tolerance 318 | specified by the first argument. This is intended to avoid failing because of 319 | minor inaccuracies introduced by floating point arithmetic. 320 | 321 | -- Fails because 0.1 + 0.2 == 0.30000000000000004 (0.1 is non-terminating in base 2) 322 | 0.1 + 0.2 |> Expect.equal 0.3 323 | 324 | -- So instead write this test, which passes 325 | 0.1 + 0.2 |> Expect.within (Absolute 0.000000001) 0.3 326 | 327 | Failures resemble code written in pipeline style, so you can tell 328 | which argument is which: 329 | 330 | -- Fails because 3.14 is not close enough to pi 331 | 3.14 |> Expect.within (Absolute 0.0001) pi 332 | 333 | {- 334 | 335 | 3.14 336 | ╷ 337 | │ Expect.within Absolute 0.0001 338 | ╵ 339 | 3.141592653589793 340 | 341 | -} 342 | 343 | -} 344 | within : FloatingPointTolerance -> Float -> Float -> Expectation 345 | within tolerance a b = 346 | nonNegativeToleranceError tolerance "within" <| 347 | compareWith ("Expect.within " ++ toString tolerance) 348 | (withinCompare tolerance) 349 | a 350 | b 351 | 352 | 353 | {-| Passes if (and only if) a call to `within` with the same arguments would have failed. 354 | -} 355 | notWithin : FloatingPointTolerance -> Float -> Float -> Expectation 356 | notWithin tolerance a b = 357 | nonNegativeToleranceError tolerance "notWithin" <| 358 | compareWith ("Expect.notWithin " ++ toString tolerance) 359 | (\a b -> not <| withinCompare tolerance a b) 360 | a 361 | b 362 | 363 | 364 | {-| Passes if the 365 | [`Result`](http://package.elm-lang.org/packages/elm-lang/core/latest/Result) is 366 | an `Err` rather than `Ok`. This is useful for tests where you expect to get an 367 | error but you don't care about what the actual error is. If your possibly 368 | erroring function returns a `Maybe`, simply use `Expect.equal Nothing`. 369 | 370 | -- Passes 371 | String.toInt "not an int" 372 | |> Expect.err 373 | 374 | Test failures will be printed with the unexpected `Ok` value contrasting with 375 | any `Err`. 376 | 377 | -- Fails 378 | String.toInt "20" 379 | |> Expect.err 380 | 381 | {- 382 | 383 | Ok 20 384 | ╷ 385 | │ Expect.err 386 | ╵ 387 | Err _ 388 | 389 | -} 390 | 391 | -} 392 | err : Result a b -> Expectation 393 | err result = 394 | case result of 395 | Ok _ -> 396 | { description = "Expect.err" 397 | , reason = Comparison "Err _" (toString result) 398 | } 399 | |> Test.Expectation.fail 400 | 401 | Err _ -> 402 | pass 403 | 404 | 405 | {-| Passes if the arguments are equal lists. 406 | 407 | -- Passes 408 | [1, 2, 3] 409 | |> Expect.equalLists [1, 2, 3] 410 | 411 | Failures resemble code written in pipeline style, so you can tell 412 | which argument is which, and reports which index the lists first 413 | differed at or which list was longer: 414 | 415 | -- Fails 416 | [ 1, 2, 4, 6 ] 417 | |> Expect.equalLists [ 1, 2, 5 ] 418 | 419 | {- 420 | 421 | [1,2,4,6] 422 | first diff at index index 2: +`4`, -`5` 423 | ╷ 424 | │ Expect.equalLists 425 | ╵ 426 | first diff at index index 2: +`5`, -`4` 427 | [1,2,5] 428 | 429 | -} 430 | 431 | -} 432 | equalLists : List a -> List a -> Expectation 433 | equalLists expected actual = 434 | if expected == actual then 435 | pass 436 | 437 | else 438 | { description = "Expect.equalLists" 439 | , reason = ListDiff (List.map toString expected) (List.map toString actual) 440 | } 441 | |> Test.Expectation.fail 442 | 443 | 444 | {-| Passes if the arguments are equal dicts. 445 | 446 | -- Passes 447 | (Dict.fromList [ ( 1, "one" ), ( 2, "two" ) ]) 448 | |> Expect.equalDicts (Dict.fromList [ ( 1, "one" ), ( 2, "two" ) ]) 449 | 450 | Failures resemble code written in pipeline style, so you can tell 451 | which argument is which, and reports which keys were missing from 452 | or added to each dict: 453 | 454 | -- Fails 455 | (Dict.fromList [ ( 1, "one" ), ( 2, "too" ) ]) 456 | |> Expect.equalDicts (Dict.fromList [ ( 1, "one" ), ( 2, "two" ), ( 3, "three" ) ]) 457 | 458 | {- 459 | 460 | Dict.fromList [(1,"one"),(2,"too")] 461 | diff: -[ (2,"two"), (3,"three") ] +[ (2,"too") ] 462 | ╷ 463 | │ Expect.equalDicts 464 | ╵ 465 | diff: +[ (2,"two"), (3,"three") ] -[ (2,"too") ] 466 | Dict.fromList [(1,"one"),(2,"two"),(3,"three")] 467 | 468 | -} 469 | 470 | -} 471 | equalDicts : Dict comparable a -> Dict comparable a -> Expectation 472 | equalDicts expected actual = 473 | if Dict.toList expected == Dict.toList actual then 474 | pass 475 | 476 | else 477 | let 478 | differ dict k v diffs = 479 | if Dict.get k dict == Just v then 480 | diffs 481 | 482 | else 483 | ( k, v ) :: diffs 484 | 485 | missingKeys = 486 | Dict.foldr (differ actual) [] expected 487 | 488 | extraKeys = 489 | Dict.foldr (differ expected) [] actual 490 | in 491 | reportCollectionFailure "Expect.equalDicts" expected actual missingKeys extraKeys 492 | 493 | 494 | {-| Passes if the arguments are equal sets. 495 | 496 | -- Passes 497 | (Set.fromList [1, 2]) 498 | |> Expect.equalSets (Set.fromList [1, 2]) 499 | 500 | Failures resemble code written in pipeline style, so you can tell 501 | which argument is which, and reports which keys were missing from 502 | or added to each set: 503 | 504 | -- Fails 505 | (Set.fromList [ 1, 2, 4, 6 ]) 506 | |> Expect.equalSets (Set.fromList [ 1, 2, 5 ]) 507 | 508 | {- 509 | 510 | Set.fromList [1,2,4,6] 511 | diff: -[ 5 ] +[ 4, 6 ] 512 | ╷ 513 | │ Expect.equalSets 514 | ╵ 515 | diff: +[ 5 ] -[ 4, 6 ] 516 | Set.fromList [1,2,5] 517 | 518 | -} 519 | 520 | -} 521 | equalSets : Set comparable -> Set comparable -> Expectation 522 | equalSets expected actual = 523 | if Set.toList expected == Set.toList actual then 524 | pass 525 | 526 | else 527 | let 528 | missingKeys = 529 | Set.diff expected actual 530 | |> Set.toList 531 | 532 | extraKeys = 533 | Set.diff actual expected 534 | |> Set.toList 535 | in 536 | reportCollectionFailure "Expect.equalSets" expected actual missingKeys extraKeys 537 | 538 | 539 | {-| Always passes. 540 | 541 | import Json.Decode exposing (decodeString, int) 542 | import Test exposing (test) 543 | import Expect 544 | 545 | 546 | test "Json.Decode.int can decode the number 42." <| 547 | \_ -> 548 | case decodeString int "42" of 549 | Ok _ -> 550 | Expect.pass 551 | 552 | Err err -> 553 | Expect.fail err 554 | 555 | -} 556 | pass : Expectation 557 | pass = 558 | Test.Expectation.Pass 559 | 560 | 561 | {-| Fails with the given message. 562 | 563 | import Json.Decode exposing (decodeString, int) 564 | import Test exposing (test) 565 | import Expect 566 | 567 | 568 | test "Json.Decode.int can decode the number 42." <| 569 | \_ -> 570 | case decodeString int "42" of 571 | Ok _ -> 572 | Expect.pass 573 | 574 | Err err -> 575 | Expect.fail err 576 | 577 | -} 578 | fail : String -> Expectation 579 | fail str = 580 | Test.Expectation.fail { description = str, reason = Custom } 581 | 582 | 583 | {-| If the given expectation fails, replace its failure message with a custom one. 584 | 585 | "something" 586 | |> Expect.equal "something else" 587 | |> Expect.onFail "thought those two strings would be the same" 588 | 589 | -} 590 | onFail : String -> Expectation -> Expectation 591 | onFail str expectation = 592 | case expectation of 593 | Test.Expectation.Pass -> 594 | expectation 595 | 596 | Test.Expectation.Fail failure -> 597 | Test.Expectation.Fail { failure | description = str, reason = Custom } 598 | 599 | 600 | {-| Passes if each of the given functions passes when applied to the subject. 601 | 602 | Passing an empty list is assumed to be a mistake, so `Expect.all []` 603 | will always return a failed expectation no matter what else it is passed. 604 | 605 | Expect.all 606 | [ Expect.greaterThan -2 607 | , Expect.lessThan 5 608 | ] 609 | (List.length []) 610 | -- Passes because (0 > -2) is True and (0 < 5) is also True 611 | 612 | Failures resemble code written in pipeline style, so you can tell 613 | which argument is which: 614 | 615 | -- Fails because (0 < -10) is False 616 | List.length [] 617 | |> Expect.all 618 | [ Expect.greaterThan -2 619 | , Expect.lessThan -10 620 | , Expect.equal 0 621 | ] 622 | {- 623 | 0 624 | ╷ 625 | │ Expect.lessThan 626 | ╵ 627 | -10 628 | -} 629 | 630 | -} 631 | all : List (subject -> Expectation) -> subject -> Expectation 632 | all list query = 633 | if List.isEmpty list then 634 | Test.Expectation.fail 635 | { reason = Invalid EmptyList 636 | , description = "Expect.all was given an empty list. You must make at least one expectation to have a valid test!" 637 | } 638 | 639 | else 640 | allHelp list query 641 | 642 | 643 | allHelp : List (subject -> Expectation) -> subject -> Expectation 644 | allHelp list query = 645 | case list of 646 | [] -> 647 | pass 648 | 649 | check :: rest -> 650 | case check query of 651 | Test.Expectation.Pass -> 652 | allHelp rest query 653 | 654 | outcome -> 655 | outcome 656 | 657 | 658 | 659 | {---- Private helper functions ----} 660 | 661 | 662 | reportFailure : String -> String -> String -> Expectation 663 | reportFailure comparison expected actual = 664 | { description = comparison 665 | , reason = Comparison (toString expected) (toString actual) 666 | } 667 | |> Test.Expectation.fail 668 | 669 | 670 | reportCollectionFailure : String -> a -> b -> List c -> List d -> Expectation 671 | reportCollectionFailure comparison expected actual missingKeys extraKeys = 672 | { description = comparison 673 | , reason = 674 | { expected = toString expected 675 | , actual = toString actual 676 | , extra = List.map toString extraKeys 677 | , missing = List.map toString missingKeys 678 | } 679 | |> CollectionDiff 680 | } 681 | |> Test.Expectation.fail 682 | 683 | 684 | {-| String arg is label, e.g. "Expect.equal". 685 | -} 686 | equateWith : String -> (a -> b -> Bool) -> b -> a -> Expectation 687 | equateWith = 688 | testWith Equality 689 | 690 | 691 | compareWith : String -> (a -> b -> Bool) -> b -> a -> Expectation 692 | compareWith = 693 | testWith Comparison 694 | 695 | 696 | testWith : (String -> String -> Reason) -> String -> (a -> b -> Bool) -> b -> a -> Expectation 697 | testWith makeReason label runTest expected actual = 698 | if runTest actual expected then 699 | pass 700 | 701 | else 702 | { description = label 703 | , reason = makeReason (toString expected) (toString actual) 704 | } 705 | |> Test.Expectation.fail 706 | 707 | 708 | 709 | {---- Private *floating point* helper functions ----} 710 | 711 | 712 | absolute : FloatingPointTolerance -> Float 713 | absolute tolerance = 714 | case tolerance of 715 | Absolute absolute -> 716 | absolute 717 | 718 | AbsoluteOrRelative absolute _ -> 719 | absolute 720 | 721 | _ -> 722 | 0 723 | 724 | 725 | relative : FloatingPointTolerance -> Float 726 | relative tolerance = 727 | case tolerance of 728 | Relative relative -> 729 | relative 730 | 731 | AbsoluteOrRelative _ relative -> 732 | relative 733 | 734 | _ -> 735 | 0 736 | 737 | 738 | nonNegativeToleranceError : FloatingPointTolerance -> String -> Expectation -> Expectation 739 | nonNegativeToleranceError tolerance name result = 740 | if absolute tolerance < 0 && relative tolerance < 0 then 741 | Test.Expectation.fail { description = "Expect." ++ name ++ " was given negative absolute and relative tolerances", reason = Custom } 742 | 743 | else if absolute tolerance < 0 then 744 | Test.Expectation.fail { description = "Expect." ++ name ++ " was given a negative absolute tolerance", reason = Custom } 745 | 746 | else if relative tolerance < 0 then 747 | Test.Expectation.fail { description = "Expect." ++ name ++ " was given a negative relative tolerance", reason = Custom } 748 | 749 | else 750 | result 751 | 752 | 753 | withinCompare : FloatingPointTolerance -> Float -> Float -> Bool 754 | withinCompare tolerance a b = 755 | let 756 | withinAbsoluteTolerance = 757 | a - absolute tolerance <= b && b <= a + absolute tolerance 758 | 759 | withinRelativeTolerance = 760 | (a - abs (a * relative tolerance) <= b && b <= a + abs (a * relative tolerance)) 761 | || (b - abs (b * relative tolerance) <= a && a <= b + abs (b * relative tolerance)) 762 | in 763 | (a == b) || withinAbsoluteTolerance || withinRelativeTolerance 764 | -------------------------------------------------------------------------------- /src/Float.elm: -------------------------------------------------------------------------------- 1 | module Float 2 | exposing 3 | ( epsilon 4 | , infinity 5 | , maxAbsValue 6 | , minAbsNormal 7 | , minAbsValue 8 | , nan 9 | ) 10 | 11 | {-| Float contains useful constants related to 64 bit floating point numbers, 12 | as specified in IEEE 754-2008. 13 | 14 | @docs epsilon, infinity, nan, minAbsNormal, minAbsValue, maxAbsValue 15 | 16 | -} 17 | 18 | 19 | {-| Largest possible rounding error in a single 64 bit floating point 20 | calculation on an x86-x64 CPU. Also known as the Machine Epsilon. 21 | 22 | If you do not know what tolerance you should use, use this number, multiplied 23 | by the number of floating point operations you're doing in your calculation. 24 | 25 | According to the [MSDN system.double.epsilon documentation] 26 | (), 27 | ARM has a machine epsilon that is too small to represent in a 64 bit float, 28 | so we're simply ignoring that. On phones, tablets, raspberry pi's and other 29 | devices with ARM chips, you might get slightly better precision than we assume 30 | here. 31 | 32 | -} 33 | epsilon : Float 34 | epsilon = 35 | 2.0 ^ -52 36 | 37 | 38 | {-| Positive infinity. Negative infinity is just -infinity. 39 | -} 40 | infinity : Float 41 | infinity = 42 | 1.0 / 0.0 43 | 44 | 45 | {-| Not a Number. NaN does not compare equal to anything, including itself. 46 | Any operation including NaN will result in NaN. 47 | -} 48 | nan : Float 49 | nan = 50 | 0.0 / 0.0 51 | 52 | 53 | {-| Smallest possible value which still has full precision. 54 | 55 | Values closer to zero are denormalized, which means that they are 56 | using some of the significant bits to emulate a slightly larger mantissa. 57 | The number of significant binary digits is proportional to the binary 58 | logarithm of the denormalized number; halving a denormalized number also 59 | halves the precision of that number. 60 | 61 | -} 62 | minAbsNormal : Float 63 | minAbsNormal = 64 | 2.0 ^ -1022 65 | 66 | 67 | {-| Smallest absolute value representable in a 64 bit float. 68 | -} 69 | minAbsValue : Float 70 | minAbsValue = 71 | 2.0 ^ -1074 72 | 73 | 74 | {-| Largest finite absolute value representable in a 64 bit float. 75 | -} 76 | maxAbsValue : Float 77 | maxAbsValue = 78 | (2.0 - epsilon) * 2.0 ^ 1023 79 | -------------------------------------------------------------------------------- /src/Fuzz.elm: -------------------------------------------------------------------------------- 1 | module Fuzz exposing (Fuzzer, andMap, andThen, array, bool, char, conditional, constant, custom, float, floatRange, frequency, int, intRange, invalid, list, map, map2, map3, map4, map5, maybe, oneOf, order, percentage, result, string, tuple, tuple3, tuple4, tuple5, unit) 2 | 3 | {-| This is a library of _fuzzers_ you can use to supply values to your fuzz 4 | tests. You can typically pick out which ones you need according to their types. 5 | 6 | A `Fuzzer a` knows how to create values of type `a` in two different ways. It 7 | can create them randomly, so that your test's expectations are run against many 8 | values. Fuzzers will often generate edge cases likely to find bugs. If the 9 | fuzzer can make your test fail, it also knows how to "shrink" that failing input 10 | into more minimal examples, some of which might also cause the tests to fail. In 11 | this way, fuzzers can usually find the smallest or simplest input that 12 | reproduces a bug. 13 | 14 | 15 | ## Common Fuzzers 16 | 17 | @docs bool, int, intRange, float, floatRange, percentage, string, maybe, result, list, array 18 | 19 | 20 | ## Working with Fuzzers 21 | 22 | @docs Fuzzer, oneOf, constant, map, map2, map3, map4, map5, andMap, andThen, frequency, conditional 23 | 24 | 25 | ## Tuple Fuzzers 26 | 27 | Instead of using a tuple, consider using `fuzzN`. 28 | @docs tuple, tuple3, tuple4, tuple5 29 | 30 | 31 | ## Uncommon Fuzzers 32 | 33 | @docs custom, char, unit, order, invalid 34 | 35 | -} 36 | 37 | import Array exposing (Array) 38 | import Char 39 | import Fuzz.Internal as Internal 40 | exposing 41 | ( Fuzzer 42 | , Valid 43 | , ValidFuzzer 44 | , combineValid 45 | , invalidReason 46 | ) 47 | import Lazy 48 | import Lazy.List exposing ((+++), LazyList) 49 | import Random.Pcg as Random exposing (Generator) 50 | import RoseTree exposing (RoseTree(..)) 51 | import Shrink exposing (Shrinker) 52 | import Util exposing (..) 53 | 54 | 55 | {-| The representation of fuzzers is opaque. Conceptually, a `Fuzzer a` 56 | consists of a way to randomly generate values of type `a`, and a way to shrink 57 | those values. 58 | -} 59 | type alias Fuzzer a = 60 | Internal.Fuzzer a 61 | 62 | 63 | {-| Build a custom `Fuzzer a` by providing a `Generator a` and a `Shrinker a`. 64 | Generators are defined in [`mgold/elm-random-pcg`](http://package.elm-lang.org/packages/mgold/elm-random-pcg/latest), 65 | which is not core's Random module but has a compatible interface. Shrinkers are 66 | defined in [`eeue56/elm-shrink`](http://package.elm-lang.org/packages/eeue56/elm-shrink/latest/). 67 | 68 | Here is an example for a record: 69 | 70 | import Random.Pcg as Random 71 | import Shrink 72 | 73 | type alias Position = 74 | { x : Int, y : Int } 75 | 76 | position : Fuzzer Position 77 | position = 78 | Fuzz.custom 79 | (Random.map2 Position (Random.int -100 100) (Random.int -100 100)) 80 | (\{ x, y } -> Shrink.map Position (Shrink.int x) |> Shrink.andMap (Shrink.int y)) 81 | 82 | Here is an example for a custom union type, assuming there is already a `genName : Generator String` defined: 83 | 84 | type Question 85 | = Name String 86 | | Age Int 87 | 88 | question = 89 | let 90 | generator = 91 | Random.bool 92 | |> Random.andThen 93 | (\b -> 94 | if b then 95 | Random.map Name genName 96 | else 97 | Random.map Age (Random.int 0 120) 98 | ) 99 | 100 | shrinker question = 101 | case question of 102 | Name n -> 103 | Shrink.string n |> Shrink.map Name 104 | 105 | Age i -> 106 | Shrink.int i |> Shrink.map Age 107 | in 108 | Fuzz.custom generator shrinker 109 | 110 | It is not possible to extract the generator and shrinker from an existing fuzzer. 111 | 112 | -} 113 | custom : Generator a -> Shrinker a -> Fuzzer a 114 | custom generator shrinker = 115 | let 116 | shrinkTree a = 117 | Rose a (Lazy.lazy <| \_ -> Lazy.force <| Lazy.List.map shrinkTree (shrinker a)) 118 | in 119 | Ok <| 120 | Random.map shrinkTree generator 121 | 122 | 123 | {-| A fuzzer for the unit value. Unit is a type with only one value, commonly 124 | used as a placeholder. 125 | -} 126 | unit : Fuzzer () 127 | unit = 128 | RoseTree.singleton () 129 | |> Random.constant 130 | |> Ok 131 | 132 | 133 | {-| A fuzzer for bool values. 134 | -} 135 | bool : Fuzzer Bool 136 | bool = 137 | custom Random.bool Shrink.bool 138 | 139 | 140 | {-| A fuzzer for order values. 141 | -} 142 | order : Fuzzer Order 143 | order = 144 | let 145 | intToOrder i = 146 | if i == 0 then 147 | LT 148 | else if i == 1 then 149 | EQ 150 | else 151 | GT 152 | in 153 | custom (Random.map intToOrder (Random.int 0 2)) Shrink.order 154 | 155 | 156 | {-| A fuzzer for int values. It will never produce `NaN`, `Infinity`, or `-Infinity`. 157 | 158 | It's possible for this fuzzer to generate any 32-bit integer, but it favors 159 | numbers between -50 and 50 and especially zero. 160 | 161 | -} 162 | int : Fuzzer Int 163 | int = 164 | let 165 | generator = 166 | Random.frequency 167 | [ ( 3, Random.int -50 50 ) 168 | , ( 0.2, Random.constant 0 ) 169 | , ( 1, Random.int 0 (Random.maxInt - Random.minInt) ) 170 | , ( 1, Random.int (Random.minInt - Random.maxInt) 0 ) 171 | ] 172 | in 173 | custom generator Shrink.int 174 | 175 | 176 | {-| A fuzzer for int values within between a given minimum and maximum value, 177 | inclusive. Shrunken values will also be within the range. 178 | 179 | Remember that [Random.maxInt](http://package.elm-lang.org/packages/elm-lang/core/latest/Random#maxInt) 180 | is the maximum possible int value, so you can do `intRange x Random.maxInt` to get all 181 | the ints x or bigger. 182 | 183 | -} 184 | intRange : Int -> Int -> Fuzzer Int 185 | intRange lo hi = 186 | if hi < lo then 187 | Err <| "Fuzz.intRange was given a lower bound of " ++ toString lo ++ " which is greater than the upper bound, " ++ toString hi ++ "." 188 | else 189 | custom 190 | (Random.frequency 191 | [ ( 8, Random.int lo hi ) 192 | , ( 1, Random.constant lo ) 193 | , ( 1, Random.constant hi ) 194 | ] 195 | ) 196 | (Shrink.keepIf (\i -> i >= lo && i <= hi) Shrink.int) 197 | 198 | 199 | {-| A fuzzer for float values. It will never produce `NaN`, `Infinity`, or `-Infinity`. 200 | 201 | It's possible for this fuzzer to generate any other floating-point value, but it 202 | favors numbers between -50 and 50, numbers between -1 and 1, and especially zero. 203 | 204 | -} 205 | float : Fuzzer Float 206 | float = 207 | let 208 | generator = 209 | Random.frequency 210 | [ ( 3, Random.float -50 50 ) 211 | , ( 0.5, Random.constant 0 ) 212 | , ( 1, Random.float -1 1 ) 213 | , ( 1, Random.float 0 (toFloat <| Random.maxInt - Random.minInt) ) 214 | , ( 1, Random.float (toFloat <| Random.minInt - Random.maxInt) 0 ) 215 | ] 216 | in 217 | custom generator Shrink.float 218 | 219 | 220 | {-| A fuzzer for float values within between a given minimum and maximum 221 | value, inclusive. Shrunken values will also be within the range. 222 | -} 223 | floatRange : Float -> Float -> Fuzzer Float 224 | floatRange lo hi = 225 | if hi < lo then 226 | Err <| "Fuzz.floatRange was given a lower bound of " ++ toString lo ++ " which is greater than the upper bound, " ++ toString hi ++ "." 227 | else 228 | custom 229 | (Random.frequency 230 | [ ( 8, Random.float lo hi ) 231 | , ( 1, Random.constant lo ) 232 | , ( 1, Random.constant hi ) 233 | ] 234 | ) 235 | (Shrink.keepIf (\i -> i >= lo && i <= hi) Shrink.float) 236 | 237 | 238 | {-| A fuzzer for percentage values. Generates random floats between `0.0` and 239 | `1.0`. It will test zero and one about 10% of the time each. 240 | -} 241 | percentage : Fuzzer Float 242 | percentage = 243 | let 244 | generator = 245 | Random.frequency 246 | [ ( 8, Random.float 0 1 ) 247 | , ( 1, Random.constant 0 ) 248 | , ( 1, Random.constant 1 ) 249 | ] 250 | in 251 | custom generator Shrink.float 252 | 253 | 254 | {-| A fuzzer for char values. Generates random ascii chars disregarding the control 255 | characters and the extended character set. 256 | -} 257 | char : Fuzzer Char 258 | char = 259 | custom asciiCharGenerator Shrink.character 260 | 261 | 262 | asciiCharGenerator : Generator Char 263 | asciiCharGenerator = 264 | Random.map Char.fromCode (Random.int 32 126) 265 | 266 | 267 | whitespaceCharGenerator : Generator Char 268 | whitespaceCharGenerator = 269 | Random.sample [ ' ', '\t', '\n' ] |> Random.map (Maybe.withDefault ' ') 270 | 271 | 272 | {-| Generates random printable ASCII strings of up to 1000 characters. 273 | 274 | Shorter strings are more common, especially the empty string. 275 | 276 | -} 277 | string : Fuzzer String 278 | string = 279 | let 280 | asciiGenerator : Generator String 281 | asciiGenerator = 282 | Random.frequency 283 | [ ( 3, Random.int 1 10 ) 284 | , ( 0.2, Random.constant 0 ) 285 | , ( 1, Random.int 11 50 ) 286 | , ( 1, Random.int 50 1000 ) 287 | ] 288 | |> Random.andThen (lengthString asciiCharGenerator) 289 | 290 | whitespaceGenerator : Generator String 291 | whitespaceGenerator = 292 | Random.int 1 10 293 | |> Random.andThen (lengthString whitespaceCharGenerator) 294 | in 295 | custom 296 | (Random.frequency 297 | [ ( 9, asciiGenerator ) 298 | , ( 1, whitespaceGenerator ) 299 | ] 300 | ) 301 | Shrink.string 302 | 303 | 304 | {-| Given a fuzzer of a type, create a fuzzer of a maybe for that type. 305 | -} 306 | maybe : Fuzzer a -> Fuzzer (Maybe a) 307 | maybe fuzzer = 308 | let 309 | toMaybe : Bool -> RoseTree a -> RoseTree (Maybe a) 310 | toMaybe useNothing tree = 311 | if useNothing then 312 | RoseTree.singleton Nothing 313 | else 314 | RoseTree.map Just tree |> RoseTree.addChild (RoseTree.singleton Nothing) 315 | in 316 | (Result.map << Random.map2 toMaybe) (Random.oneIn 4) fuzzer 317 | 318 | 319 | {-| Given fuzzers for an error type and a success type, create a fuzzer for 320 | a result. 321 | -} 322 | result : Fuzzer error -> Fuzzer value -> Fuzzer (Result error value) 323 | result fuzzerError fuzzerValue = 324 | let 325 | toResult : Bool -> RoseTree error -> RoseTree value -> RoseTree (Result error value) 326 | toResult useError errorTree valueTree = 327 | if useError then 328 | RoseTree.map Err errorTree 329 | else 330 | RoseTree.map Ok valueTree 331 | in 332 | (Result.map2 <| Random.map3 toResult (Random.oneIn 4)) fuzzerError fuzzerValue 333 | 334 | 335 | {-| Given a fuzzer of a type, create a fuzzer of a list of that type. 336 | Generates random lists of varying length, favoring shorter lists. 337 | -} 338 | list : Fuzzer a -> Fuzzer (List a) 339 | list fuzzer = 340 | let 341 | genLength = 342 | Random.frequency 343 | [ ( 1, Random.constant 0 ) 344 | , ( 1, Random.constant 1 ) 345 | , ( 3, Random.int 2 10 ) 346 | , ( 2, Random.int 10 100 ) 347 | , ( 0.5, Random.int 100 400 ) 348 | ] 349 | in 350 | fuzzer 351 | |> Result.map 352 | (\validFuzzer -> 353 | genLength 354 | |> Random.andThen (flip Random.list validFuzzer) 355 | |> Random.map listShrinkHelp 356 | ) 357 | 358 | 359 | listShrinkHelp : List (RoseTree a) -> RoseTree (List a) 360 | listShrinkHelp listOfTrees = 361 | {- This extends listShrinkRecurse algorithm with an attempt to shrink directly to the empty list. -} 362 | listShrinkRecurse listOfTrees 363 | |> mapChildren (Lazy.List.cons <| RoseTree.singleton []) 364 | 365 | 366 | mapChildren : (LazyList (RoseTree a) -> LazyList (RoseTree a)) -> RoseTree a -> RoseTree a 367 | mapChildren fn (Rose root children) = 368 | Rose root (fn children) 369 | 370 | 371 | listShrinkRecurse : List (RoseTree a) -> RoseTree (List a) 372 | listShrinkRecurse listOfTrees = 373 | {- Shrinking a list of RoseTrees 374 | We need to do two things. First, shrink individual values. Second, shorten the list. 375 | To shrink individual values, we create every list copy of the input list where any 376 | one value is replaced by a shrunken form. 377 | To shorten the length of the list, remove elements at various positions in the list. 378 | In all cases, recurse! The goal is to make a little forward progress and then recurse. 379 | -} 380 | let 381 | n = 382 | List.length listOfTrees 383 | 384 | root = 385 | List.map RoseTree.root listOfTrees 386 | 387 | dropFirstHalf : List (RoseTree a) -> RoseTree (List a) 388 | dropFirstHalf list_ = 389 | List.drop (List.length list_ // 2) list_ 390 | |> listShrinkRecurse 391 | 392 | dropSecondHalf : List (RoseTree a) -> RoseTree (List a) 393 | dropSecondHalf list_ = 394 | List.take (List.length list_ // 2) list_ 395 | |> listShrinkRecurse 396 | 397 | halved : LazyList (RoseTree (List a)) 398 | halved = 399 | -- The list halving shortcut is useful only for large lists. 400 | -- For small lists attempting to remove elements one by one is good enough. 401 | if n >= 8 then 402 | Lazy.lazy <| 403 | \_ -> 404 | Lazy.List.fromList [ dropFirstHalf listOfTrees, dropSecondHalf listOfTrees ] 405 | |> Lazy.force 406 | else 407 | Lazy.List.empty 408 | 409 | shrinkOne prefix list = 410 | case list of 411 | [] -> 412 | Lazy.List.empty 413 | 414 | (Rose x shrunkenXs) :: more -> 415 | Lazy.List.map (\childTree -> prefix ++ (childTree :: more) |> listShrinkRecurse) shrunkenXs 416 | 417 | shrunkenVals = 418 | Lazy.lazy <| 419 | \_ -> 420 | Lazy.List.numbers 421 | |> Lazy.List.map (\i -> i - 1) 422 | |> Lazy.List.take n 423 | |> Lazy.List.andThen 424 | (\i -> shrinkOne (List.take i listOfTrees) (List.drop i listOfTrees)) 425 | |> Lazy.force 426 | 427 | shortened = 428 | Lazy.lazy <| 429 | \_ -> 430 | List.range 0 (n - 1) 431 | |> Lazy.List.fromList 432 | |> Lazy.List.map (\index -> removeOne index listOfTrees) 433 | |> Lazy.List.map listShrinkRecurse 434 | |> Lazy.force 435 | 436 | removeOne index list = 437 | List.append 438 | (List.take index list) 439 | (List.drop (index + 1) list) 440 | in 441 | Rose root (halved +++ shortened +++ shrunkenVals) 442 | 443 | 444 | {-| Given a fuzzer of a type, create a fuzzer of an array of that type. 445 | Generates random arrays of varying length, favoring shorter arrays. 446 | -} 447 | array : Fuzzer a -> Fuzzer (Array a) 448 | array fuzzer = 449 | map Array.fromList (list fuzzer) 450 | 451 | 452 | {-| Turn a tuple of fuzzers into a fuzzer of tuples. 453 | -} 454 | tuple : ( Fuzzer a, Fuzzer b ) -> Fuzzer ( a, b ) 455 | tuple ( fuzzerA, fuzzerB ) = 456 | map2 (,) fuzzerA fuzzerB 457 | 458 | 459 | {-| Turn a 3-tuple of fuzzers into a fuzzer of 3-tuples. 460 | -} 461 | tuple3 : ( Fuzzer a, Fuzzer b, Fuzzer c ) -> Fuzzer ( a, b, c ) 462 | tuple3 ( fuzzerA, fuzzerB, fuzzerC ) = 463 | map3 (,,) fuzzerA fuzzerB fuzzerC 464 | 465 | 466 | {-| Turn a 4-tuple of fuzzers into a fuzzer of 4-tuples. 467 | -} 468 | tuple4 : ( Fuzzer a, Fuzzer b, Fuzzer c, Fuzzer d ) -> Fuzzer ( a, b, c, d ) 469 | tuple4 ( fuzzerA, fuzzerB, fuzzerC, fuzzerD ) = 470 | map4 (,,,) fuzzerA fuzzerB fuzzerC fuzzerD 471 | 472 | 473 | {-| Turn a 5-tuple of fuzzers into a fuzzer of 5-tuples. 474 | -} 475 | tuple5 : ( Fuzzer a, Fuzzer b, Fuzzer c, Fuzzer d, Fuzzer e ) -> Fuzzer ( a, b, c, d, e ) 476 | tuple5 ( fuzzerA, fuzzerB, fuzzerC, fuzzerD, fuzzerE ) = 477 | map5 (,,,,) fuzzerA fuzzerB fuzzerC fuzzerD fuzzerE 478 | 479 | 480 | {-| Create a fuzzer that only and always returns the value provided, and performs no shrinking. This is hardly random, 481 | and so this function is best used as a helper when creating more complicated fuzzers. 482 | -} 483 | constant : a -> Fuzzer a 484 | constant x = 485 | Ok <| Random.constant (RoseTree.singleton x) 486 | 487 | 488 | {-| Map a function over a fuzzer. This applies to both the generated and the shrunken values. 489 | -} 490 | map : (a -> b) -> Fuzzer a -> Fuzzer b 491 | map = 492 | Internal.map 493 | 494 | 495 | {-| Map over two fuzzers. 496 | -} 497 | map2 : (a -> b -> c) -> Fuzzer a -> Fuzzer b -> Fuzzer c 498 | map2 transform fuzzA fuzzB = 499 | (Result.map2 << Random.map2 << map2RoseTree) transform fuzzA fuzzB 500 | 501 | 502 | {-| Map over three fuzzers. 503 | -} 504 | map3 : (a -> b -> c -> d) -> Fuzzer a -> Fuzzer b -> Fuzzer c -> Fuzzer d 505 | map3 transform fuzzA fuzzB fuzzC = 506 | (Result.map3 << Random.map3 << map3RoseTree) transform fuzzA fuzzB fuzzC 507 | 508 | 509 | {-| Map over four fuzzers. 510 | -} 511 | map4 : (a -> b -> c -> d -> e) -> Fuzzer a -> Fuzzer b -> Fuzzer c -> Fuzzer d -> Fuzzer e 512 | map4 transform fuzzA fuzzB fuzzC fuzzD = 513 | (Result.map4 << Random.map4 << map4RoseTree) transform fuzzA fuzzB fuzzC fuzzD 514 | 515 | 516 | {-| Map over five fuzzers. 517 | -} 518 | map5 : (a -> b -> c -> d -> e -> f) -> Fuzzer a -> Fuzzer b -> Fuzzer c -> Fuzzer d -> Fuzzer e -> Fuzzer f 519 | map5 transform fuzzA fuzzB fuzzC fuzzD fuzzE = 520 | (Result.map5 << Random.map5 << map5RoseTree) transform fuzzA fuzzB fuzzC fuzzD fuzzE 521 | 522 | 523 | {-| Map over many fuzzers. This can act as mapN for N > 5. 524 | The argument order is meant to accommodate chaining: 525 | map f aFuzzer 526 | |> andMap anotherFuzzer 527 | |> andMap aThirdFuzzer 528 | Note that shrinking may be better using mapN. 529 | -} 530 | andMap : Fuzzer a -> Fuzzer (a -> b) -> Fuzzer b 531 | andMap = 532 | map2 (|>) 533 | 534 | 535 | {-| Create a fuzzer based on the result of another fuzzer. 536 | -} 537 | andThen : (a -> Fuzzer b) -> Fuzzer a -> Fuzzer b 538 | andThen = 539 | Internal.andThen 540 | 541 | 542 | {-| Conditionally filter a fuzzer to remove occasional undesirable 543 | input. Takes a limit for how many retries to attempt, and a fallback 544 | function to, if no acceptable input can be found, create one from an 545 | unacceptable one. Also takes a condition to determine if the input is 546 | acceptable or not, and finally the fuzzer itself. 547 | 548 | A good number of max retries is ten. A large number of retries might 549 | blow the stack. 550 | 551 | -} 552 | conditional : { retries : Int, fallback : a -> a, condition : a -> Bool } -> Fuzzer a -> Fuzzer a 553 | conditional opts fuzzer = 554 | Result.map (conditionalHelper opts) fuzzer 555 | 556 | 557 | conditionalHelper : { retries : Int, fallback : a -> a, condition : a -> Bool } -> ValidFuzzer a -> ValidFuzzer a 558 | conditionalHelper opts validFuzzer = 559 | if opts.retries <= 0 then 560 | Random.map 561 | (RoseTree.map opts.fallback >> RoseTree.filterBranches opts.condition) 562 | validFuzzer 563 | else 564 | validFuzzer 565 | |> Random.andThen 566 | (\tree -> 567 | case RoseTree.filter opts.condition tree of 568 | Just tree -> 569 | Random.constant tree 570 | 571 | Nothing -> 572 | conditionalHelper { opts | retries = opts.retries - 1 } validFuzzer 573 | ) 574 | 575 | 576 | {-| Create a new `Fuzzer` by providing a list of probabilistic weights to use 577 | with other fuzzers. 578 | For example, to create a `Fuzzer` that has a 1/4 chance of generating an int 579 | between -1 and -100, and a 3/4 chance of generating one between 1 and 100, 580 | you could do this: 581 | 582 | Fuzz.frequency 583 | [ ( 1, Fuzz.intRange -100 -1 ) 584 | , ( 3, Fuzz.intRange 1 100 ) 585 | ] 586 | 587 | There are a few circumstances in which this function will return an invalid 588 | fuzzer, which causes it to fail any test that uses it: 589 | 590 | - If you provide an empty list of frequencies 591 | - If any of the weights are less than 0 592 | - If the weights sum to 0 593 | 594 | Be careful recursively using this fuzzer in its arguments. Often using `map` 595 | is a better way to do what you want. If you are fuzzing a tree-like data 596 | structure, you should include a depth limit so to avoid infinite recursion, like 597 | so: 598 | 599 | type Tree 600 | = Leaf 601 | | Branch Tree Tree 602 | 603 | tree : Int -> Fuzzer Tree 604 | tree i = 605 | if i <= 0 then 606 | Fuzz.constant Leaf 607 | else 608 | Fuzz.frequency 609 | [ ( 1, Fuzz.constant Leaf ) 610 | , ( 2, Fuzz.map2 Branch (tree (i - 1)) (tree (i - 1)) ) 611 | ] 612 | 613 | -} 614 | frequency : List ( Float, Fuzzer a ) -> Fuzzer a 615 | frequency list = 616 | if List.isEmpty list then 617 | invalid "You must provide at least one frequency pair." 618 | else if List.any (\( weight, _ ) -> weight < 0) list then 619 | invalid "No frequency weights can be less than 0." 620 | else if List.sum (List.map Tuple.first list) <= 0 then 621 | invalid "Frequency weights must sum to more than 0." 622 | else 623 | list 624 | |> List.map extractValid 625 | |> combineValid 626 | |> Result.map Random.frequency 627 | 628 | 629 | extractValid : ( a, Valid b ) -> Valid ( a, b ) 630 | extractValid ( a, valid ) = 631 | Result.map ((,) a) valid 632 | 633 | 634 | {-| Choose one of the given fuzzers at random. Each fuzzer has an equal chance 635 | of being chosen; to customize the probabilities, use [`frequency`](#frequency). 636 | 637 | Fuzz.oneOf 638 | [ Fuzz.intRange 0 3 639 | , Fuzz.intRange 7 9 640 | ] 641 | 642 | -} 643 | oneOf : List (Fuzzer a) -> Fuzzer a 644 | oneOf list = 645 | if List.isEmpty list then 646 | invalid "You must pass at least one Fuzzer to Fuzz.oneOf." 647 | else 648 | list 649 | |> List.map (\fuzzer -> ( 1, fuzzer )) 650 | |> frequency 651 | 652 | 653 | {-| A fuzzer that is invalid for the provided reason. Any fuzzers built with it 654 | are also invalid. Any tests using an invalid fuzzer fail. 655 | -} 656 | invalid : String -> Fuzzer a 657 | invalid reason = 658 | Err reason 659 | 660 | 661 | map2RoseTree : (a -> b -> c) -> RoseTree a -> RoseTree b -> RoseTree c 662 | map2RoseTree transform ((Rose root1 children1) as rose1) ((Rose root2 children2) as rose2) = 663 | {- Shrinking a pair of RoseTrees 664 | Recurse on all pairs created by substituting one element for any of its shrunken values. 665 | A weakness of this algorithm is that it expects that values can be shrunken independently. 666 | That is, to shrink from (a,b) to (a',b'), we must go through (a',b) or (a,b'). 667 | "No pairs sum to zero" is a pathological predicate that cannot be shrunken this way. 668 | -} 669 | let 670 | root = 671 | transform root1 root2 672 | 673 | shrink1 = 674 | Lazy.List.map (\subtree -> map2RoseTree transform subtree rose2) children1 675 | 676 | shrink2 = 677 | Lazy.List.map (\subtree -> map2RoseTree transform rose1 subtree) children2 678 | in 679 | Rose root (shrink1 +++ shrink2) 680 | 681 | 682 | 683 | -- The RoseTree 'mapN, n > 2' functions below follow the same strategy as map2RoseTree. 684 | -- They're implemented separately instead of in terms of `andMap` because this has significant perfomance benefits. 685 | 686 | 687 | map3RoseTree : (a -> b -> c -> d) -> RoseTree a -> RoseTree b -> RoseTree c -> RoseTree d 688 | map3RoseTree transform ((Rose root1 children1) as rose1) ((Rose root2 children2) as rose2) ((Rose root3 children3) as rose3) = 689 | let 690 | root = 691 | transform root1 root2 root3 692 | 693 | shrink1 = 694 | Lazy.List.map (\childOf1 -> map3RoseTree transform childOf1 rose2 rose3) children1 695 | 696 | shrink2 = 697 | Lazy.List.map (\childOf2 -> map3RoseTree transform rose1 childOf2 rose3) children2 698 | 699 | shrink3 = 700 | Lazy.List.map (\childOf3 -> map3RoseTree transform rose1 rose2 childOf3) children3 701 | in 702 | Rose root (shrink1 +++ shrink2 +++ shrink3) 703 | 704 | 705 | map4RoseTree : (a -> b -> c -> d -> e) -> RoseTree a -> RoseTree b -> RoseTree c -> RoseTree d -> RoseTree e 706 | map4RoseTree transform ((Rose root1 children1) as rose1) ((Rose root2 children2) as rose2) ((Rose root3 children3) as rose3) ((Rose root4 children4) as rose4) = 707 | let 708 | root = 709 | transform root1 root2 root3 root4 710 | 711 | shrink1 = 712 | Lazy.List.map (\childOf1 -> map4RoseTree transform childOf1 rose2 rose3 rose4) children1 713 | 714 | shrink2 = 715 | Lazy.List.map (\childOf2 -> map4RoseTree transform rose1 childOf2 rose3 rose4) children2 716 | 717 | shrink3 = 718 | Lazy.List.map (\childOf3 -> map4RoseTree transform rose1 rose2 childOf3 rose4) children3 719 | 720 | shrink4 = 721 | Lazy.List.map (\childOf4 -> map4RoseTree transform rose1 rose2 rose3 childOf4) children4 722 | in 723 | Rose root (shrink1 +++ shrink2 +++ shrink3 +++ shrink4) 724 | 725 | 726 | map5RoseTree : (a -> b -> c -> d -> e -> f) -> RoseTree a -> RoseTree b -> RoseTree c -> RoseTree d -> RoseTree e -> RoseTree f 727 | map5RoseTree transform ((Rose root1 children1) as rose1) ((Rose root2 children2) as rose2) ((Rose root3 children3) as rose3) ((Rose root4 children4) as rose4) ((Rose root5 children5) as rose5) = 728 | let 729 | root = 730 | transform root1 root2 root3 root4 root5 731 | 732 | shrink1 = 733 | Lazy.List.map (\childOf1 -> map5RoseTree transform childOf1 rose2 rose3 rose4 rose5) children1 734 | 735 | shrink2 = 736 | Lazy.List.map (\childOf2 -> map5RoseTree transform rose1 childOf2 rose3 rose4 rose5) children2 737 | 738 | shrink3 = 739 | Lazy.List.map (\childOf3 -> map5RoseTree transform rose1 rose2 childOf3 rose4 rose5) children3 740 | 741 | shrink4 = 742 | Lazy.List.map (\childOf4 -> map5RoseTree transform rose1 rose2 rose3 childOf4 rose5) children4 743 | 744 | shrink5 = 745 | Lazy.List.map (\childOf5 -> map5RoseTree transform rose1 rose2 rose3 rose4 childOf5) children5 746 | in 747 | Rose root (shrink1 +++ shrink2 +++ shrink3 +++ shrink4 +++ shrink5) 748 | -------------------------------------------------------------------------------- /src/Fuzz/Internal.elm: -------------------------------------------------------------------------------- 1 | module Fuzz.Internal exposing (Fuzzer, Valid, ValidFuzzer, andThen, combineValid, invalidReason, map) 2 | 3 | import Lazy 4 | import Lazy.List exposing ((:::), LazyList) 5 | import Random.Pcg as Random exposing (Generator) 6 | import RoseTree exposing (RoseTree(Rose)) 7 | 8 | 9 | type alias Fuzzer a = 10 | Valid (ValidFuzzer a) 11 | 12 | 13 | type alias Valid a = 14 | Result String a 15 | 16 | 17 | type alias ValidFuzzer a = 18 | Generator (RoseTree a) 19 | 20 | 21 | combineValid : List (Valid a) -> Valid (List a) 22 | combineValid valids = 23 | case valids of 24 | [] -> 25 | Ok [] 26 | 27 | (Ok x) :: rest -> 28 | Result.map ((::) x) (combineValid rest) 29 | 30 | (Err reason) :: _ -> 31 | Err reason 32 | 33 | 34 | map : (a -> b) -> Fuzzer a -> Fuzzer b 35 | map fn fuzzer = 36 | (Result.map << Random.map << RoseTree.map) fn fuzzer 37 | 38 | 39 | andThen : (a -> Fuzzer b) -> Fuzzer a -> Fuzzer b 40 | andThen fn fuzzer = 41 | let 42 | helper : (a -> Fuzzer b) -> RoseTree a -> ValidFuzzer b 43 | helper fn xs = 44 | RoseTree.map fn xs 45 | |> removeInvalid 46 | |> sequenceRoseTree 47 | |> Random.map RoseTree.flatten 48 | in 49 | Result.map (Random.andThen (helper fn)) fuzzer 50 | 51 | 52 | removeInvalid : RoseTree (Valid a) -> RoseTree a 53 | removeInvalid tree = 54 | case RoseTree.filterMap getValid tree of 55 | Just newTree -> 56 | newTree 57 | 58 | Nothing -> 59 | Debug.crash "Returning an invalid fuzzer from `andThen` is currently unsupported" 60 | 61 | 62 | sequenceRoseTree : RoseTree (Generator a) -> Generator (RoseTree a) 63 | sequenceRoseTree (Rose root branches) = 64 | Random.map2 65 | Rose 66 | root 67 | (Lazy.List.map sequenceRoseTree branches |> sequenceLazyList) 68 | 69 | 70 | sequenceLazyList : LazyList (Generator a) -> Generator (LazyList a) 71 | sequenceLazyList xs = 72 | Random.independentSeed 73 | |> Random.map (runAll xs) 74 | 75 | 76 | runAll : LazyList (Generator a) -> Random.Seed -> LazyList a 77 | runAll xs seed = 78 | Lazy.lazy <| 79 | \_ -> 80 | case Lazy.force xs of 81 | Lazy.List.Nil -> 82 | Lazy.List.Nil 83 | 84 | Lazy.List.Cons firstGenerator rest -> 85 | let 86 | ( x, newSeed ) = 87 | Random.step firstGenerator seed 88 | in 89 | Lazy.List.Cons x (runAll rest newSeed) 90 | 91 | 92 | getValid : Valid a -> Maybe a 93 | getValid valid = 94 | case valid of 95 | Ok x -> 96 | Just x 97 | 98 | Err _ -> 99 | Nothing 100 | 101 | 102 | invalidReason : Valid a -> Maybe String 103 | invalidReason valid = 104 | case valid of 105 | Ok _ -> 106 | Nothing 107 | 108 | Err reason -> 109 | Just reason 110 | -------------------------------------------------------------------------------- /src/RoseTree.elm: -------------------------------------------------------------------------------- 1 | module RoseTree exposing (..) 2 | 3 | {-| RoseTree implementation in Elm using Lazy Lists. 4 | 5 | This implementation is private to elm-test and has non-essential functions removed. 6 | If you need a complete RoseTree implementation, one can be found on elm-package. 7 | 8 | -} 9 | 10 | import Lazy.List as LazyList exposing ((+++), (:::), LazyList) 11 | 12 | 13 | {-| RoseTree type. 14 | A rosetree is a tree with a root whose children are themselves 15 | rosetrees. 16 | -} 17 | type RoseTree a 18 | = Rose a (LazyList (RoseTree a)) 19 | 20 | 21 | {-| Make a singleton rosetree 22 | -} 23 | singleton : a -> RoseTree a 24 | singleton a = 25 | Rose a LazyList.empty 26 | 27 | 28 | {-| Get the root of a rosetree 29 | -} 30 | root : RoseTree a -> a 31 | root (Rose a _) = 32 | a 33 | 34 | 35 | {-| Get the children of a rosetree 36 | -} 37 | children : RoseTree a -> LazyList (RoseTree a) 38 | children (Rose _ c) = 39 | c 40 | 41 | 42 | {-| Add a child to the rosetree. 43 | -} 44 | addChild : RoseTree a -> RoseTree a -> RoseTree a 45 | addChild child (Rose a c) = 46 | Rose a (child ::: c) 47 | 48 | 49 | {-| Map a function over a rosetree 50 | -} 51 | map : (a -> b) -> RoseTree a -> RoseTree b 52 | map f (Rose a c) = 53 | Rose (f a) (LazyList.map (map f) c) 54 | 55 | 56 | filter : (a -> Bool) -> RoseTree a -> Maybe (RoseTree a) 57 | filter predicate tree = 58 | let 59 | maybeKeep x = 60 | if predicate x then 61 | Just x 62 | else 63 | Nothing 64 | in 65 | filterMap maybeKeep tree 66 | 67 | 68 | {-| filterMap a function over a rosetree 69 | -} 70 | filterMap : (a -> Maybe b) -> RoseTree a -> Maybe (RoseTree b) 71 | filterMap f (Rose a c) = 72 | case f a of 73 | Just newA -> 74 | Just <| Rose newA (LazyList.filterMap (filterMap f) c) 75 | 76 | Nothing -> 77 | Nothing 78 | 79 | 80 | filterBranches : (a -> Bool) -> RoseTree a -> RoseTree a 81 | filterBranches predicate (Rose root branches) = 82 | Rose 83 | root 84 | (LazyList.filterMap (filter predicate) branches) 85 | 86 | 87 | {-| Flatten a rosetree of rosetrees. 88 | -} 89 | flatten : RoseTree (RoseTree a) -> RoseTree a 90 | flatten (Rose (Rose a c) cs) = 91 | Rose a (c +++ LazyList.map flatten cs) 92 | -------------------------------------------------------------------------------- /src/Test.elm: -------------------------------------------------------------------------------- 1 | module Test exposing (FuzzOptions, Test, concat, describe, fuzz, fuzz2, fuzz3, fuzz4, fuzz5, fuzzWith, only, skip, test, todo) 2 | 3 | {-| A module containing functions for creating and managing tests. 4 | 5 | @docs Test, test 6 | 7 | 8 | ## Organizing Tests 9 | 10 | @docs describe, concat, todo, skip, only 11 | 12 | 13 | ## Fuzz Testing 14 | 15 | @docs fuzz, fuzz2, fuzz3, fuzz4, fuzz5, fuzzWith, FuzzOptions 16 | 17 | -} 18 | 19 | import Expect exposing (Expectation) 20 | import Fuzz exposing (Fuzzer) 21 | import Set 22 | import Test.Fuzz 23 | import Test.Internal as Internal 24 | import Test.Runner.Failure exposing (InvalidReason(..), Reason(..)) 25 | 26 | 27 | {-| A test which has yet to be evaluated. When evaluated, it produces one 28 | or more [`Expectation`](../Expect#Expectation)s. 29 | 30 | See [`test`](#test) and [`fuzz`](#fuzz) for some ways to create a `Test`. 31 | 32 | -} 33 | type alias Test = 34 | Internal.Test 35 | 36 | 37 | {-| Run each of the given tests. 38 | 39 | concat [ testDecoder, testSorting ] 40 | 41 | -} 42 | concat : List Test -> Test 43 | concat tests = 44 | if List.isEmpty tests then 45 | Internal.failNow 46 | { description = "This `concat` has no tests in it. Let's give it some!" 47 | , reason = Invalid EmptyList 48 | } 49 | else 50 | case Internal.duplicatedName tests of 51 | Err duped -> 52 | Internal.failNow 53 | { description = "A test group contains multiple tests named '" ++ duped ++ "'. Do some renaming so that tests have unique names." 54 | , reason = Invalid DuplicatedName 55 | } 56 | 57 | Ok _ -> 58 | Internal.Batch tests 59 | 60 | 61 | {-| Apply a description to a list of tests. 62 | 63 | import Test exposing (describe, test, fuzz) 64 | import Fuzz exposing (int) 65 | import Expect 66 | 67 | 68 | describe "List" 69 | [ describe "reverse" 70 | [ test "has no effect on an empty list" <| 71 | \_ -> 72 | List.reverse [] 73 | |> Expect.equal [] 74 | , fuzz int "has no effect on a one-item list" <| 75 | \num -> 76 | List.reverse [ num ] 77 | |> Expect.equal [ num ] 78 | ] 79 | ] 80 | 81 | Passing an empty list will result in a failing test, because you either made a 82 | mistake or are creating a placeholder. 83 | 84 | -} 85 | describe : String -> List Test -> Test 86 | describe untrimmedDesc tests = 87 | let 88 | desc = 89 | String.trim untrimmedDesc 90 | in 91 | if String.isEmpty desc then 92 | Internal.failNow 93 | { description = "This `describe` has a blank description. Let's give it a useful one!" 94 | , reason = Invalid BadDescription 95 | } 96 | else if List.isEmpty tests then 97 | Internal.failNow 98 | { description = "This `describe " ++ toString desc ++ "` has no tests in it. Let's give it some!" 99 | , reason = Invalid EmptyList 100 | } 101 | else 102 | case Internal.duplicatedName tests of 103 | Err duped -> 104 | Internal.failNow 105 | { description = "The tests '" ++ desc ++ "' contain multiple tests named '" ++ duped ++ "'. Let's rename them so we know which is which." 106 | , reason = Invalid DuplicatedName 107 | } 108 | 109 | Ok childrenNames -> 110 | if Set.member desc childrenNames then 111 | Internal.failNow 112 | { description = "The test '" ++ desc ++ "' contains a child test of the same name. Let's rename them so we know which is which." 113 | , reason = Invalid DuplicatedName 114 | } 115 | else 116 | Internal.Labeled desc (Internal.Batch tests) 117 | 118 | 119 | {-| Return a [`Test`](#Test) that evaluates a single 120 | [`Expectation`](../Expect#Expectation). 121 | 122 | import Test exposing (fuzz) 123 | import Expect 124 | 125 | 126 | test "the empty list has 0 length" <| 127 | \_ -> 128 | List.length [] 129 | |> Expect.equal 0 130 | 131 | -} 132 | test : String -> (() -> Expectation) -> Test 133 | test untrimmedDesc thunk = 134 | let 135 | desc = 136 | String.trim untrimmedDesc 137 | in 138 | if String.isEmpty desc then 139 | Internal.blankDescriptionFailure 140 | else 141 | Internal.Labeled desc (Internal.UnitTest (\() -> [ thunk () ])) 142 | 143 | 144 | {-| Returns a [`Test`](#Test) that is "TODO" (not yet implemented). These tests 145 | always fail, but test runners will only include them in their output if there 146 | are no other failures. 147 | 148 | These tests aren't meant to be committed to version control. Instead, use them 149 | when you're brainstorming lots of tests you'd like to write, but you can't 150 | implement them all at once. When you replace `todo` with a real test, you'll be 151 | able to see if it fails without clutter from tests still not implemented. But, 152 | unlike leaving yourself comments, you'll be prompted to implement these tests 153 | because your suite will fail. 154 | 155 | describe "a new thing" 156 | [ todo "does what is expected in the common case" 157 | , todo "correctly handles an edge case I just thought of" 158 | ] 159 | 160 | This functionality is similar to "pending" tests in other frameworks, except 161 | that a TODO test is considered failing but a pending test often is not. 162 | 163 | -} 164 | todo : String -> Test 165 | todo desc = 166 | Internal.failNow 167 | { description = desc 168 | , reason = TODO 169 | } 170 | 171 | 172 | {-| Returns a [`Test`](#Test) that causes other tests to be skipped, and 173 | only runs the given one. 174 | 175 | Calls to `only` aren't meant to be committed to version control. Instead, use 176 | them when you want to focus on getting a particular subset of your tests to pass. 177 | If you use `only`, your entire test suite will fail, even if 178 | each of the individual tests pass. This is to help avoid accidentally 179 | committing a `only` to version control. 180 | 181 | If you you use `only` on multiple tests, only those tests will run. If you 182 | put a `only` inside another `only`, only the outermost `only` 183 | will affect which tests gets run. 184 | 185 | See also [`skip`](#skip). Note that `skip` takes precedence over `only`; 186 | if you use a `skip` inside an `only`, it will still get skipped, and if you use 187 | an `only` inside a `skip`, it will also get skipped. 188 | 189 | describe "List" 190 | [ only <| describe "reverse" 191 | [ test "has no effect on an empty list" <| 192 | \_ -> 193 | List.reverse [] 194 | |> Expect.equal [] 195 | , fuzz int "has no effect on a one-item list" <| 196 | \num -> 197 | List.reverse [ num ] 198 | |> Expect.equal [ num ] 199 | ] 200 | , test "This will not get run, because of the `only` above!" <| 201 | \_ -> 202 | List.length [] 203 | |> Expect.equal 0 204 | ] 205 | 206 | -} 207 | only : Test -> Test 208 | only = 209 | Internal.Only 210 | 211 | 212 | {-| Returns a [`Test`](#Test) that gets skipped. 213 | 214 | Calls to `skip` aren't meant to be committed to version control. Instead, use 215 | it when you want to focus on getting a particular subset of your tests to 216 | pass. If you use `skip`, your entire test suite will fail, even if 217 | each of the individual tests pass. This is to help avoid accidentally 218 | committing a `skip` to version control. 219 | 220 | See also [`only`](#only). Note that `skip` takes precedence over `only`; 221 | if you use a `skip` inside an `only`, it will still get skipped, and if you use 222 | an `only` inside a `skip`, it will also get skipped. 223 | 224 | describe "List" 225 | [ skip <| describe "reverse" 226 | [ test "has no effect on an empty list" <| 227 | \_ -> 228 | List.reverse [] 229 | |> Expect.equal [] 230 | , fuzz int "has no effect on a one-item list" <| 231 | \num -> 232 | List.reverse [ num ] 233 | |> Expect.equal [ num ] 234 | ] 235 | , test "This is the only test that will get run; the other was skipped!" <| 236 | \_ -> 237 | List.length [] 238 | |> Expect.equal 0 239 | ] 240 | 241 | -} 242 | skip : Test -> Test 243 | skip = 244 | Internal.Skipped 245 | 246 | 247 | {-| Options [`fuzzWith`](#fuzzWith) accepts. Currently there is only one but this 248 | API is designed so that it can accept more in the future. 249 | 250 | 251 | ### `runs` 252 | 253 | The number of times to run each fuzz test. (Default is 100.) 254 | 255 | import Test exposing (fuzzWith) 256 | import Fuzz exposing (list, int) 257 | import Expect 258 | 259 | 260 | fuzzWith { runs = 350 } (list int) "List.length should always be positive" <| 261 | -- This anonymous function will be run 350 times, each time with a 262 | -- randomly-generated fuzzList value. (It will always be a list of ints 263 | -- because of (list int) above.) 264 | \fuzzList -> 265 | fuzzList 266 | |> List.length 267 | |> Expect.atLeast 0 268 | 269 | -} 270 | type alias FuzzOptions = 271 | { runs : Int } 272 | 273 | 274 | {-| Run a [`fuzz`](#fuzz) test with the given [`FuzzOptions`](#FuzzOptions). 275 | 276 | Note that there is no `fuzzWith2`, but you can always pass more fuzz values in 277 | using [`Fuzz.tuple`](Fuzz#tuple), [`Fuzz.tuple3`](Fuzz#tuple3), 278 | for example like this: 279 | 280 | import Test exposing (fuzzWith) 281 | import Fuzz exposing (tuple, list, int) 282 | import Expect 283 | 284 | 285 | fuzzWith { runs = 4200 } 286 | (tuple ( list int, int )) 287 | "List.reverse never influences List.member" <| 288 | \(nums, target) -> 289 | List.member target (List.reverse nums) 290 | |> Expect.equal (List.member target nums) 291 | 292 | -} 293 | fuzzWith : FuzzOptions -> Fuzzer a -> String -> (a -> Expectation) -> Test 294 | fuzzWith options fuzzer desc getTest = 295 | if options.runs < 1 then 296 | Internal.failNow 297 | { description = "Fuzz tests must have a run count of at least 1, not " ++ toString options.runs ++ "." 298 | , reason = Invalid NonpositiveFuzzCount 299 | } 300 | else 301 | fuzzWithHelp options (fuzz fuzzer desc getTest) 302 | 303 | 304 | fuzzWithHelp : FuzzOptions -> Test -> Test 305 | fuzzWithHelp options test = 306 | case test of 307 | Internal.UnitTest _ -> 308 | test 309 | 310 | Internal.FuzzTest run -> 311 | Internal.FuzzTest (\seed _ -> run seed options.runs) 312 | 313 | Internal.Labeled label subTest -> 314 | Internal.Labeled label (fuzzWithHelp options subTest) 315 | 316 | Internal.Skipped subTest -> 317 | -- It's important to treat skipped tests exactly the same as normal, 318 | -- until after seed distribution has completed. 319 | fuzzWithHelp options subTest 320 | |> Internal.Only 321 | 322 | Internal.Only subTest -> 323 | fuzzWithHelp options subTest 324 | |> Internal.Only 325 | 326 | Internal.Batch tests -> 327 | tests 328 | |> List.map (fuzzWithHelp options) 329 | |> Internal.Batch 330 | 331 | 332 | {-| Take a function that produces a test, and calls it several (usually 100) times, using a randomly-generated input 333 | from a [`Fuzzer`](http://package.elm-lang.org/packages/elm-community/elm-test/latest/Fuzz) each time. This allows you to 334 | test that a property that should always be true is indeed true under a wide variety of conditions. The function also 335 | takes a string describing the test. 336 | 337 | These are called "[fuzz tests](https://en.wikipedia.org/wiki/Fuzz_testing)" because of the randomness. 338 | You may find them elsewhere called [property-based tests](http://blog.jessitron.com/2013/04/property-based-testing-what-is-it.html), 339 | [generative tests](http://www.pivotaltracker.com/community/tracker-blog/generative-testing), or 340 | [QuickCheck-style tests](https://en.wikipedia.org/wiki/QuickCheck). 341 | 342 | import Test exposing (fuzz) 343 | import Fuzz exposing (list, int) 344 | import Expect 345 | 346 | 347 | fuzz (list int) "List.length should always be positive" <| 348 | -- This anonymous function will be run 100 times, each time with a 349 | -- randomly-generated fuzzList value. 350 | \fuzzList -> 351 | fuzzList 352 | |> List.length 353 | |> Expect.atLeast 0 354 | 355 | -} 356 | fuzz : 357 | Fuzzer a 358 | -> String 359 | -> (a -> Expectation) 360 | -> Test 361 | fuzz = 362 | Test.Fuzz.fuzzTest 363 | 364 | 365 | {-| Run a [fuzz test](#fuzz) using two random inputs. 366 | 367 | This is a convenience function that lets you skip calling [`Fuzz.tuple`](Fuzz#tuple). 368 | 369 | See [`fuzzWith`](#fuzzWith) for an example of writing this in tuple style. 370 | 371 | import Test exposing (fuzz2) 372 | import Fuzz exposing (list, int) 373 | 374 | 375 | fuzz2 (list int) int "List.reverse never influences List.member" <| 376 | \nums target -> 377 | List.member target (List.reverse nums) 378 | |> Expect.equal (List.member target nums) 379 | 380 | -} 381 | fuzz2 : 382 | Fuzzer a 383 | -> Fuzzer b 384 | -> String 385 | -> (a -> b -> Expectation) 386 | -> Test 387 | fuzz2 fuzzA fuzzB desc = 388 | let 389 | fuzzer = 390 | Fuzz.tuple ( fuzzA, fuzzB ) 391 | in 392 | uncurry >> fuzz fuzzer desc 393 | 394 | 395 | {-| Run a [fuzz test](#fuzz) using three random inputs. 396 | 397 | This is a convenience function that lets you skip calling [`Fuzz.tuple3`](Fuzz#tuple3). 398 | 399 | -} 400 | fuzz3 : 401 | Fuzzer a 402 | -> Fuzzer b 403 | -> Fuzzer c 404 | -> String 405 | -> (a -> b -> c -> Expectation) 406 | -> Test 407 | fuzz3 fuzzA fuzzB fuzzC desc = 408 | let 409 | fuzzer = 410 | Fuzz.tuple3 ( fuzzA, fuzzB, fuzzC ) 411 | in 412 | uncurry3 >> fuzz fuzzer desc 413 | 414 | 415 | {-| Run a [fuzz test](#fuzz) using four random inputs. 416 | 417 | This is a convenience function that lets you skip calling [`Fuzz.tuple4`](Fuzz#tuple4). 418 | 419 | -} 420 | fuzz4 : 421 | Fuzzer a 422 | -> Fuzzer b 423 | -> Fuzzer c 424 | -> Fuzzer d 425 | -> String 426 | -> (a -> b -> c -> d -> Expectation) 427 | -> Test 428 | fuzz4 fuzzA fuzzB fuzzC fuzzD desc = 429 | let 430 | fuzzer = 431 | Fuzz.tuple4 ( fuzzA, fuzzB, fuzzC, fuzzD ) 432 | in 433 | uncurry4 >> fuzz fuzzer desc 434 | 435 | 436 | {-| Run a [fuzz test](#fuzz) using five random inputs. 437 | 438 | This is a convenience function that lets you skip calling [`Fuzz.tuple5`](Fuzz#tuple5). 439 | 440 | -} 441 | fuzz5 : 442 | Fuzzer a 443 | -> Fuzzer b 444 | -> Fuzzer c 445 | -> Fuzzer d 446 | -> Fuzzer e 447 | -> String 448 | -> (a -> b -> c -> d -> e -> Expectation) 449 | -> Test 450 | fuzz5 fuzzA fuzzB fuzzC fuzzD fuzzE desc = 451 | let 452 | fuzzer = 453 | Fuzz.tuple5 ( fuzzA, fuzzB, fuzzC, fuzzD, fuzzE ) 454 | in 455 | uncurry5 >> fuzz fuzzer desc 456 | 457 | 458 | 459 | -- INTERNAL HELPERS -- 460 | 461 | 462 | uncurry3 : (a -> b -> c -> d) -> ( a, b, c ) -> d 463 | uncurry3 fn ( a, b, c ) = 464 | fn a b c 465 | 466 | 467 | uncurry4 : (a -> b -> c -> d -> e) -> ( a, b, c, d ) -> e 468 | uncurry4 fn ( a, b, c, d ) = 469 | fn a b c d 470 | 471 | 472 | uncurry5 : (a -> b -> c -> d -> e -> f) -> ( a, b, c, d, e ) -> f 473 | uncurry5 fn ( a, b, c, d, e ) = 474 | fn a b c d e 475 | -------------------------------------------------------------------------------- /src/Test/Expectation.elm: -------------------------------------------------------------------------------- 1 | module Test.Expectation exposing (Expectation(..), fail, withGiven) 2 | 3 | import Test.Runner.Failure exposing (Reason) 4 | 5 | 6 | type Expectation 7 | = Pass 8 | | Fail { given : Maybe String, description : String, reason : Reason } 9 | 10 | 11 | {-| Create a failure without specifying the given. 12 | -} 13 | fail : { description : String, reason : Reason } -> Expectation 14 | fail { description, reason } = 15 | Fail { given = Nothing, description = description, reason = reason } 16 | 17 | 18 | {-| Set the given (fuzz test input) of an expectation. 19 | -} 20 | withGiven : String -> Expectation -> Expectation 21 | withGiven newGiven expectation = 22 | case expectation of 23 | Fail failure -> 24 | Fail { failure | given = Just newGiven } 25 | 26 | Pass -> 27 | expectation 28 | -------------------------------------------------------------------------------- /src/Test/Fuzz.elm: -------------------------------------------------------------------------------- 1 | module Test.Fuzz exposing (fuzzTest) 2 | 3 | import Dict exposing (Dict) 4 | import Fuzz exposing (Fuzzer) 5 | import Fuzz.Internal exposing (ValidFuzzer) 6 | import Lazy.List 7 | import Random.Pcg as Random exposing (Generator) 8 | import RoseTree exposing (RoseTree(..)) 9 | import Test.Expectation exposing (Expectation(..)) 10 | import Test.Internal exposing (Test(..), blankDescriptionFailure, failNow) 11 | import Test.Runner.Failure exposing (InvalidReason(..), Reason(..)) 12 | 13 | 14 | {-| Reject always-failing tests because of bad names or invalid fuzzers. 15 | -} 16 | fuzzTest : Fuzzer a -> String -> (a -> Expectation) -> Test 17 | fuzzTest fuzzer untrimmedDesc getExpectation = 18 | let 19 | desc = 20 | String.trim untrimmedDesc 21 | in 22 | if String.isEmpty desc then 23 | blankDescriptionFailure 24 | else 25 | case fuzzer of 26 | Err reason -> 27 | failNow 28 | { description = reason 29 | , reason = Invalid InvalidFuzzer 30 | } 31 | 32 | Ok validFuzzer -> 33 | -- Preliminary checks passed; run the fuzz test 34 | validatedFuzzTest validFuzzer desc getExpectation 35 | 36 | 37 | {-| Knowing that the fuzz test isn't obviously invalid, run the test and package up the results. 38 | -} 39 | validatedFuzzTest : ValidFuzzer a -> String -> (a -> Expectation) -> Test 40 | validatedFuzzTest fuzzer desc getExpectation = 41 | let 42 | run seed runs = 43 | let 44 | failures = 45 | getFailures fuzzer getExpectation seed runs 46 | in 47 | -- Make sure if we passed, we don't do any more work. 48 | if Dict.isEmpty failures then 49 | [ Pass ] 50 | else 51 | failures 52 | |> Dict.toList 53 | |> List.map formatExpectation 54 | in 55 | Labeled desc (FuzzTest run) 56 | 57 | 58 | type alias Failures = 59 | Dict String Expectation 60 | 61 | 62 | getFailures : ValidFuzzer a -> (a -> Expectation) -> Random.Seed -> Int -> Dict String Expectation 63 | getFailures fuzzer getExpectation initialSeed totalRuns = 64 | {- Fuzz test algorithm with memoization and opt-in RoseTrees: 65 | Generate a single value from the fuzzer's genVal random generator 66 | Determine if the value is memoized. If so, skip. Otherwise continue. 67 | Run the test on that value. If it fails: 68 | Generate the rosetree by passing the fuzzer False *and the same random seed* 69 | Find the new failure by looking at the children for any shrunken values: 70 | If a shrunken value causes a failure, recurse on its children 71 | If no shrunken value replicates the failure, use the root 72 | Whether it passes or fails, do this n times 73 | -} 74 | let 75 | genVal = 76 | Random.map RoseTree.root fuzzer 77 | 78 | initialFailures = 79 | Dict.empty 80 | 81 | helper currentSeed remainingRuns failures = 82 | let 83 | ( value, nextSeed ) = 84 | Random.step genVal currentSeed 85 | 86 | newFailures = 87 | findNewFailure fuzzer getExpectation failures currentSeed value 88 | in 89 | if remainingRuns <= 1 then 90 | newFailures 91 | else 92 | helper nextSeed (remainingRuns - 1) newFailures 93 | in 94 | helper initialSeed totalRuns initialFailures 95 | 96 | 97 | {-| Knowing that a value in not in the cache, determine if it causes the test to pass or fail. 98 | -} 99 | findNewFailure : 100 | ValidFuzzer a 101 | -> (a -> Expectation) 102 | -> Failures 103 | -> Random.Seed 104 | -> a 105 | -> Failures 106 | findNewFailure fuzzer getExpectation failures currentSeed value = 107 | case getExpectation value of 108 | Pass -> 109 | failures 110 | 111 | failedExpectation -> 112 | let 113 | ( rosetree, nextSeed ) = 114 | -- nextSeed is not used here because caller function has currentSeed 115 | Random.step fuzzer currentSeed 116 | in 117 | shrinkAndAdd rosetree getExpectation failedExpectation failures 118 | 119 | 120 | {-| Knowing that the rosetree's root already failed, finds the shrunken failure. 121 | Returns the updated failures dictionary. 122 | -} 123 | shrinkAndAdd : 124 | RoseTree a 125 | -> (a -> Expectation) 126 | -> Expectation 127 | -> Failures 128 | -> Failures 129 | shrinkAndAdd rootTree getExpectation rootsExpectation failures = 130 | let 131 | shrink : Expectation -> RoseTree a -> ( a, Expectation ) 132 | shrink oldExpectation (Rose failingValue branches) = 133 | case Lazy.List.headAndTail branches of 134 | Just ( (Rose possiblyFailingValue _) as rosetree, moreLazyRoseTrees ) -> 135 | -- either way, recurse with the most recent failing expectation, and failing input with its list of shrunken values 136 | case getExpectation possiblyFailingValue of 137 | Pass -> 138 | shrink oldExpectation 139 | (Rose failingValue moreLazyRoseTrees) 140 | 141 | newExpectation -> 142 | let 143 | ( minimalValue, finalExpectation ) = 144 | shrink newExpectation rosetree 145 | in 146 | ( minimalValue 147 | , finalExpectation 148 | ) 149 | 150 | Nothing -> 151 | ( failingValue, oldExpectation ) 152 | 153 | (Rose failingValue _) = 154 | rootTree 155 | 156 | ( minimalValue, finalExpectation ) = 157 | shrink rootsExpectation rootTree 158 | in 159 | Dict.insert (toString minimalValue) finalExpectation failures 160 | 161 | 162 | formatExpectation : ( String, Expectation ) -> Expectation 163 | formatExpectation ( given, expectation ) = 164 | Test.Expectation.withGiven given expectation 165 | -------------------------------------------------------------------------------- /src/Test/Internal.elm: -------------------------------------------------------------------------------- 1 | module Test.Internal exposing (Test(..), blankDescriptionFailure, duplicatedName, failNow) 2 | 3 | import Random.Pcg as Random exposing (Generator) 4 | import Set exposing (Set) 5 | import Test.Expectation exposing (Expectation(..)) 6 | import Test.Runner.Failure exposing (InvalidReason(..), Reason(..)) 7 | 8 | 9 | type Test 10 | = UnitTest (() -> List Expectation) 11 | | FuzzTest (Random.Seed -> Int -> List Expectation) 12 | | Labeled String Test 13 | | Skipped Test 14 | | Only Test 15 | | Batch (List Test) 16 | 17 | 18 | {-| Create a test that always fails for the given reason and description. 19 | -} 20 | failNow : { description : String, reason : Reason } -> Test 21 | failNow record = 22 | UnitTest 23 | (\() -> [ Test.Expectation.fail record ]) 24 | 25 | 26 | blankDescriptionFailure : Test 27 | blankDescriptionFailure = 28 | failNow 29 | { description = "This test has a blank description. Let's give it a useful one!" 30 | , reason = Invalid BadDescription 31 | } 32 | 33 | 34 | duplicatedName : List Test -> Result String (Set String) 35 | duplicatedName = 36 | let 37 | names : Test -> List String 38 | names test = 39 | case test of 40 | Labeled str _ -> 41 | [ str ] 42 | 43 | Batch subtests -> 44 | List.concatMap names subtests 45 | 46 | UnitTest _ -> 47 | [] 48 | 49 | FuzzTest _ -> 50 | [] 51 | 52 | Skipped subTest -> 53 | names subTest 54 | 55 | Only subTest -> 56 | names subTest 57 | 58 | insertOrFail : String -> Result String (Set String) -> Result String (Set String) 59 | insertOrFail newName = 60 | Result.andThen 61 | (\oldNames -> 62 | if Set.member newName oldNames then 63 | Err newName 64 | else 65 | Ok <| Set.insert newName oldNames 66 | ) 67 | in 68 | List.concatMap names 69 | >> List.foldl insertOrFail (Ok Set.empty) 70 | -------------------------------------------------------------------------------- /src/Test/Runner.elm: -------------------------------------------------------------------------------- 1 | module Test.Runner 2 | exposing 3 | ( Runner 4 | , SeededRunners(..) 5 | , Shrinkable 6 | , formatLabels 7 | , fromTest 8 | , fuzz 9 | , getFailure 10 | , getFailureReason 11 | , isTodo 12 | , shrink 13 | ) 14 | 15 | {-| This is an "experts only" module that exposes functions needed to run and 16 | display tests. A typical user will use an existing runner library for Node or 17 | the browser, which is implemented using this interface. A list of these runners 18 | can be found in the `README`. 19 | 20 | 21 | ## Runner 22 | 23 | @docs Runner, SeededRunners, fromTest 24 | 25 | 26 | ## Expectations 27 | 28 | @docs getFailure, getFailureReason, isTodo 29 | 30 | 31 | ## Formatting 32 | 33 | @docs formatLabels 34 | 35 | 36 | ## Fuzzers 37 | 38 | These functions give you the ability to run fuzzers separate of running fuzz tests. 39 | 40 | @docs Shrinkable, fuzz, shrink 41 | 42 | -} 43 | 44 | import Bitwise 45 | import Char 46 | import Expect exposing (Expectation) 47 | import Fuzz exposing (Fuzzer) 48 | import Lazy.List as LazyList exposing (LazyList) 49 | import Random.Pcg as Random 50 | import RoseTree exposing (RoseTree(Rose)) 51 | import String 52 | import Test exposing (Test) 53 | import Test.Expectation 54 | import Test.Internal as Internal 55 | import Test.Runner.Failure exposing (Reason(..)) 56 | 57 | 58 | {-| An unevaluated test. Run it with [`run`](#run) to evaluate it into a 59 | list of `Expectation`s. 60 | -} 61 | type Runnable 62 | = Thunk (() -> List Expectation) 63 | 64 | 65 | {-| A function which, when evaluated, produces a list of expectations. Also a 66 | list of labels which apply to this outcome. 67 | -} 68 | type alias Runner = 69 | { run : () -> List Expectation 70 | , labels : List String 71 | } 72 | 73 | 74 | {-| A structured test runner, incorporating: 75 | 76 | - The expectations to run 77 | - The hierarchy of description strings that describe the results 78 | 79 | -} 80 | type RunnableTree 81 | = Runnable Runnable 82 | | Labeled String RunnableTree 83 | | Batch (List RunnableTree) 84 | 85 | 86 | {-| Convert a `Test` into `SeededRunners`. 87 | 88 | In order to run any fuzz tests that the `Test` may have, it requires a default run count as well 89 | as an initial `Random.Seed`. `100` is a good run count. To obtain a good random seed, pass a 90 | random 32-bit integer to `Random.initialSeed`. You can obtain such an integer by running 91 | `Math.floor(Math.random()*0xFFFFFFFF)` in Node. It's typically fine to hard-code this value into 92 | your Elm code; it's easy and makes your tests reproducible. 93 | 94 | -} 95 | fromTest : Int -> Random.Seed -> Test -> SeededRunners 96 | fromTest runs seed test = 97 | if runs < 1 then 98 | Invalid ("Test runner run count must be at least 1, not " ++ toString runs) 99 | else 100 | let 101 | distribution = 102 | distributeSeeds runs seed test 103 | in 104 | if List.isEmpty distribution.only then 105 | if countAllRunnables distribution.skipped == 0 then 106 | distribution.all 107 | |> List.concatMap fromRunnableTree 108 | |> Plain 109 | else 110 | distribution.all 111 | |> List.concatMap fromRunnableTree 112 | |> Skipping 113 | else 114 | distribution.only 115 | |> List.concatMap fromRunnableTree 116 | |> Only 117 | 118 | 119 | countAllRunnables : List RunnableTree -> Int 120 | countAllRunnables = 121 | List.foldl (countRunnables >> (+)) 0 122 | 123 | 124 | countRunnables : RunnableTree -> Int 125 | countRunnables runnable = 126 | case runnable of 127 | Runnable _ -> 128 | 1 129 | 130 | Labeled _ runner -> 131 | countRunnables runner 132 | 133 | Batch runners -> 134 | countAllRunnables runners 135 | 136 | 137 | run : Runnable -> List Expectation 138 | run (Thunk fn) = 139 | fn () 140 | 141 | 142 | fromRunnableTree : RunnableTree -> List Runner 143 | fromRunnableTree = 144 | fromRunnableTreeHelp [] 145 | 146 | 147 | fromRunnableTreeHelp : List String -> RunnableTree -> List Runner 148 | fromRunnableTreeHelp labels runner = 149 | case runner of 150 | Runnable runnable -> 151 | [ { labels = labels 152 | , run = \_ -> run runnable 153 | } 154 | ] 155 | 156 | Labeled label subRunner -> 157 | fromRunnableTreeHelp (label :: labels) subRunner 158 | 159 | Batch runners -> 160 | List.concatMap (fromRunnableTreeHelp labels) runners 161 | 162 | 163 | type alias Distribution = 164 | { seed : Random.Seed 165 | , only : List RunnableTree 166 | , all : List RunnableTree 167 | , skipped : List RunnableTree 168 | } 169 | 170 | 171 | {-| Test Runners which have had seeds distributed to them, and which are now 172 | either invalid or are ready to run. Seeded runners include some metadata: 173 | 174 | - `Invalid` runners had a problem (e.g. two sibling tests had the same description) making them un-runnable. 175 | - `Only` runners can be run, but `Test.only` was used somewhere, so ultimately they will lead to a failed test run even if each test that gets run passes. 176 | - `Skipping` runners can be run, but `Test.skip` was used somewhere, so ultimately they will lead to a failed test run even if each test that gets run passes. 177 | - `Plain` runners are ready to run, and have none of these issues. 178 | 179 | -} 180 | type SeededRunners 181 | = Plain (List Runner) 182 | | Only (List Runner) 183 | | Skipping (List Runner) 184 | | Invalid String 185 | 186 | 187 | emptyDistribution : Random.Seed -> Distribution 188 | emptyDistribution seed = 189 | { seed = seed 190 | , all = [] 191 | , only = [] 192 | , skipped = [] 193 | } 194 | 195 | 196 | {-| This breaks down a test into individual Runners, while assigning different 197 | random number seeds to them. Along the way it also does a few other things: 198 | 199 | 1. Collect any tests created with `Test.only` so later we can run only those. 200 | 2. Collect any tests created with `Test.todo` so later we can fail the run. 201 | 3. Validate that the run count is at least 1. 202 | 203 | Some design notes: 204 | 205 | 1. `only` tests and `skip` tests do not affect seed distribution. This is 206 | important for the case where a user runs tests, sees one failure, and decides 207 | to isolate it by using both `only` and providing the same seed as before. If 208 | `only` changes seed distribution, then that test result might not reproduce! 209 | This would be very frustrating, as it would mean you could reproduce the 210 | failure when not using `only`, but it magically disappeared as soon as you 211 | tried to isolate it. The same logic applies to `skip`. 212 | 213 | 2. Theoretically this could become tail-recursive. However, the Labeled and Batch 214 | cases would presumably become very gnarly, and it's unclear whether there would 215 | be a performance benefit or penalty in the end. If some brave soul wants to 216 | attempt it for kicks, beware that this is not a performance optimization for 217 | the faint of heart. Practically speaking, it seems unlikely to be worthwhile 218 | unless somehow people start seeing stack overflows during seed distribution - 219 | which would presumably require some absurdly deeply nested `describe` calls. 220 | 221 | -} 222 | distributeSeeds : Int -> Random.Seed -> Test -> Distribution 223 | distributeSeeds = 224 | distributeSeedsHelp False 225 | 226 | 227 | distributeSeedsHelp : Bool -> Int -> Random.Seed -> Test -> Distribution 228 | distributeSeedsHelp hashed runs seed test = 229 | case test of 230 | Internal.UnitTest run -> 231 | { seed = seed 232 | , all = [ Runnable (Thunk (\_ -> run ())) ] 233 | , only = [] 234 | , skipped = [] 235 | } 236 | 237 | Internal.FuzzTest run -> 238 | let 239 | ( firstSeed, nextSeed ) = 240 | Random.step Random.independentSeed seed 241 | in 242 | { seed = nextSeed 243 | , all = [ Runnable (Thunk (\_ -> run firstSeed runs)) ] 244 | , only = [] 245 | , skipped = [] 246 | } 247 | 248 | Internal.Labeled description subTest -> 249 | -- This fixes https://github.com/elm-community/elm-test/issues/192 250 | -- The first time we hit a Labeled, we want to use the hash of 251 | -- that label, along with the original seed, as our starting 252 | -- point for distribution. Repeating this process more than 253 | -- once would be a waste. 254 | if hashed then 255 | let 256 | next = 257 | distributeSeedsHelp True runs seed subTest 258 | in 259 | { seed = next.seed 260 | , all = List.map (Labeled description) next.all 261 | , only = List.map (Labeled description) next.only 262 | , skipped = List.map (Labeled description) next.skipped 263 | } 264 | else 265 | let 266 | intFromSeed = 267 | -- At this point, this seed will be the original 268 | -- one passed into distributeSeeds. We know this 269 | -- because the only other branch that does a 270 | -- Random.step on that seed is the Internal.Test 271 | -- branch, and you can't have a Labeled inside a 272 | -- Test, so that couldn't have come up yet. 273 | seed 274 | -- Convert the Seed back to an Int 275 | |> Random.step (Random.int 0 Random.maxInt) 276 | |> Tuple.first 277 | 278 | hashedSeed = 279 | description 280 | -- Hash from String to Int 281 | |> fnvHashString fnvInit 282 | -- Incorporate the originally passed-in seed 283 | |> fnvHash intFromSeed 284 | -- Convert Int back to Seed 285 | |> Random.initialSeed 286 | 287 | next = 288 | distributeSeedsHelp True runs hashedSeed subTest 289 | in 290 | -- Using seed instead of next.seed fixes https://github.com/elm-community/elm-test/issues/192 291 | -- by making it so that all the tests underneath this Label begin 292 | -- with the hashed seed, but subsequent sibling tests in this Batch 293 | -- get the same seed as before. 294 | { seed = seed 295 | , all = List.map (Labeled description) next.all 296 | , only = List.map (Labeled description) next.only 297 | , skipped = List.map (Labeled description) next.skipped 298 | } 299 | 300 | Internal.Skipped subTest -> 301 | let 302 | -- Go through the motions in order to obtain the seed, but then 303 | -- move everything to skipped. 304 | next = 305 | distributeSeedsHelp hashed runs seed subTest 306 | in 307 | { seed = next.seed 308 | , all = [] 309 | , only = [] 310 | , skipped = next.all 311 | } 312 | 313 | Internal.Only subTest -> 314 | let 315 | next = 316 | distributeSeedsHelp hashed runs seed subTest 317 | in 318 | -- `only` all the things! 319 | { next | only = next.all } 320 | 321 | Internal.Batch tests -> 322 | List.foldl (batchDistribute hashed runs) (emptyDistribution seed) tests 323 | 324 | 325 | batchDistribute : Bool -> Int -> Test -> Distribution -> Distribution 326 | batchDistribute hashed runs test prev = 327 | let 328 | next = 329 | distributeSeedsHelp hashed runs prev.seed test 330 | in 331 | { seed = next.seed 332 | , all = prev.all ++ next.all 333 | , only = prev.only ++ next.only 334 | , skipped = prev.skipped ++ next.skipped 335 | } 336 | 337 | 338 | {-| FNV-1a initial hash value 339 | -} 340 | fnvInit : Int 341 | fnvInit = 342 | 2166136261 343 | 344 | 345 | {-| FNV-1a helper for strings, using Char.toCode 346 | -} 347 | fnvHashString : Int -> String -> Int 348 | fnvHashString hash str = 349 | str |> String.toList |> List.map Char.toCode |> List.foldl fnvHash hash 350 | 351 | 352 | {-| FNV-1a implementation. 353 | -} 354 | fnvHash : Int -> Int -> Int 355 | fnvHash a b = 356 | Bitwise.xor a b * 16777619 |> Bitwise.shiftRightZfBy 0 357 | 358 | 359 | {-| **DEPRECATED.** Please use [`getFailureReason`](#getFailureReason) instead. 360 | This function will be removed in the next major version. 361 | 362 | Return `Nothing` if the given [`Expectation`](#Expectation) is a [`pass`](#pass). 363 | 364 | If it is a [`fail`](#fail), return a record containing the failure message, 365 | along with the given inputs if it was a fuzz test. (If no inputs were involved, 366 | the record's `given` field will be `Nothing`). 367 | 368 | For example, if a fuzz test generates random integers, this might return 369 | `{ message = "it was supposed to be positive", given = "-1" }` 370 | 371 | getFailure (Expect.fail "this failed") 372 | -- Just { message = "this failed", given = "" } 373 | 374 | getFailure (Expect.pass) 375 | -- Nothing 376 | 377 | -} 378 | getFailure : Expectation -> Maybe { given : Maybe String, message : String } 379 | getFailure expectation = 380 | case expectation of 381 | Test.Expectation.Pass -> 382 | Nothing 383 | 384 | Test.Expectation.Fail { given, description, reason } -> 385 | Just 386 | { given = given 387 | , message = Test.Runner.Failure.format description reason 388 | } 389 | 390 | 391 | {-| Return `Nothing` if the given [`Expectation`](#Expectation) is a [`pass`](#pass). 392 | 393 | If it is a [`fail`](#fail), return a record containing the expectation 394 | description, the [`Reason`](#Reason) the test failed, and the given inputs if 395 | it was a fuzz test. (If it was not a fuzz test, the record's `given` field 396 | will be `Nothing`). 397 | 398 | For example: 399 | 400 | getFailureReason (Expect.equal 1 2) 401 | -- Just { reason = Equal 1 2, description = "Expect.equal", given = Nothing } 402 | 403 | getFailureReason (Expect.equal 1 1) 404 | -- Nothing 405 | 406 | -} 407 | getFailureReason : 408 | Expectation 409 | -> 410 | Maybe 411 | { given : Maybe String 412 | , description : String 413 | , reason : Reason 414 | } 415 | getFailureReason expectation = 416 | case expectation of 417 | Test.Expectation.Pass -> 418 | Nothing 419 | 420 | Test.Expectation.Fail record -> 421 | Just record 422 | 423 | 424 | {-| Determine if an expectation was created by a call to `Test.todo`. Runners 425 | may treat these tests differently in their output. 426 | -} 427 | isTodo : Expectation -> Bool 428 | isTodo expectation = 429 | case expectation of 430 | Test.Expectation.Pass -> 431 | False 432 | 433 | Test.Expectation.Fail { reason } -> 434 | reason == TODO 435 | 436 | 437 | {-| A standard way to format descriptions and test labels, to keep things 438 | consistent across test runner implementations. 439 | 440 | The HTML, Node, String, and Log runners all use this. 441 | 442 | What it does: 443 | 444 | - drop any labels that are empty strings 445 | - format the first label differently from the others 446 | - reverse the resulting list 447 | 448 | Example: 449 | 450 | [ "the actual test that failed" 451 | , "nested description failure" 452 | , "top-level description failure" 453 | ] 454 | |> formatLabels ((++) "↓ ") ((++) "✗ ") 455 | 456 | {- 457 | [ "↓ top-level description failure" 458 | , "↓ nested description failure" 459 | , "✗ the actual test that failed" 460 | ] 461 | -} 462 | 463 | -} 464 | formatLabels : 465 | (String -> format) 466 | -> (String -> format) 467 | -> List String 468 | -> List format 469 | formatLabels formatDescription formatTest labels = 470 | case List.filter (not << String.isEmpty) labels of 471 | [] -> 472 | [] 473 | 474 | test :: descriptions -> 475 | descriptions 476 | |> List.map formatDescription 477 | |> (::) (formatTest test) 478 | |> List.reverse 479 | 480 | 481 | type alias Shrunken a = 482 | { down : LazyList (RoseTree a) 483 | , over : LazyList (RoseTree a) 484 | } 485 | 486 | 487 | {-| A `Shrinkable a` is an opaque type that allows you to obtain a value of type 488 | `a` that is smaller than the one you've previously obtained. 489 | -} 490 | type Shrinkable a 491 | = Shrinkable (Shrunken a) 492 | 493 | 494 | {-| Given a fuzzer, return a random generator to produce a value and a 495 | Shrinkable. The value is what a fuzz test would have received as input. 496 | -} 497 | fuzz : Fuzzer a -> Random.Generator ( a, Shrinkable a ) 498 | fuzz fuzzer = 499 | case fuzzer of 500 | Ok validFuzzer -> 501 | validFuzzer 502 | |> Random.map 503 | (\(Rose root children) -> 504 | ( root, Shrinkable { down = children, over = LazyList.empty } ) 505 | ) 506 | 507 | Err reason -> 508 | Debug.crash <| "Cannot call `fuzz` with an invalid fuzzer: " ++ reason 509 | 510 | 511 | {-| Given a Shrinkable, attempt to shrink the value further. Pass `False` to 512 | indicate that the last value you've seen (from either `fuzz` or this function) 513 | caused the test to **fail**. This will attempt to find a smaller value. Pass 514 | `True` if the test passed. If you have already seen a failure, this will attempt 515 | to shrink that failure in another way. In both cases, it may be impossible to 516 | shrink the value, represented by `Nothing`. 517 | -} 518 | shrink : Bool -> Shrinkable a -> Maybe ( a, Shrinkable a ) 519 | shrink causedPass (Shrinkable { down, over }) = 520 | let 521 | tryNext = 522 | if causedPass then 523 | over 524 | else 525 | down 526 | in 527 | case LazyList.headAndTail tryNext of 528 | Just ( Rose root children, tl ) -> 529 | Just ( root, Shrinkable { down = children, over = tl } ) 530 | 531 | Nothing -> 532 | Nothing 533 | -------------------------------------------------------------------------------- /src/Test/Runner/Failure.elm: -------------------------------------------------------------------------------- 1 | module Test.Runner.Failure exposing (InvalidReason(..), Reason(..), format) 2 | 3 | {-| The reason a test failed. 4 | 5 | @docs Reason, InvalidReason, format 6 | 7 | -} 8 | 9 | 10 | {-| The reason a test failed. 11 | 12 | Test runners can use this to provide nice output, e.g. by doing diffs on the 13 | two parts of an `Expect.equal` failure. 14 | 15 | -} 16 | type Reason 17 | = Custom 18 | | Equality String String 19 | | Comparison String String 20 | -- Expected, actual, (index of problem, expected element, actual element) 21 | | ListDiff (List String) (List String) 22 | {- I don't think we need to show the diff twice with + and - reversed. Just show it after the main vertical bar. 23 | "Extra" and "missing" are relative to the actual value. 24 | -} 25 | | CollectionDiff 26 | { expected : String 27 | , actual : String 28 | , extra : List String 29 | , missing : List String 30 | } 31 | | TODO 32 | | Invalid InvalidReason 33 | 34 | 35 | {-| The reason a test run was invalid. 36 | 37 | Test runners should report these to the user in whatever format is appropriate. 38 | 39 | -} 40 | type InvalidReason 41 | = EmptyList 42 | | NonpositiveFuzzCount 43 | | InvalidFuzzer 44 | | BadDescription 45 | | DuplicatedName 46 | 47 | 48 | verticalBar : String -> String -> String -> String 49 | verticalBar comparison expected actual = 50 | [ actual 51 | , "╵" 52 | , "│ " ++ comparison 53 | , "╷" 54 | , expected 55 | ] 56 | |> String.join "\n" 57 | 58 | 59 | {-| DEPRECATED. In the future, test runners should implement versions of this 60 | that make sense for their own environments. 61 | 62 | Format test run failures in a reasonable way. 63 | 64 | -} 65 | format : String -> Reason -> String 66 | format description reason = 67 | case reason of 68 | Custom -> 69 | description 70 | 71 | Equality e a -> 72 | verticalBar description e a 73 | 74 | Comparison e a -> 75 | verticalBar description e a 76 | 77 | TODO -> 78 | description 79 | 80 | Invalid BadDescription -> 81 | if description == "" then 82 | "The empty string is not a valid test description." 83 | else 84 | "This is an invalid test description: " ++ description 85 | 86 | Invalid _ -> 87 | description 88 | 89 | ListDiff expected actual -> 90 | listDiffToString 0 91 | description 92 | { expected = expected 93 | , actual = actual 94 | } 95 | { originalExpected = expected 96 | , originalActual = actual 97 | } 98 | 99 | CollectionDiff { expected, actual, extra, missing } -> 100 | let 101 | extraStr = 102 | if List.isEmpty extra then 103 | "" 104 | else 105 | "\nThese keys are extra: " 106 | ++ (extra |> String.join ", " |> (\d -> "[ " ++ d ++ " ]")) 107 | 108 | missingStr = 109 | if List.isEmpty missing then 110 | "" 111 | else 112 | "\nThese keys are missing: " 113 | ++ (missing |> String.join ", " |> (\d -> "[ " ++ d ++ " ]")) 114 | in 115 | String.join "" 116 | [ verticalBar description expected actual 117 | , "\n" 118 | , extraStr 119 | , missingStr 120 | ] 121 | 122 | 123 | listDiffToString : 124 | Int 125 | -> String 126 | -> { expected : List String, actual : List String } 127 | -> { originalExpected : List String, originalActual : List String } 128 | -> String 129 | listDiffToString index description { expected, actual } originals = 130 | case ( expected, actual ) of 131 | ( [], [] ) -> 132 | [ "Two lists were unequal previously, yet ended up equal later." 133 | , "This should never happen!" 134 | , "Please report this bug to https://github.com/elm-community/elm-test/issues - and include these lists: " 135 | , "\n" 136 | , toString originals.originalExpected 137 | , "\n" 138 | , toString originals.originalActual 139 | ] 140 | |> String.join "" 141 | 142 | ( first :: _, [] ) -> 143 | verticalBar (description ++ " was shorter than") 144 | (toString originals.originalExpected) 145 | (toString originals.originalActual) 146 | 147 | ( [], first :: _ ) -> 148 | verticalBar (description ++ " was longer than") 149 | (toString originals.originalExpected) 150 | (toString originals.originalActual) 151 | 152 | ( firstExpected :: restExpected, firstActual :: restActual ) -> 153 | if firstExpected == firstActual then 154 | -- They're still the same so far; keep going. 155 | listDiffToString (index + 1) 156 | description 157 | { expected = restExpected 158 | , actual = restActual 159 | } 160 | originals 161 | else 162 | -- We found elements that differ; fail! 163 | String.join "" 164 | [ verticalBar description 165 | (toString originals.originalExpected) 166 | (toString originals.originalActual) 167 | , "\n\nThe first diff is at index " 168 | , toString index 169 | , ": it was `" 170 | , firstActual 171 | , "`, but `" 172 | , firstExpected 173 | , "` was expected." 174 | ] 175 | -------------------------------------------------------------------------------- /src/Util.elm: -------------------------------------------------------------------------------- 1 | module Util exposing (..) 2 | 3 | {-| This is where I'm sticking Random helper functions I don't want to add to Pcg. 4 | -} 5 | 6 | import Array exposing (Array) 7 | import Random.Pcg exposing (..) 8 | import String 9 | 10 | 11 | rangeLengthList : Int -> Int -> Generator a -> Generator (List a) 12 | rangeLengthList minLength maxLength generator = 13 | int minLength maxLength 14 | |> andThen (\len -> list len generator) 15 | 16 | 17 | rangeLengthArray : Int -> Int -> Generator a -> Generator (Array a) 18 | rangeLengthArray minLength maxLength generator = 19 | rangeLengthList minLength maxLength generator 20 | |> map Array.fromList 21 | 22 | 23 | rangeLengthString : Int -> Int -> Generator Char -> Generator String 24 | rangeLengthString minLength maxLength charGenerator = 25 | int minLength maxLength 26 | |> andThen (lengthString charGenerator) 27 | 28 | 29 | lengthString : Generator Char -> Int -> Generator String 30 | lengthString charGenerator stringLength = 31 | list stringLength charGenerator 32 | |> map String.fromList 33 | -------------------------------------------------------------------------------- /tests/FloatWithinTests.elm: -------------------------------------------------------------------------------- 1 | module FloatWithinTests exposing (floatWithinTests) 2 | 3 | import Expect exposing (FloatingPointTolerance(Absolute, AbsoluteOrRelative, Relative)) 4 | import Fuzz exposing (..) 5 | import Helpers exposing (..) 6 | import Test exposing (..) 7 | 8 | 9 | floatWithinTests : Test 10 | floatWithinTests = 11 | describe "Expect.within" 12 | [ describe "use-cases" 13 | [ fuzz float "pythagorean identity" <| 14 | \x -> 15 | sin x ^ 2 + cos x ^ 2 |> Expect.within (AbsoluteOrRelative 0.000001 0.00001) 1.0 16 | , test "floats known to not add exactly" <| 17 | \_ -> 0.1 + 0.2 |> Expect.within (Absolute 0.000000001) 0.3 18 | , test "approximation of pi" <| 19 | \_ -> 3.14 |> Expect.within (Absolute 0.01) pi 20 | , fuzz (floatRange 0.000001 100000) "relative tolerance of circle circumference using pi approximation" <| 21 | \radius -> 22 | (radius * pi) 23 | |> Expect.within (Relative 0.001) (radius * 3.14) 24 | , expectToFail <| 25 | test "approximation of pi is not considered too accurate" <| 26 | \_ -> 3.14 |> Expect.within (Absolute 0.001) pi 27 | , expectToFail <| 28 | fuzz (floatRange 0.000001 100000) "too high absolute tolerance of circle circumference using pi approximation" <| 29 | \radius -> 30 | (radius * pi) 31 | |> Expect.within (Absolute 0.001) (radius * 3.14) 32 | , expectToFail <| 33 | fuzz (floatRange 0.000001 100000) "too high relative tolerance of circle circumference using pi approximation" <| 34 | \radius -> 35 | (radius * pi) 36 | |> Expect.within (Relative 0.0001) (radius * 3.14) 37 | ] 38 | , describe "use-cases with negative nominal and/or actual values" 39 | [ test "negative nominal and actual with Absolute" <| 40 | \_ -> -2.9 |> Expect.within (Absolute 0.1) -3 41 | , test "negative nominal and actual with Relative" <| 42 | \_ -> -2.9 |> Expect.within (Relative 0.1) -3 43 | , test "negative nominal and actual with AbsoluteOrRelative and pass on Absolute" <| 44 | \_ -> -2.9 |> Expect.within (AbsoluteOrRelative 0.1 0.0001) -3 45 | , test "negative nominal and actual with AbsoluteOrRelative and pass on Relative" <| 46 | \_ -> -2.9 |> Expect.within (AbsoluteOrRelative 0.001 0.05) -3 47 | , test "negative nominal and positive actual with Absolute" <| 48 | \_ -> 0.001 |> Expect.within (Absolute 3.3) -3 49 | , test "negative nominal and positive actual with Relative" <| 50 | \_ -> 0.001 |> Expect.within (Relative 1.1) -3 51 | , test "negative actual and positive nominal with Absolute" <| 52 | \_ -> -0.001 |> Expect.within (Absolute 3.3) 3 53 | , test "negative actual and positive nominal with Relative" <| 54 | \_ -> -0.001 |> Expect.within (Relative 1.1) 3 55 | , expectToFail <| 56 | test "negative nominal should fail as actual is close, but positive with Absolute" <| 57 | \_ -> 2.9 |> Expect.within (Absolute 0.1) -3 58 | , expectToFail <| 59 | test "negative nominal should fail as actual is close, but positive with Relative" <| 60 | \_ -> 2.9 |> Expect.within (Relative 0.1) -3 61 | ] 62 | , describe "edge-cases" 63 | [ fuzz2 float float "self equality" <| 64 | \epsilon value -> 65 | let 66 | eps = 67 | if epsilon /= 0 then 68 | epsilon 69 | else 70 | 1 71 | in 72 | value |> Expect.within (Relative (abs eps)) value 73 | , fuzz float "NaN inequality" <| 74 | \epsilon -> 75 | let 76 | nan = 77 | 0.0 / 0.0 78 | in 79 | nan |> Expect.notWithin (Relative (abs epsilon)) nan 80 | , fuzz2 float float "NaN does not equal anything" <| 81 | \epsilon a -> 82 | let 83 | nan = 84 | 0.0 / 0.0 85 | in 86 | nan |> Expect.notWithin (Relative (abs epsilon)) a 87 | , fuzz float "Infinity equality" <| 88 | \epsilon -> 89 | let 90 | infinity = 91 | 1.0 / 0.0 92 | in 93 | infinity |> Expect.within (Relative (abs epsilon)) infinity 94 | , fuzz float "Negative infinity equality" <| 95 | \epsilon -> 96 | let 97 | negativeInfinity = 98 | -1.0 / 0.0 99 | in 100 | negativeInfinity |> Expect.within (Relative (abs epsilon)) negativeInfinity 101 | , fuzz3 float float float "within and notWithin should never agree on relative tolerance" <| 102 | \epsilon a b -> 103 | let 104 | withinTest = 105 | a |> Expect.within (Relative (abs epsilon)) b 106 | 107 | notWithinTest = 108 | a |> Expect.notWithin (Relative (abs epsilon)) b 109 | in 110 | different withinTest notWithinTest 111 | , fuzz3 float float float "within and notWithin should never agree on absolute tolerance" <| 112 | \epsilon a b -> 113 | let 114 | withinTest = 115 | a |> Expect.within (Absolute (abs epsilon)) b 116 | 117 | notWithinTest = 118 | a |> Expect.notWithin (Absolute (abs epsilon)) b 119 | in 120 | different withinTest notWithinTest 121 | , fuzz4 float float float float "within and notWithin should never agree on absolute or relative tolerance" <| 122 | \absoluteEpsilon relativeEpsilon a b -> 123 | let 124 | withinTest = 125 | a |> Expect.within (AbsoluteOrRelative (abs absoluteEpsilon) (abs relativeEpsilon)) b 126 | 127 | notWithinTest = 128 | a |> Expect.notWithin (AbsoluteOrRelative (abs absoluteEpsilon) (abs relativeEpsilon)) b 129 | in 130 | different withinTest notWithinTest 131 | , fuzz float "Zero equality" <| 132 | \epsilon -> 0.0 |> Expect.within (Relative (abs epsilon)) 0.0 133 | , fuzz3 float float float "within absolute commutativity" <| 134 | \epsilon a b -> 135 | same (Expect.within (Absolute (abs epsilon)) a b) (Expect.within (Absolute (abs epsilon)) b a) 136 | , fuzz3 float float float "notWithin absolute commutativity" <| 137 | \epsilon a b -> 138 | same (Expect.notWithin (Absolute (abs epsilon)) a b) (Expect.notWithin (Absolute (abs epsilon)) b a) 139 | , fuzz2 float float "within absolute reflexive" <| 140 | \epsilon a -> 141 | Expect.within (Absolute (abs epsilon)) a a 142 | , fuzz3 float float float "within relative commutativity" <| 143 | \epsilon a b -> 144 | same (Expect.within (Relative (abs epsilon)) a b) (Expect.within (Relative (abs epsilon)) b a) 145 | , fuzz3 float float float "notWithin relative commutativity" <| 146 | \epsilon a b -> 147 | same (Expect.notWithin (Relative (abs epsilon)) a b) (Expect.notWithin (Relative (abs epsilon)) b a) 148 | , fuzz2 float float "within relative reflexive" <| 149 | \epsilon a -> 150 | Expect.within (Relative (abs epsilon)) a a 151 | ] 152 | ] 153 | -------------------------------------------------------------------------------- /tests/FuzzerTests.elm: -------------------------------------------------------------------------------- 1 | module FuzzerTests exposing (fuzzerTests) 2 | 3 | import Expect 4 | import Fuzz exposing (..) 5 | import Helpers exposing (..) 6 | import Lazy.List 7 | import Random.Pcg as Random 8 | import RoseTree 9 | import Test exposing (..) 10 | import Test.Runner 11 | 12 | 13 | die : Fuzzer Int 14 | die = 15 | Fuzz.intRange 1 6 16 | 17 | 18 | seed : Fuzzer Random.Seed 19 | seed = 20 | Fuzz.custom 21 | (Random.int Random.minInt Random.maxInt |> Random.map Random.initialSeed) 22 | (always Lazy.List.empty) 23 | 24 | 25 | fuzzerTests : Test 26 | fuzzerTests = 27 | describe "Fuzzer methods that use Debug.crash don't call it" 28 | [ describe "FuzzN (uses tupleN) testing string length properties" 29 | [ fuzz2 string string "fuzz2" <| 30 | \a b -> 31 | testStringLengthIsPreserved [ a, b ] 32 | , fuzz3 string string string "fuzz3" <| 33 | \a b c -> 34 | testStringLengthIsPreserved [ a, b, c ] 35 | , fuzz4 string string string string "fuzz4" <| 36 | \a b c d -> 37 | testStringLengthIsPreserved [ a, b, c, d ] 38 | , fuzz5 string string string string string "fuzz5" <| 39 | \a b c d e -> 40 | testStringLengthIsPreserved [ a, b, c, d, e ] 41 | ] 42 | , fuzz 43 | (intRange 1 6) 44 | "intRange" 45 | (Expect.greaterThan 0) 46 | , fuzz 47 | (frequency [ ( 1, intRange 1 6 ), ( 1, intRange 1 20 ) ]) 48 | "Fuzz.frequency" 49 | (Expect.greaterThan 0) 50 | , fuzz (result string int) "Fuzz.result" <| \r -> Expect.pass 51 | , fuzz (andThen (\i -> intRange 0 (2 ^ i)) (intRange 1 8)) 52 | "Fuzz.andThen" 53 | (Expect.atMost 256) 54 | , fuzz 55 | (map2 (,) die die 56 | |> conditional 57 | { retries = 10 58 | , fallback = \( a, b ) -> ( a, (b + 1) % 6 ) 59 | , condition = \( a, b ) -> a /= b 60 | } 61 | ) 62 | "conditional: reroll dice until they are not equal" 63 | <| 64 | \( roll1, roll2 ) -> 65 | roll1 |> Expect.notEqual roll2 66 | , fuzz seed "conditional: shrunken values all pass condition" <| 67 | \seed -> 68 | let 69 | evenInt : Fuzzer Int 70 | evenInt = 71 | Fuzz.intRange 0 10 72 | |> Fuzz.conditional 73 | { retries = 3 74 | , fallback = (+) 1 75 | , condition = even 76 | } 77 | 78 | even : Int -> Bool 79 | even n = 80 | (n % 2) == 0 81 | 82 | shrinkable : Test.Runner.Shrinkable Int 83 | shrinkable = 84 | Test.Runner.fuzz evenInt 85 | |> flip Random.step seed 86 | |> Tuple.first 87 | |> Tuple.second 88 | 89 | testShrinkable : Test.Runner.Shrinkable Int -> Expect.Expectation 90 | testShrinkable shrinkable = 91 | case Test.Runner.shrink False shrinkable of 92 | Nothing -> 93 | Expect.pass 94 | 95 | Just ( value, next ) -> 96 | if even value then 97 | testShrinkable next 98 | 99 | else 100 | Expect.fail <| "Shrunken value does not pass conditional: " ++ toString value 101 | in 102 | testShrinkable shrinkable 103 | , describe "Whitebox testing using Fuzz.Internal" 104 | [ fuzz randomSeedFuzzer "the same value is generated with and without shrinking" <| 105 | \seed -> 106 | let 107 | step gen = 108 | Random.step gen seed 109 | 110 | aFuzzer = 111 | tuple5 112 | ( tuple ( list int, array float ) 113 | , maybe bool 114 | , result unit char 115 | , tuple3 116 | ( percentage 117 | , map2 (+) int int 118 | , frequency [ ( 1, constant True ), ( 3, constant False ) ] 119 | ) 120 | , tuple3 ( intRange 0 100, floatRange -51 pi, map abs int ) 121 | ) 122 | 123 | valNoShrink = 124 | aFuzzer |> Result.map (Random.map RoseTree.root >> step >> Tuple.first) 125 | 126 | valWithShrink = 127 | aFuzzer |> Result.map (step >> Tuple.first >> RoseTree.root) 128 | in 129 | Expect.equal valNoShrink valWithShrink 130 | , shrinkingTests 131 | , manualFuzzerTests 132 | ] 133 | ] 134 | 135 | 136 | shrinkingTests : Test 137 | shrinkingTests = 138 | let 139 | -- To test shrinking, we have to fail some tests so we can shrink their inputs. 140 | -- The best place we found for storing the expected last state(s) of the shrinking procedure is the description field, which is why we have this function here. 141 | -- Previously, we (ab)used Expect.true for this, but since that was removed, here we are. 142 | expectTrueAndExpectShrinkResultToEqualString label a = 143 | Expect.equal True a |> Expect.onFail label 144 | in 145 | testShrinking <| 146 | describe "tests that fail intentionally to test shrinking" 147 | [ fuzz2 int int "Every pair of ints has a zero" <| 148 | \i j -> 149 | (i == 0) 150 | || (j == 0) 151 | |> expectTrueAndExpectShrinkResultToEqualString "(1,1)" 152 | , fuzz3 int int int "Every triple of ints has a zero" <| 153 | \i j k -> 154 | (i == 0) 155 | || (j == 0) 156 | || (k == 0) 157 | |> expectTrueAndExpectShrinkResultToEqualString "(1,1,1)" 158 | , fuzz4 int int int int "Every 4-tuple of ints has a zero" <| 159 | \i j k l -> 160 | (i == 0) 161 | || (j == 0) 162 | || (k == 0) 163 | || (l == 0) 164 | |> expectTrueAndExpectShrinkResultToEqualString "(1,1,1,1)" 165 | , fuzz5 int int int int int "Every 5-tuple of ints has a zero" <| 166 | \i j k l m -> 167 | (i == 0) 168 | || (j == 0) 169 | || (k == 0) 170 | || (l == 0) 171 | || (m == 0) 172 | |> expectTrueAndExpectShrinkResultToEqualString "(1,1,1,1,1)" 173 | , fuzz (list int) "All lists are sorted" <| 174 | \aList -> 175 | let 176 | checkPair l = 177 | case l of 178 | a :: b :: more -> 179 | if a > b then 180 | False 181 | 182 | else 183 | checkPair (b :: more) 184 | 185 | _ -> 186 | True 187 | in 188 | checkPair aList |> expectTrueAndExpectShrinkResultToEqualString "[1,0]|[0,-1]" 189 | , fuzz (intRange 1 8 |> andThen (\i -> intRange 0 (2 ^ i))) "Fuzz.andThen shrinks a number" <| 190 | \i -> 191 | i <= 2 |> expectTrueAndExpectShrinkResultToEqualString "3" 192 | ] 193 | 194 | 195 | type alias ShrinkResult a = 196 | Maybe ( a, Test.Runner.Shrinkable a ) 197 | 198 | 199 | manualFuzzerTests : Test 200 | manualFuzzerTests = 201 | describe "Test.Runner.{fuzz, shrink}" 202 | [ fuzz randomSeedFuzzer "Claim there are no even numbers" <| 203 | \seed -> 204 | let 205 | -- fuzzer is guaranteed to produce an even number 206 | fuzzer = 207 | Fuzz.intRange 2 10000 208 | |> Fuzz.map 209 | (\n -> 210 | if failsTest n then 211 | n 212 | 213 | else 214 | n + 1 215 | ) 216 | 217 | failsTest n = 218 | n % 2 == 0 219 | 220 | pair = 221 | Random.step (Test.Runner.fuzz fuzzer) seed 222 | |> Tuple.first 223 | |> Just 224 | 225 | unfold acc maybePair = 226 | case maybePair of 227 | Just ( valN, shrinkN ) -> 228 | if failsTest valN then 229 | unfold (valN :: acc) (Test.Runner.shrink False shrinkN) 230 | 231 | else 232 | unfold acc (Test.Runner.shrink True shrinkN) 233 | 234 | Nothing -> 235 | acc 236 | in 237 | unfold [] pair 238 | |> Expect.all 239 | [ List.all failsTest >> Expect.equal True >> Expect.onFail "Not all elements were even" 240 | , List.head 241 | >> Maybe.map (Expect.all [ Expect.lessThan 5, Expect.atLeast 0 ]) 242 | >> Maybe.withDefault (Expect.fail "Did not cause failure") 243 | , List.reverse >> List.head >> Expect.equal (Maybe.map Tuple.first pair) 244 | ] 245 | , fuzz randomSeedFuzzer "No strings contain the letter e" <| 246 | \seed -> 247 | let 248 | -- fuzzer is guaranteed to produce a string with the letter e 249 | fuzzer = 250 | map2 (\pre suf -> pre ++ "e" ++ suf) string string 251 | 252 | failsTest = 253 | String.contains "e" 254 | 255 | pair = 256 | Random.step (Test.Runner.fuzz fuzzer) seed 257 | |> Tuple.first 258 | |> Just 259 | 260 | unfold acc maybePair = 261 | case maybePair of 262 | Just ( valN, shrinkN ) -> 263 | if failsTest valN then 264 | unfold (valN :: acc) (Test.Runner.shrink False shrinkN) 265 | 266 | else 267 | unfold acc (Test.Runner.shrink True shrinkN) 268 | 269 | Nothing -> 270 | acc 271 | in 272 | unfold [] pair 273 | |> Expect.all 274 | [ List.all failsTest >> Expect.equal True >> Expect.onFail "Not all contained the letter e" 275 | , List.head >> Expect.equal (Just "e") 276 | , List.reverse >> List.head >> Expect.equal (Maybe.map Tuple.first pair) 277 | ] 278 | , fuzz randomSeedFuzzer "List shrinker finds the smallest counter example" <| 279 | \seed -> 280 | let 281 | fuzzer : Fuzzer (List Int) 282 | fuzzer = 283 | Fuzz.list Fuzz.int 284 | 285 | allEven : List Int -> Bool 286 | allEven xs = 287 | List.all (\x -> x % 2 == 0) xs 288 | 289 | initialShrink : ShrinkResult (List Int) 290 | initialShrink = 291 | Random.step (Test.Runner.fuzz fuzzer) seed 292 | |> Tuple.first 293 | |> Just 294 | 295 | shrink : Maybe (List Int) -> ShrinkResult (List Int) -> Maybe (List Int) 296 | shrink shrunken lastShrink = 297 | case lastShrink of 298 | Just ( valN, shrinkN ) -> 299 | shrink 300 | (if allEven valN then 301 | shrunken 302 | 303 | else 304 | Just valN 305 | ) 306 | (Test.Runner.shrink (allEven valN) shrinkN) 307 | 308 | Nothing -> 309 | shrunken 310 | in 311 | case shrink Nothing initialShrink of 312 | Just shrunken -> 313 | Expect.equal [ 1 ] shrunken 314 | 315 | Nothing -> 316 | Expect.pass 317 | ] 318 | -------------------------------------------------------------------------------- /tests/Helpers.elm: -------------------------------------------------------------------------------- 1 | module Helpers exposing (different, expectPass, expectToFail, randomSeedFuzzer, same, succeeded, testShrinking, testStringLengthIsPreserved) 2 | 3 | import Expect 4 | import Fuzz exposing (Fuzzer) 5 | import Random.Pcg as Random 6 | import Shrink 7 | import Test exposing (Test) 8 | import Test.Expectation exposing (Expectation(..)) 9 | import Test.Internal as Internal 10 | import Test.Runner.Failure exposing (Reason(..)) 11 | 12 | 13 | expectPass : a -> Expectation 14 | expectPass _ = 15 | Expect.pass 16 | 17 | 18 | testStringLengthIsPreserved : List String -> Expectation 19 | testStringLengthIsPreserved strings = 20 | strings 21 | |> List.map String.length 22 | |> List.sum 23 | |> Expect.equal (String.length (List.foldl (++) "" strings)) 24 | 25 | 26 | expectToFail : Test -> Test 27 | expectToFail = 28 | expectFailureHelper (always Nothing) 29 | 30 | 31 | succeeded : Expectation -> Bool 32 | succeeded expectation = 33 | case expectation of 34 | Pass -> 35 | True 36 | 37 | Fail _ -> 38 | False 39 | 40 | 41 | passesToFails : 42 | ({ reason : Reason 43 | , description : String 44 | , given : Maybe String 45 | } 46 | -> Maybe String 47 | ) 48 | -> List Expectation 49 | -> List Expectation 50 | passesToFails f expectations = 51 | expectations 52 | |> List.filterMap (passToFail f) 53 | |> List.map Expect.fail 54 | |> (\list -> 55 | if List.isEmpty list then 56 | [ Expect.pass ] 57 | else 58 | list 59 | ) 60 | 61 | 62 | passToFail : 63 | ({ reason : Reason 64 | , description : String 65 | , given : Maybe String 66 | } 67 | -> Maybe String 68 | ) 69 | -> Expectation 70 | -> Maybe String 71 | passToFail f expectation = 72 | case expectation of 73 | Pass -> 74 | Just "Expected this test to fail, but it passed!" 75 | 76 | Fail record -> 77 | f record 78 | 79 | 80 | expectFailureHelper : ({ description : String, given : Maybe String, reason : Reason } -> Maybe String) -> Test -> Test 81 | expectFailureHelper f test = 82 | case test of 83 | Internal.UnitTest runTest -> 84 | Internal.UnitTest <| 85 | \() -> 86 | passesToFails f (runTest ()) 87 | 88 | Internal.FuzzTest runTest -> 89 | Internal.FuzzTest <| 90 | \seed runs -> 91 | passesToFails f (runTest seed runs) 92 | 93 | Internal.Labeled desc labeledTest -> 94 | Internal.Labeled desc (expectFailureHelper f labeledTest) 95 | 96 | Internal.Batch tests -> 97 | Internal.Batch (List.map (expectFailureHelper f) tests) 98 | 99 | Internal.Skipped subTest -> 100 | expectFailureHelper f subTest 101 | |> Internal.Skipped 102 | 103 | Internal.Only subTest -> 104 | expectFailureHelper f subTest 105 | |> Internal.Only 106 | 107 | 108 | testShrinking : Test -> Test 109 | testShrinking = 110 | let 111 | handleFailure { given, description } = 112 | let 113 | acceptable = 114 | String.split "|" description 115 | in 116 | case given of 117 | Nothing -> 118 | Just "Expected this test to have a given value!" 119 | 120 | Just g -> 121 | if List.member g acceptable then 122 | Nothing 123 | else 124 | Just <| "Got shrunken value " ++ g ++ " but expected " ++ String.join " or " acceptable 125 | in 126 | expectFailureHelper handleFailure 127 | 128 | 129 | {-| get a good distribution of random seeds, and don't shrink our seeds! 130 | -} 131 | randomSeedFuzzer : Fuzzer Random.Seed 132 | randomSeedFuzzer = 133 | Fuzz.custom (Random.int 0 0xFFFFFFFF) Shrink.noShrink |> Fuzz.map Random.initialSeed 134 | 135 | 136 | same : Expectation -> Expectation -> Expectation 137 | same a b = 138 | case ( a, b ) of 139 | ( Test.Expectation.Pass, Test.Expectation.Pass ) -> 140 | Test.Expectation.Pass 141 | 142 | ( Test.Expectation.Fail _, Test.Expectation.Fail _ ) -> 143 | Test.Expectation.Pass 144 | 145 | ( a, b ) -> 146 | Test.Expectation.fail { description = "expected both arguments to fail, or both to succeed", reason = Equality (toString a) (toString b) } 147 | 148 | 149 | different : Expectation -> Expectation -> Expectation 150 | different a b = 151 | case ( a, b ) of 152 | ( Test.Expectation.Pass, Test.Expectation.Fail _ ) -> 153 | Test.Expectation.Pass 154 | 155 | ( Test.Expectation.Fail _, Test.Expectation.Pass ) -> 156 | Test.Expectation.Pass 157 | 158 | ( a, b ) -> 159 | Test.Expectation.fail { description = "expected one argument to fail", reason = Equality (toString a) (toString b) } 160 | -------------------------------------------------------------------------------- /tests/Main.elm: -------------------------------------------------------------------------------- 1 | module Main exposing (..) 2 | 3 | {-| HOW TO RUN THESE TESTS 4 | 5 | $ npm test 6 | 7 | Note that this always uses an initial seed of 902101337, since it can't do effects. 8 | 9 | -} 10 | 11 | import Platform 12 | import Runner.Log 13 | import Runner.String exposing (Summary) 14 | import SeedTests 15 | import Tests 16 | 17 | 18 | main : Program Never () msg 19 | main = 20 | let 21 | program = 22 | Platform.program 23 | { init = ( (), Cmd.none ) 24 | , update = \_ _ -> ( (), Cmd.none ) 25 | , subscriptions = \_ -> Sub.none 26 | } 27 | in 28 | runAllTests program 29 | 30 | 31 | runAllTests : a -> a 32 | runAllTests a = 33 | let 34 | runSeedTest = 35 | Runner.String.runWithOptions 1 SeedTests.fixedSeed 36 | 37 | _ = 38 | [ [ Runner.String.run Tests.all ] 39 | , List.map runSeedTest SeedTests.tests 40 | , List.map (runSeedTest >> removeAutoFail) SeedTests.noAutoFail 41 | ] 42 | |> List.concat 43 | |> List.foldl combineSummaries emptySummary 44 | |> Runner.Log.logOutput 45 | in 46 | a 47 | 48 | 49 | emptySummary : Summary 50 | emptySummary = 51 | { output = "", passed = 0, failed = 0, autoFail = Nothing } 52 | 53 | 54 | {-| Considers autoFail as pass so we can actually write tests about Test.skip 55 | and Test.only which do not automatically fail. 56 | -} 57 | removeAutoFail : Summary -> Summary 58 | removeAutoFail summary = 59 | { summary | autoFail = Nothing } 60 | 61 | 62 | combineSummaries : Summary -> Summary -> Summary 63 | combineSummaries first second = 64 | { output = first.output ++ second.output 65 | , passed = first.passed + second.passed 66 | , failed = first.failed + second.failed 67 | , autoFail = 68 | case ( first.autoFail, second.autoFail ) of 69 | ( Nothing, Nothing ) -> 70 | Nothing 71 | 72 | ( Nothing, second ) -> 73 | second 74 | 75 | ( first, Nothing ) -> 76 | first 77 | 78 | ( Just first, Just second ) -> 79 | [ first, second ] 80 | |> String.join "\n" 81 | |> Just 82 | } 83 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | ## Running the tests for elm-test itself 2 | 3 | 1. `cd` into this directory 4 | 2. `npm install` 5 | 3. `elm package install --yes` 6 | 4. `npm test` 7 | -------------------------------------------------------------------------------- /tests/Runner/Log.elm: -------------------------------------------------------------------------------- 1 | module Runner.Log exposing (logOutput, run, runWithOptions) 2 | 3 | {-| Log Runner 4 | 5 | Runs a test and outputs its results using `Debug.log`, then calls `Debug.crash` 6 | if there are any failures. 7 | 8 | This is not the prettiest runner, but it is simple and cross-platform. For 9 | example, you can use it as a crude Node runner like so: 10 | 11 | $ elm-make LogRunnerExample.elm --output=elm.js 12 | $ node elm.js 13 | 14 | This will log the test results to the console, then exit with exit code 0 15 | if the tests all passed, and 1 if any failed. 16 | 17 | @docs run, runWithOptions 18 | 19 | -} 20 | 21 | import Random.Pcg as Random 22 | import Runner.String exposing (Summary) 23 | import String 24 | import Test exposing (Test) 25 | 26 | 27 | {-| Run the test using the default `Test.Runner.String` options. 28 | -} 29 | run : Test -> () 30 | run test = 31 | Runner.String.run test 32 | |> logOutput 33 | 34 | 35 | {-| Run the test using the provided options. 36 | -} 37 | runWithOptions : Int -> Random.Seed -> Test -> () 38 | runWithOptions runs seed test = 39 | Runner.String.runWithOptions runs seed test 40 | |> logOutput 41 | 42 | 43 | summarize : Summary -> String 44 | summarize { output, passed, failed, autoFail } = 45 | let 46 | headline = 47 | if failed > 0 then 48 | output ++ "\n\nTEST RUN FAILED" 49 | else 50 | case autoFail of 51 | Nothing -> 52 | "TEST RUN PASSED" 53 | 54 | Just reason -> 55 | "TEST RUN FAILED because " ++ reason 56 | in 57 | String.join "\n" 58 | [ output 59 | , headline ++ "\n" 60 | , "Passed: " ++ toString passed 61 | , "Failed: " ++ toString failed 62 | ] 63 | 64 | 65 | logOutput : Summary -> () 66 | logOutput summary = 67 | let 68 | output = 69 | summarize summary ++ "\n\nExit code" 70 | 71 | _ = 72 | if summary.failed > 0 || summary.autoFail /= Nothing then 73 | output 74 | |> flip Debug.log 1 75 | |> (\_ -> Debug.crash "FAILED TEST RUN") 76 | |> (\_ -> ()) 77 | else 78 | output 79 | |> flip Debug.log 0 80 | |> (\_ -> ()) 81 | in 82 | () 83 | -------------------------------------------------------------------------------- /tests/Runner/String.elm: -------------------------------------------------------------------------------- 1 | module Runner.String exposing (Summary, run, runWithOptions) 2 | 3 | {-| String Runner 4 | 5 | Run a test and present its results as a nicely-formatted String, along with 6 | a count of how many tests passed and failed. 7 | 8 | Note that this always uses an initial seed of 902101337, since it can't do effects. 9 | 10 | @docs Summary, run, runWithOptions 11 | 12 | -} 13 | 14 | import Expect exposing (Expectation) 15 | import Random.Pcg as Random 16 | import Runner.String.Format 17 | import Test exposing (Test) 18 | import Test.Runner exposing (Runner, SeededRunners(..)) 19 | 20 | 21 | {-| The output string, the number of passed tests, 22 | and the number of failed tests. 23 | -} 24 | type alias Summary = 25 | { output : String, passed : Int, failed : Int, autoFail : Maybe String } 26 | 27 | 28 | toOutput : Summary -> SeededRunners -> Summary 29 | toOutput summary seededRunners = 30 | let 31 | render = 32 | List.foldl (toOutputHelp []) 33 | in 34 | case seededRunners of 35 | Plain runners -> 36 | render { summary | autoFail = Nothing } runners 37 | 38 | Only runners -> 39 | render { summary | autoFail = Just "Test.only was used" } runners 40 | 41 | Skipping runners -> 42 | render { summary | autoFail = Just "Test.skip was used" } runners 43 | 44 | Invalid message -> 45 | { output = message, passed = 0, failed = 0, autoFail = Nothing } 46 | 47 | 48 | toOutputHelp : List String -> Runner -> Summary -> Summary 49 | toOutputHelp labels runner summary = 50 | runner.run () 51 | |> List.foldl fromExpectation summary 52 | 53 | 54 | fromExpectation : Expectation -> Summary -> Summary 55 | fromExpectation expectation summary = 56 | case Test.Runner.getFailureReason expectation of 57 | Nothing -> 58 | { summary | passed = summary.passed + 1 } 59 | 60 | Just { given, description, reason } -> 61 | let 62 | message = 63 | Runner.String.Format.format description reason 64 | 65 | prefix = 66 | case given of 67 | Nothing -> 68 | "" 69 | 70 | Just g -> 71 | "Given " ++ g ++ "\n\n" 72 | 73 | newOutput = 74 | "\n\n" ++ (prefix ++ indentLines message) ++ "\n" 75 | in 76 | { summary 77 | | output = summary.output ++ newOutput 78 | , failed = summary.failed + 1 79 | , passed = summary.passed 80 | } 81 | 82 | 83 | outputLabels : List String -> String 84 | outputLabels labels = 85 | labels 86 | |> Test.Runner.formatLabels ((++) "↓ ") ((++) "✗ ") 87 | |> String.join "\n" 88 | 89 | 90 | defaultSeed : Random.Seed 91 | defaultSeed = 92 | Random.initialSeed 902101337 93 | 94 | 95 | defaultRuns : Int 96 | defaultRuns = 97 | 100 98 | 99 | 100 | indentLines : String -> String 101 | indentLines str = 102 | str 103 | |> String.split "\n" 104 | |> List.map ((++) " ") 105 | |> String.join "\n" 106 | 107 | 108 | {-| Run a test and return a tuple of the output message and the number of 109 | tests that failed. 110 | 111 | Fuzz tests use a default run count of 100, and a fixed initial seed. 112 | 113 | -} 114 | run : Test -> Summary 115 | run = 116 | runWithOptions defaultRuns defaultSeed 117 | 118 | 119 | {-| Run a test and return a tuple of the output message and the number of 120 | tests that failed. 121 | -} 122 | runWithOptions : Int -> Random.Seed -> Test -> Summary 123 | runWithOptions runs seed test = 124 | let 125 | seededRunners = 126 | Test.Runner.fromTest runs seed test 127 | in 128 | toOutput 129 | { output = "" 130 | , passed = 0 131 | , failed = 0 132 | , autoFail = Just "no tests were run" 133 | } 134 | seededRunners 135 | -------------------------------------------------------------------------------- /tests/Runner/String/Format.elm: -------------------------------------------------------------------------------- 1 | module Runner.String.Format exposing (format) 2 | 3 | import Diff exposing (Change(..)) 4 | import Test.Runner.Failure exposing (InvalidReason(BadDescription), Reason(..)) 5 | 6 | 7 | format : String -> Reason -> String 8 | format description reason = 9 | case reason of 10 | Custom -> 11 | description 12 | 13 | Equality expected actual -> 14 | equalityToString { operation = description, expected = expected, actual = actual } 15 | 16 | Comparison first second -> 17 | verticalBar description first second 18 | 19 | TODO -> 20 | description 21 | 22 | Invalid BadDescription -> 23 | if description == "" then 24 | "The empty string is not a valid test description." 25 | else 26 | "This is an invalid test description: " ++ description 27 | 28 | Invalid _ -> 29 | description 30 | 31 | ListDiff expected actual -> 32 | listDiffToString 0 33 | description 34 | { expected = expected 35 | , actual = actual 36 | } 37 | { originalExpected = expected 38 | , originalActual = actual 39 | } 40 | 41 | CollectionDiff { expected, actual, extra, missing } -> 42 | let 43 | extraStr = 44 | if List.isEmpty extra then 45 | "" 46 | else 47 | "\nThese keys are extra: " 48 | ++ (extra |> String.join ", " |> (\d -> "[ " ++ d ++ " ]")) 49 | 50 | missingStr = 51 | if List.isEmpty missing then 52 | "" 53 | else 54 | "\nThese keys are missing: " 55 | ++ (missing |> String.join ", " |> (\d -> "[ " ++ d ++ " ]")) 56 | in 57 | String.join "" 58 | [ verticalBar description expected actual 59 | , "\n" 60 | , extraStr 61 | , missingStr 62 | ] 63 | 64 | 65 | verticalBar : String -> String -> String -> String 66 | verticalBar comparison expected actual = 67 | [ actual 68 | , "╵" 69 | , "│ " ++ comparison 70 | , "╷" 71 | , expected 72 | ] 73 | |> String.join "\n" 74 | 75 | 76 | listDiffToString : 77 | Int 78 | -> String 79 | -> { expected : List String, actual : List String } 80 | -> { originalExpected : List String, originalActual : List String } 81 | -> String 82 | listDiffToString index description { expected, actual } originals = 83 | case ( expected, actual ) of 84 | ( [], [] ) -> 85 | [ "Two lists were unequal previously, yet ended up equal later." 86 | , "This should never happen!" 87 | , "Please report this bug to https://github.com/elm-community/elm-test/issues - and include these lists: " 88 | , "\n" 89 | , toString originals.originalExpected 90 | , "\n" 91 | , toString originals.originalActual 92 | ] 93 | |> String.join "" 94 | 95 | ( first :: _, [] ) -> 96 | verticalBar (description ++ " was shorter than") 97 | (toString originals.originalExpected) 98 | (toString originals.originalActual) 99 | 100 | ( [], first :: _ ) -> 101 | verticalBar (description ++ " was longer than") 102 | (toString originals.originalExpected) 103 | (toString originals.originalActual) 104 | 105 | ( firstExpected :: restExpected, firstActual :: restActual ) -> 106 | if firstExpected == firstActual then 107 | -- They're still the same so far; keep going. 108 | listDiffToString (index + 1) 109 | description 110 | { expected = restExpected 111 | , actual = restActual 112 | } 113 | originals 114 | else 115 | -- We found elements that differ; fail! 116 | String.join "" 117 | [ verticalBar description 118 | (toString originals.originalExpected) 119 | (toString originals.originalActual) 120 | , "\n\nThe first diff is at index " 121 | , toString index 122 | , ": it was `" 123 | , firstActual 124 | , "`, but `" 125 | , firstExpected 126 | , "` was expected." 127 | ] 128 | 129 | 130 | equalityToString : { operation : String, expected : String, actual : String } -> String 131 | equalityToString { operation, expected, actual } = 132 | -- TODO make sure this looks reasonable for multiline strings 133 | let 134 | ( formattedExpected, belowFormattedExpected ) = 135 | Diff.diff (String.toList expected) (String.toList actual) 136 | |> List.map formatExpectedChange 137 | |> List.unzip 138 | 139 | ( formattedActual, belowFormattedActual ) = 140 | Diff.diff (String.toList actual) (String.toList expected) 141 | |> List.map formatActualChange 142 | |> List.unzip 143 | 144 | combinedExpected = 145 | String.join "\n" 146 | [ String.join "" formattedExpected 147 | , String.join "" belowFormattedExpected 148 | ] 149 | 150 | combinedActual = 151 | String.join "\n" 152 | [ String.join "" formattedActual 153 | , String.join "" belowFormattedActual 154 | ] 155 | in 156 | verticalBar operation combinedExpected combinedActual 157 | 158 | 159 | formatExpectedChange : Change Char -> ( String, String ) 160 | formatExpectedChange diff = 161 | case diff of 162 | Added char -> 163 | ( "", "" ) 164 | 165 | Removed char -> 166 | ( String.fromChar char, "▲" ) 167 | 168 | NoChange char -> 169 | ( String.fromChar char, " " ) 170 | 171 | 172 | formatActualChange : Change Char -> ( String, String ) 173 | formatActualChange diff = 174 | case diff of 175 | Added char -> 176 | ( "", "" ) 177 | 178 | Removed char -> 179 | ( "▼", String.fromChar char ) 180 | 181 | NoChange char -> 182 | ( " ", String.fromChar char ) 183 | -------------------------------------------------------------------------------- /tests/RunnerTests.elm: -------------------------------------------------------------------------------- 1 | module RunnerTests exposing (all) 2 | 3 | import Expect 4 | import Fuzz exposing (..) 5 | import Helpers exposing (expectPass) 6 | import Random.Pcg as Random 7 | import Test exposing (..) 8 | import Test.Runner exposing (SeededRunners(..)) 9 | 10 | 11 | all : Test 12 | all = 13 | Test.concat 14 | [ fromTest ] 15 | 16 | 17 | toSeededRunners : Test -> SeededRunners 18 | toSeededRunners = 19 | Test.Runner.fromTest 5 (Random.initialSeed 42) 20 | 21 | 22 | fromTest : Test 23 | fromTest = 24 | describe "TestRunner.fromTest" 25 | [ describe "test length" 26 | [ fuzz2 int int "only positive tests runs are valid" <| 27 | \runs intSeed -> 28 | case Test.Runner.fromTest runs (Random.initialSeed intSeed) passing of 29 | Invalid str -> 30 | if runs > 0 then 31 | Expect.fail ("Expected a run count of " ++ toString runs ++ " to be valid, but was invalid with this message: " ++ toString str) 32 | else 33 | Expect.pass 34 | 35 | val -> 36 | if runs > 0 then 37 | Expect.pass 38 | else 39 | Expect.fail ("Expected a run count of " ++ toString runs ++ " to be invalid, but was valid with this value: " ++ toString val) 40 | , test "an only inside another only has no effect" <| 41 | \_ -> 42 | let 43 | runners = 44 | toSeededRunners <| 45 | describe "three tests" 46 | [ test "passes" expectPass 47 | , Test.only <| 48 | describe "two tests" 49 | [ test "fails" <| 50 | \_ -> Expect.fail "failed on purpose" 51 | , Test.only <| 52 | test "is an only" <| 53 | \_ -> Expect.fail "failed on purpose" 54 | ] 55 | ] 56 | in 57 | case runners of 58 | Only runners -> 59 | runners 60 | |> List.length 61 | |> Expect.equal 2 62 | 63 | val -> 64 | Expect.fail ("Expected SeededRunner to be Only, but was " ++ toString val) 65 | , test "a skip inside an only takes effect" <| 66 | \_ -> 67 | let 68 | runners = 69 | toSeededRunners <| 70 | describe "three tests" 71 | [ test "passes" expectPass 72 | , Test.only <| 73 | describe "two tests" 74 | [ test "fails" <| 75 | \_ -> Expect.fail "failed on purpose" 76 | , Test.skip <| 77 | test "is skipped" <| 78 | \_ -> Expect.fail "failed on purpose" 79 | ] 80 | ] 81 | in 82 | case runners of 83 | Only runners -> 84 | runners 85 | |> List.length 86 | |> Expect.equal 1 87 | 88 | val -> 89 | Expect.fail ("Expected SeededRunner to be Only, but was " ++ toString val) 90 | , test "an only inside a skip has no effect" <| 91 | \_ -> 92 | let 93 | runners = 94 | toSeededRunners <| 95 | describe "three tests" 96 | [ test "passes" expectPass 97 | , Test.skip <| 98 | describe "two tests" 99 | [ test "fails" <| 100 | \_ -> Expect.fail "failed on purpose" 101 | , Test.only <| 102 | test "is skipped" <| 103 | \_ -> Expect.fail "failed on purpose" 104 | ] 105 | ] 106 | in 107 | case runners of 108 | Skipping runners -> 109 | runners 110 | |> List.length 111 | |> Expect.equal 1 112 | 113 | val -> 114 | Expect.fail ("Expected SeededRunner to be Skipping, but was " ++ toString val) 115 | , test "a test that uses only is an Only summary" <| 116 | \_ -> 117 | case toSeededRunners (Test.only <| test "passes" expectPass) of 118 | Only runners -> 119 | runners 120 | |> List.length 121 | |> Expect.equal 1 122 | 123 | val -> 124 | Expect.fail ("Expected SeededRunner to be Only, but was " ++ toString val) 125 | , test "a skip inside another skip has no effect" <| 126 | \_ -> 127 | let 128 | runners = 129 | toSeededRunners <| 130 | describe "three tests" 131 | [ test "passes" expectPass 132 | , Test.skip <| 133 | describe "two tests" 134 | [ test "fails" <| 135 | \_ -> Expect.fail "failed on purpose" 136 | , Test.skip <| 137 | test "is skipped" <| 138 | \_ -> Expect.fail "failed on purpose" 139 | ] 140 | ] 141 | in 142 | case runners of 143 | Skipping runners -> 144 | runners 145 | |> List.length 146 | |> Expect.equal 1 147 | 148 | val -> 149 | Expect.fail ("Expected SeededRunner to be Skipping, but was " ++ toString val) 150 | , test "a pair of tests where one uses skip is a Skipping summary" <| 151 | \_ -> 152 | let 153 | runners = 154 | toSeededRunners <| 155 | describe "two tests" 156 | [ test "passes" expectPass 157 | , Test.skip <| 158 | test "fails" <| 159 | \_ -> Expect.fail "failed on purpose" 160 | ] 161 | in 162 | case runners of 163 | Skipping runners -> 164 | runners 165 | |> List.length 166 | |> Expect.equal 1 167 | 168 | val -> 169 | Expect.fail ("Expected SeededRunner to be Skipping, but was " ++ toString val) 170 | , test "when all tests are skipped, we get an empty Skipping summary" <| 171 | \_ -> 172 | case toSeededRunners (Test.skip <| test "passes" expectPass) of 173 | Skipping runners -> 174 | runners 175 | |> List.length 176 | |> Expect.equal 0 177 | 178 | val -> 179 | Expect.fail ("Expected SeededRunner to be Skipping, but was " ++ toString val) 180 | , test "a test that does not use only or skip is a Plain summary" <| 181 | \_ -> 182 | case toSeededRunners (test "passes" expectPass) of 183 | Plain runners -> 184 | runners 185 | |> List.length 186 | |> Expect.equal 1 187 | 188 | val -> 189 | Expect.fail ("Expected SeededRunner to be Plain, but was " ++ toString val) 190 | ] 191 | ] 192 | 193 | 194 | passing : Test 195 | passing = 196 | test "A passing test" expectPass 197 | -------------------------------------------------------------------------------- /tests/SeedTests.elm: -------------------------------------------------------------------------------- 1 | module SeedTests exposing (fixedSeed, noAutoFail, tests) 2 | 3 | import Expect exposing (FloatingPointTolerance(Absolute, AbsoluteOrRelative, Relative)) 4 | import Fuzz exposing (..) 5 | import Random.Pcg as Random 6 | import Test exposing (..) 7 | 8 | 9 | -- NOTE: These tests are only here so that we can watch out for regressions. All constants in this file are what the implementation happened to output, not what we expected the implementation to output. 10 | 11 | 12 | expectedNum : Int 13 | expectedNum = 14 | -3954212174 15 | 16 | 17 | oneSeedAlreadyDistributed : Int 18 | oneSeedAlreadyDistributed = 19 | 198384431 20 | 21 | 22 | fixedSeed : Random.Seed 23 | fixedSeed = 24 | Random.initialSeed 133742 25 | 26 | 27 | {-| Most of the tests will use this, but we won't run it directly. 28 | 29 | When these tests are run using fixedSeed and a run count of 1, this is the 30 | exact number they will get when the description around this fuzz test is 31 | exactly the string "Seed test". 32 | 33 | -} 34 | fuzzTest : Test 35 | fuzzTest = 36 | fuzz int "It receives the expected number" <| 37 | \num -> 38 | Expect.equal num expectedNum 39 | 40 | 41 | fuzzTestAfterOneDistributed : Test 42 | fuzzTestAfterOneDistributed = 43 | fuzz int "This should be different than expectedNum, because there is a fuzz test before it." <| 44 | \num -> 45 | Expect.equal num oneSeedAlreadyDistributed 46 | 47 | 48 | tests : List Test 49 | tests = 50 | [ describe "Seed test" 51 | [ fuzzTest ] 52 | , describe "Seed test" 53 | [ fuzz int "It receives the expected number even though this text is different" <| 54 | \num -> 55 | Expect.equal num expectedNum 56 | ] 57 | , describe "Seed test" 58 | [ describe "Nested describes shouldn't affect seed distribution" 59 | [ fuzzTest ] 60 | ] 61 | , describe "Seed test" 62 | [ test "Unit tests before should not affect seed distribution" <| 63 | \_ -> 64 | Expect.pass 65 | , fuzzTest 66 | , test "Unit tests after should not affect seed distribution" <| 67 | \_ -> 68 | Expect.pass 69 | ] 70 | , -- Wrapping in a Test.concat shouldn't change anything 71 | Test.concat 72 | [ describe "Seed test" 73 | [ fuzzTest ] 74 | ] 75 | , -- Wrapping in a Test.concat wth unit tests shouldn't change anything 76 | Test.concat 77 | [ describe "Seed test" 78 | [ test "Unit tests before should not affect seed distribution" <| 79 | \_ -> 80 | Expect.pass 81 | , fuzzTest 82 | , test "Unit tests after should not affect seed distribution" <| 83 | \_ -> 84 | Expect.pass 85 | ] 86 | ] 87 | , -- Putting a fuzz test before it, within a second label, *should* change things 88 | Test.concat 89 | [ describe "Seed test" 90 | [ fuzzTest 91 | , fuzzTestAfterOneDistributed 92 | ] 93 | ] 94 | , Test.concat 95 | [ fuzz int "top-level fuzz tests don't affect subsequent top-level fuzz tests, since they use their labels to get different seeds" <| 96 | \num -> 97 | Expect.equal num 409469537 98 | , describe "Seed test" 99 | [ fuzzTest ] 100 | , describe "another top-level fuzz test" 101 | [ fuzz int "it still gets different values, due to computing the seed as a hash of the label, and these labels must be unique" <| 102 | \num -> 103 | Expect.equal num 0 104 | ] 105 | ] 106 | , describe "Fuzz tests with different outer describe texts get different seeds" 107 | [ fuzz int "It receives the expected number" <| 108 | \num -> 109 | Expect.equal num 2049737128 110 | ] 111 | ] 112 | 113 | 114 | noAutoFail : List Test 115 | noAutoFail = 116 | [ -- Test.skip does not affect seed distribution 117 | Test.concat 118 | [ describe "Seed test" 119 | [ skip fuzzTest 120 | , fuzzTestAfterOneDistributed 121 | ] 122 | ] 123 | , -- Test.only does not affect seed distribution 124 | Test.concat 125 | [ describe "Seed test" 126 | [ only fuzzTest ] 127 | ] 128 | , -- Test.only skips the other tests in question 129 | Test.concat 130 | [ describe "Seed test" 131 | [ skip <| 132 | test "Autofail" <| 133 | \_ -> 134 | Expect.fail "Test.skip is broken! This should not have been run." 135 | , fuzzTest 136 | ] 137 | ] 138 | , -- Test.only skips the other tests. 139 | Test.concat 140 | [ describe "Seed test" 141 | [ only <| 142 | fuzz int "No Autofail here" <| 143 | \num -> 144 | Expect.equal num expectedNum 145 | , test "This should never get run" <| 146 | \() -> 147 | Expect.fail "Test.only is broken! This should not have been run." 148 | ] 149 | ] 150 | , -- Test.skip skips the test in question 151 | describe "Seed test" 152 | [ skip <| 153 | fuzz int "Skip test sanity check" <| 154 | \_ -> 155 | Expect.fail "Test.skip is broken! This should not have been run." 156 | , fuzzTestAfterOneDistributed 157 | ] 158 | , -- the previous test gets the same answer if Test.skip is removed 159 | describe "Seed test" 160 | [ fuzz int "Skip test sanity check" <| 161 | \_ -> 162 | Expect.pass 163 | , fuzzTestAfterOneDistributed 164 | ] 165 | , -- Test.only skips the other tests. 166 | describe "Seed test" 167 | [ only <| 168 | fuzz int "No Autofail here" <| 169 | \num -> 170 | Expect.equal num expectedNum 171 | , test "this should never get run" <| 172 | \() -> 173 | Expect.fail "Test.only is broken! This should not have been run." 174 | ] 175 | , -- Test.only does not affect seed distribution 176 | describe "Seed test" 177 | [ test "Autofail" <| 178 | \_ -> Expect.fail "Test.only is broken! This should not have been run." 179 | , fuzzTest 180 | , only <| 181 | fuzzTestAfterOneDistributed 182 | ] 183 | , -- the previous test gets the same answer if Test.only is removed 184 | describe "Seed test" 185 | [ test "Autofail" <| 186 | \_ -> Expect.pass 187 | , fuzzTest 188 | , fuzzTestAfterOneDistributed 189 | ] 190 | ] 191 | -------------------------------------------------------------------------------- /tests/Tests.elm: -------------------------------------------------------------------------------- 1 | module Tests exposing (all) 2 | 3 | import Expect exposing (FloatingPointTolerance(Absolute, AbsoluteOrRelative, Relative)) 4 | import FloatWithinTests exposing (floatWithinTests) 5 | import Fuzz exposing (..) 6 | import FuzzerTests exposing (fuzzerTests) 7 | import Helpers exposing (..) 8 | import Random.Pcg as Random 9 | import RunnerTests 10 | import Shrink 11 | import Test exposing (..) 12 | import Test.Expectation exposing (Expectation(..)) 13 | import Test.Runner 14 | import Test.Runner.Failure exposing (Reason(..)) 15 | 16 | 17 | all : Test 18 | all = 19 | Test.concat 20 | [ readmeExample 21 | , regressions 22 | , testTests 23 | , expectationTests 24 | , fuzzerTests 25 | , floatWithinTests 26 | , RunnerTests.all 27 | ] 28 | 29 | 30 | readmeExample : Test 31 | readmeExample = 32 | describe "The String module" 33 | [ describe "String.reverse" 34 | [ test "has no effect on a palindrome" <| 35 | \_ -> 36 | let 37 | palindrome = 38 | "hannah" 39 | in 40 | Expect.equal palindrome (String.reverse palindrome) 41 | , test "reverses a known string" <| 42 | \_ -> 43 | "ABCDEFG" 44 | |> String.reverse 45 | |> Expect.equal "GFEDCBA" 46 | , fuzz string "restores the original string if you run it again" <| 47 | \randomlyGeneratedString -> 48 | randomlyGeneratedString 49 | |> String.reverse 50 | |> String.reverse 51 | |> Expect.equal randomlyGeneratedString 52 | ] 53 | ] 54 | 55 | 56 | expectationTests : Test 57 | expectationTests = 58 | describe "Expectations" 59 | [ describe "Expect.err" 60 | [ test "passes on Err _" <| 61 | \_ -> 62 | Err 12 |> Expect.err 63 | , expectToFail <| 64 | test "passes on Ok _" <| 65 | \_ -> 66 | Ok 12 |> Expect.err 67 | ] 68 | , describe "Expect.all" 69 | [ expectToFail <| 70 | test "fails with empty list" <| 71 | \_ -> "dummy subject" |> Expect.all [] 72 | ] 73 | ] 74 | 75 | 76 | regressions : Test 77 | regressions = 78 | describe "regression tests" 79 | [ fuzz (intRange 1 32) "for #39" <| 80 | \positiveInt -> 81 | positiveInt 82 | |> Expect.greaterThan 0 83 | , fuzz 84 | (custom (Random.int 1 8) Shrink.noShrink) 85 | "fuzz tests run 100 times" 86 | (Expect.notEqual 5) 87 | |> expectToFail 88 | 89 | {- If fuzz tests actually run 100 times, then asserting that no number 90 | in 1..8 equals 5 fails with 0.999998 probability. If they only run 91 | once, or stop after a duplicate due to #127, then it's much more 92 | likely (but not guaranteed) that the 5 won't turn up. See #128. 93 | -} 94 | ] 95 | 96 | 97 | testTests : Test 98 | testTests = 99 | describe "functions that create tests" 100 | [ describe "describe" 101 | [ expectToFail <| describe "fails with empty list" [] 102 | , expectToFail <| describe "" [ test "describe with empty description fail" expectPass ] 103 | ] 104 | , describe "test" 105 | [ expectToFail <| test "" expectPass 106 | ] 107 | , describe "fuzz" 108 | [ expectToFail <| fuzz Fuzz.bool "" expectPass 109 | ] 110 | , describe "fuzzWith" 111 | [ expectToFail <| fuzzWith { runs = 0 } Fuzz.bool "nonpositive" expectPass 112 | , expectToFail <| fuzzWith { runs = 1 } Fuzz.bool "" expectPass 113 | ] 114 | , describe "Test.todo" 115 | [ expectToFail <| todo "a TODO test fails" 116 | , test "Passes are not TODO" 117 | (\_ -> Expect.pass |> Test.Runner.isTodo |> Expect.equal False) 118 | , test "Simple failures are not TODO" <| 119 | \_ -> 120 | Expect.fail "reason" |> Test.Runner.isTodo |> Expect.equal False 121 | , test "Failures with TODO reason are TODO" <| 122 | \_ -> 123 | Test.Expectation.fail { description = "", reason = TODO } 124 | |> Test.Runner.isTodo 125 | |> Expect.equal True 126 | ] 127 | , identicalNamesAreRejectedTests 128 | ] 129 | 130 | 131 | identicalNamesAreRejectedTests : Test 132 | identicalNamesAreRejectedTests = 133 | describe "Identically-named sibling and parent/child tests fail" 134 | [ expectToFail <| 135 | describe "a describe with two identically named children fails" 136 | [ test "foo" expectPass 137 | , test "foo" expectPass 138 | ] 139 | , expectToFail <| 140 | describe "a describe with the same name as a child test fails" 141 | [ test "a describe with the same name as a child test fails" expectPass 142 | ] 143 | , expectToFail <| 144 | describe "a describe with the same name as a child describe fails" 145 | [ describe "a describe with the same name as a child describe fails" 146 | [ test "a test" expectPass ] 147 | ] 148 | , expectToFail <| 149 | Test.concat 150 | [ describe "a describe with the same name as a sibling describe fails" 151 | [ test "a test" expectPass ] 152 | , describe "a describe with the same name as a sibling describe fails" 153 | [ test "another test" expectPass ] 154 | ] 155 | , expectToFail <| 156 | Test.concat 157 | [ Test.concat 158 | [ describe "a describe with the same name as a de facto sibling describe fails" 159 | [ test "a test" expectPass ] 160 | ] 161 | , describe "a describe with the same name as a de facto sibling describe fails" 162 | [ test "another test" expectPass ] 163 | ] 164 | , expectToFail <| 165 | Test.concat 166 | [ Test.concat 167 | [ describe "a describe with the same name as a de facto sibling describe fails" 168 | [ test "a test" expectPass ] 169 | ] 170 | , Test.concat 171 | [ describe "a describe with the same name as a de facto sibling describe fails" 172 | [ test "another test" expectPass ] 173 | ] 174 | ] 175 | ] 176 | -------------------------------------------------------------------------------- /tests/elm-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.1", 3 | "summary": "tests for elm-test, so you can elm-test while you elm-test", 4 | "repository": "https://github.com/elm-community/elm-test.git", 5 | "license": "BSD-3-Clause", 6 | "source-directories": [ 7 | ".", 8 | "../src" 9 | ], 10 | "exposed-modules": [], 11 | "dependencies": { 12 | "eeue56/elm-lazy-list": "1.0.0 <= v < 2.0.0", 13 | "eeue56/elm-shrink": "1.0.0 <= v < 2.0.0", 14 | "elm-lang/core": "5.0.0 <= v < 6.0.0", 15 | "jinjor/elm-diff": "1.0.0 <= v < 2.0.0", 16 | "eeue56/elm-lazy": "1.0.0 <= v < 2.0.0", 17 | "mgold/elm-random-pcg": "4.0.2 <= v < 5.0.0" 18 | }, 19 | "elm-version": "0.18.0 <= v < 0.19.0" 20 | } 21 | -------------------------------------------------------------------------------- /tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elm-test-tests", 3 | "version": "0.0.0", 4 | "description": "tests for elm-test, so you can elm-test while you elm-test", 5 | "main": "elm.js", 6 | "scripts": { 7 | "test": "node run-tests.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/elm-community/elm-test.git" 12 | }, 13 | "author": "Richard Feldman", 14 | "license": "BSD-3-Clause", 15 | "bugs": { 16 | "url": "https://github.com/elm-community/elm-test/issues" 17 | }, 18 | "homepage": "https://github.com/elm-community/elm-test#readme", 19 | "devDependencies": { 20 | "node-elm-compiler": "4.1.3" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/run-tests.js: -------------------------------------------------------------------------------- 1 | var compiler = require("node-elm-compiler"); 2 | 3 | testFile = "Main.elm"; 4 | 5 | compiler 6 | .compileToString([testFile], {}) 7 | .then(function(str) { 8 | try { 9 | eval(str); 10 | 11 | process.exit(0); 12 | } catch (err) { 13 | console.error(err); 14 | 15 | process.exit(1); 16 | } 17 | }) 18 | .catch(function(err) { 19 | console.error(err); 20 | 21 | process.exit(1); 22 | }); 23 | --------------------------------------------------------------------------------