├── .gitignore ├── examples ├── elm-package.json ├── quick_start_guide.elm └── debugging_a_failing_claim.elm ├── elm-package.json ├── src ├── Check │ ├── Test.elm │ └── Producer.elm └── Check.elm └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | elm-stuff 2 | elm.js 3 | -------------------------------------------------------------------------------- /examples/elm-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.0.1", 3 | "summary": "DO NOT RELEASE: tests for elm-check", 4 | "repository": "https://github.com/user/project.git", 5 | "license": "BSD3", 6 | "source-directories": [ 7 | ".", 8 | "../src" 9 | ], 10 | "exposed-modules": [], 11 | "dependencies": { 12 | "NoRedInk/elm-lazy-list": "2.0.0 <= v < 3.0.0", 13 | "NoRedInk/elm-random-extra": "2.1.1 <= v < 3.0.0", 14 | "NoRedInk/elm-shrink": "1.0.3 <= v < 2.0.0", 15 | "deadfoxygrandpa/elm-test": "3.1.1 <= v < 4.0.0", 16 | "elm-lang/core": "3.0.0 <= v < 4.0.0" 17 | }, 18 | "elm-version": "0.16.0 <= v < 0.17.0" 19 | } 20 | -------------------------------------------------------------------------------- /elm-package.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.1", 3 | "summary": "[deprecated] Property-based testing in Elm", 4 | "repository": "https://github.com/elm-community/elm-check.git", 5 | "license": "MIT", 6 | "source-directories": [ 7 | "src" 8 | ], 9 | "exposed-modules": [ 10 | "Check", 11 | "Check.Producer", 12 | "Check.Test" 13 | ], 14 | "dependencies": { 15 | "elm-community/elm-test": "3.0.0 <= v < 4.0.0", 16 | "elm-community/lazy-list": "1.0.0 <= v < 2.0.0", 17 | "elm-community/random-extra": "2.0.0 <= v < 3.0.0", 18 | "elm-community/shrink": "2.0.0 <= v < 3.0.0", 19 | "elm-lang/core": "5.0.0 <= v < 6.0.0", 20 | "elm-lang/trampoline": "1.0.1 <= v < 2.0.0", 21 | "mgold/elm-random-pcg": "4.0.2 <= v < 5.0.0" 22 | }, 23 | "elm-version": "0.18.0 <= v < 0.19.0" 24 | } 25 | -------------------------------------------------------------------------------- /examples/quick_start_guide.elm: -------------------------------------------------------------------------------- 1 | module Main exposing (..) 2 | 3 | import List exposing (reverse, length) 4 | import Check exposing (Claim, Evidence, suite, claim, that, is, for, quickCheck) 5 | import Check.Producer exposing (list, int) 6 | import Check.Test 7 | import ElmTest 8 | 9 | 10 | myClaims : Claim 11 | myClaims = 12 | suite "List Reverse" 13 | [ claim "Reversing a list twice yields the original list" 14 | `that` (\list -> reverse (reverse list)) 15 | `is` identity 16 | `for` list int 17 | , claim "Reversing a list does not modify its length" 18 | `that` (\list -> length (reverse list)) 19 | `is` (\list -> length list) 20 | `for` list int 21 | ] 22 | 23 | 24 | evidence : Evidence 25 | evidence = 26 | quickCheck myClaims 27 | 28 | 29 | main = 30 | ElmTest.runSuite (Check.Test.evidenceToTest evidence) 31 | -------------------------------------------------------------------------------- /examples/debugging_a_failing_claim.elm: -------------------------------------------------------------------------------- 1 | module Main exposing (..) 2 | 3 | import Check exposing (Claim, Evidence, suite, claim, that, is, for, true, quickCheck) 4 | import Check.Producer exposing (tuple, float, filter) 5 | import Check.Test 6 | import ElmTest 7 | 8 | 9 | testWithZero : Claim 10 | testWithZero = 11 | claim "Multiplication and division are inverse operations" 12 | `that` (\( x, y ) -> x * y / y) 13 | `is` (\( x, y ) -> x) 14 | `for` tuple ( float, float ) 15 | 16 | 17 | testWithoutZero : Claim 18 | testWithoutZero = 19 | claim "Multiplication and division are inverse operations, if zero is omitted" 20 | `that` (\( x, y ) -> x * y / y) 21 | `is` (\( x, y ) -> x) 22 | `for` filter (\( x, y ) -> y /= 0) (tuple ( float, float )) 23 | 24 | 25 | testForNearness : Claim 26 | testForNearness = 27 | claim "Multiplication and division are near inverse operations, if zero is omitted" 28 | `true` (\( x, y ) -> abs ((x * y / y) - x) < 1.0e-6) 29 | `for` filter (\( x, y ) -> y /= 0) (tuple ( float, float )) 30 | 31 | 32 | myClaims : Claim 33 | myClaims = 34 | suite "Claims about multiplication and division" 35 | [ testWithZero, testWithoutZero, testForNearness ] 36 | 37 | 38 | evidence : Evidence 39 | evidence = 40 | quickCheck myClaims 41 | 42 | 43 | main = 44 | ElmTest.runSuite (Check.Test.evidenceToTest evidence) 45 | -------------------------------------------------------------------------------- /src/Check/Test.elm: -------------------------------------------------------------------------------- 1 | module Check.Test exposing (evidenceToTest) 2 | 3 | {-| This module provides integration with 4 | [`elm-test`](http://package.elm-lang.org/packages/deadfoxygrandpa/elm-test/latest/). 5 | 6 | # Convert to Tests 7 | @docs evidenceToTest 8 | 9 | -} 10 | 11 | import Check 12 | import Test exposing (Test) 13 | import Expect 14 | 15 | 16 | {-| Convert elm-check's Evidence into an elm-test Test. You can use elm-test's 17 | runners to view the results of your property-based tests, alongside the results 18 | of unit tests. 19 | -} 20 | evidenceToTest : Check.Evidence -> Test 21 | evidenceToTest evidence = 22 | case evidence of 23 | Check.Multiple name more -> 24 | Test.describe name (List.map evidenceToTest more) 25 | 26 | Check.Unit (Ok { name, numberOfChecks }) -> 27 | Test.test (name ++ " [" ++ nChecks numberOfChecks ++ "]") <| 28 | \() -> Expect.pass 29 | 30 | Check.Unit (Err { name, numberOfChecks, expected, actual, counterExample }) -> 31 | Test.test name <| 32 | \() -> 33 | Expect.fail <| 34 | "\nOn check " 35 | ++ toString numberOfChecks 36 | ++ ", found counterexample: " 37 | ++ counterExample 38 | ++ "\nExpected: " 39 | ++ expected 40 | ++ "\nBut It Was: " 41 | ++ actual 42 | 43 | 44 | nChecks : Int -> String 45 | nChecks n = 46 | if n == 1 then 47 | "1 check" 48 | else 49 | toString n ++ " checks" 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Property Based Testing in Elm with elm-check 2 | 3 | > This package is **deprecated**. Use `elm-test`'s fuzz tests instead. 4 | 5 | Traditional unit-testing consists in asserting that certain inputs yield certain outputs. Property-based testing makes 6 | claims relating input and output. These claims can then be automatically tested over as many randomly-generated inputs 7 | as desired. If a failing input is found, it can be "shrunk" to compute a minimal failing case which is more 8 | representative of the bug. The goal of `elm-check` is to automate this process. 9 | 10 | This library can replace many unit tests, but it cannot test asynchronous, UI, or end-to-end functionality. 11 | 12 | ## Quick-Start Guide 13 | 14 | Suppose you wanted to test `List.reverse`. A correct implementation will obey a number of properties (or assertions), 15 | *regardless of the list being reversed*, including: 16 | 17 | 1. Reversing a list twice yields the original list. 18 | 2. Reversing does not modify the length of a list. 19 | 20 | You can make these claims in `elm-check` as follows: 21 | 22 | ```elm 23 | myClaims : Claim 24 | myClaims = 25 | suite "List Reverse" 26 | [ claim 27 | "Reversing a list twice yields the original list" 28 | `that` 29 | (\list -> reverse (reverse list)) 30 | `is` 31 | identity 32 | `for` 33 | list int 34 | 35 | , claim 36 | "Reversing a list does not modify its length" 37 | `that` 38 | (\list -> length (reverse list)) 39 | `is` 40 | (\list -> length list) 41 | `for` 42 | list int 43 | ] 44 | ``` 45 | 46 | As, you can see, `elm-check` defines a Domain-Specific Language (DSL) for writing claims. It may look odd at first, but 47 | the code is actually very straightforward to work with. 48 | 49 | > ***Straightforward?!*** It might help to review some language features being used. First, `suite` takes a string and a 50 | > list, which forms most of the code. The list actually has only two items, the result of calling `claim` twice. (See 51 | > the comma right before the second `claim`?) Backticks indicate that a function is being called infix. `(\x -> thing x)` 52 | > is an anonymous function. 53 | 54 | Let's examine each component of a claim. 55 | 56 | 1. `claim ` This is the name of the test and is used when output is displayed, so make it descriptive. 57 | 2. `that ` This is the "actual" value, the result of the code or feature under test. 58 | 3. `is ` This is the "expected" value. Think of it like a control in a science experiment. It's the value that 59 | isn't complicated. A test claims that, for any input `x`, `actual x == expected x`. 60 | 4. `for ` An `Producer` is basically a way to randomly create values for the inputs to the functions. So 61 | rather than operating on a single example, like unit testing, it can test that a relationship holds for many values. 62 | There's an entire module full of `Producer`s so you can test almost anything. 63 | 64 | We also group our two claims into a suite. Suites can be nested within other suites as deep as you like, so they're 65 | useful for organizing tests of many features or modules. 66 | 67 | Once you've built your claims, verifying them is easy: 68 | 69 | ```elm 70 | evidence : Evidence 71 | evidence = quickCheck myClaims 72 | ``` 73 | 74 | `quickCheck` will take either a single claim or a suite of claims and will run 100 checks on each claim to attempt to 75 | disprove each claim. `quickCheck` will then return a descriptive result of the checks performed, in the `Evidence` type. 76 | 77 | You can dive into these results if you like, but the simplest way to know "did my tests pass" is to use 78 | [elm-test](http://package.elm-lang.org/packages/deadfoxygrandpa/elm-test/latest). 79 | 80 | ```elm 81 | main = ElmTest.elementRunner (Check.Test.evidenceToTest evidence) 82 | ``` 83 | 84 | Running the page in `elm reactor` will inform you that all tests have passed. (You can find the complete code under 85 | `examples`.) 86 | 87 | ## Debugging a Failing Claim 88 | 89 | Suppose you start with a number `x`. Mathematically, if you multiply by another number `y`, and then divide by `y`, you 90 | should be left with `x`. You would make this claim as follows: 91 | 92 | ```elm 93 | myClaims = 94 | claim 95 | "Multiplication and division are inverse operations" 96 | `that` 97 | (\(x, y) -> x * y / y) 98 | `is` 99 | (\(x, y) -> x) 100 | `for` 101 | tuple (float, float) 102 | ``` 103 | 104 | Note that we're using the `tuple` producer because the functions we pass must take exactly one argument. If you put 105 | this into the program above, you'd get: 106 | 107 | > Multiplication and division are inverse operations: FAILED. 108 | > On check 23, found counterexample: (0,0) 109 | > Expected: 0 110 | > But It Was: NaN 111 | 112 | This result shows that `elm-check` has found a counter example, namely `(0,0)` which falsifies the claim. This is 113 | obviously true because division by 0 is undefined, hence the `NaN` value. 114 | 115 | We can easily exclude zero by filtering the producer. Change the last line to: 116 | 117 | ```elm 118 | filter (\(x, y) -> y /= 0) (tuple (float, float)) 119 | ``` 120 | 121 | This function (in `Check.Producer`) will only use values that meet our criteria (not being equal to zero). This is 122 | preferable to changing the expected and actual functions because it's simpler, and it doesn't reduce the number of 123 | inputs we try. 124 | 125 | Now we get a different error. 126 | 127 | > Multiplication and division are inverse operations, if zero is omitted: FAILED. 128 | > On check 20, found counterexample: (0.00019869294196802492,0.0001670854544888915) 129 | > Expected: 0.00019869294196802492 130 | > But It Was: 0.00019869294196802494 131 | 132 | Floating point arithmetic strikes again! Notice that the expect and the actual values only differ by a tiny amount. 133 | 134 | Instead of claiming equality, we want to claim that the two values are near to each other. In particular, we want to say 135 | that the difference of these values is very close to zero. Rather than supplying expected and actual, we will supply a 136 | function that we expect to always be true. 137 | 138 | ```elm 139 | myClaims : Claim 140 | myClaims = 141 | claim 142 | "Multiplication and division are near inverse operations" 143 | `true` 144 | (\(x, y) -> abs ((x * y / y) - x) < 1e-6) 145 | `for` 146 | filter (\(x, y) -> y /= 0) (tuple (float, float)) 147 | ``` 148 | 149 | The test now passes. This gives us confidence that multiplication and division are very nearly inverses, for any pair of 150 | floats where the second one isn't zero. 151 | 152 | ## Debugging Compiler Errors 153 | 154 | The DSL can give difficult error messages. Ensure that each claim uses one of these three patterns: 155 | 156 | 1. claim - (string) - that - (actual) - is - (expected) - for - (producer) 157 | 2. claim - (string) - true - (predicate) - for - (producer) 158 | 3. claim - (string) - false - (predicate) - for - (producer) 159 | 160 | Ensure that each of these words except `claim` is surrounded by backticks. 161 | 162 | If you're putting main claims together in a suite, ensure that you have commas between each claim. 163 | 164 | Ensure that the two functions you pass have the same type. Ensure the input type matches the producer. Ensure the 165 | output type is something equatable -- functions aren't, so be sure you fully apply them. 166 | 167 | ## Writing Good Properties 168 | 169 | It can be difficult to write claims about a system, especially if it's not simple mathematics or a data structure. 170 | 171 | [Jessica Kerr](https://vimeo.com/106759186) suggests writing "a box around the API". Rather than specifying an expected 172 | value exactly, you should try to indicate a range in which it can reasonably fall. 173 | 174 | ## Shrinking 175 | 176 | You may have noticed in the division example that the second pair of failing values were both very close to zero. This 177 | is because of a process called *shrinking*, which in the case of floats, happens to bring them closer to zero. It makes 178 | lists, strings, and most other things smaller. 179 | 180 | Here's how it works, when `elm-check` encounters a failing test, it has strategies to shrink the input that caused the 181 | failure. If any of *those* inputs cause a failure, it tries to shrink them in turn, until it has found a minimal failing 182 | test case. Small examples of failure tend to be much more helpful for debugging. 183 | 184 | Here's the thing: all of this happens automatically. You get smaller, easier-to-understand counterexamples, for free. 185 | 186 | ## Customization 187 | 188 | We used the `quickCheck` function above to run our tests. There is also `check`, which allows you to supply a random seed 189 | and specify the number of tests to run per claim, in case you think 100 is insufficient. More tests increase the 190 | likelihood of finding obscure bugs, but take longer. 191 | 192 | Once again, the easiest way to view the results of your tests is `Check.Test.evidenceToTest`. The resulting value can be 193 | used with any of `elm-test`'s runners, including on the console for CI builds. 194 | 195 | If you *really* want to explore the results of your tests, the `Evidence` type is fully exposed and includes a large 196 | amount of information. 197 | 198 | You may want to test a function whose input does not have an producer available. If possible, convert or map over an 199 | existing producer to obtain the one you need. If necessary, you can write your own because the definition of `Producer` 200 | is exported. You'll need to dive into `elm-shrink`, as well and the `Random` module. 201 | 202 | ## Upgrading from 2.x 203 | 204 | The `Investigator` type has been renamed `Producer`. You should do a find-and-replace. If you defined your own 205 | investigators, you'll need to use the type alias directly. So `investigator generator shrinker` should become `Producer 206 | generator shrinker`. `keepIf` and `dropIf` have been changed to `filter`. `void` is now `unit`. 207 | 208 | The arguments to `check` have been reordered so that the `Claim` is last. 209 | 210 | If you relied on `claimN`, `claimNTrue`, and so on, you will need to rewrite your tests in the DSL. If you used the DSL 211 | in `Check.Test`, you will need to rewrite your tests using the main DSL, and then use `Check.Test.evidenceToTest` for 212 | Integration with `elm-test`. 213 | -------------------------------------------------------------------------------- /src/Check/Producer.elm: -------------------------------------------------------------------------------- 1 | module Check.Producer exposing (..) 2 | 3 | {-| This is a library of `Producer`s you can use to supply values to your tests. 4 | You can typically pick out which ones you need according to their types. 5 | 6 | A `Producer a` knows how to create values of type `a`. It can create them 7 | randomly, and it can shrink them to more minimal values. Producers can be 8 | filtered and mapped over. 9 | 10 | # Common Producers 11 | @docs bool, int, rangeInt, float, rangeFloat, percentage, string, maybe, result, list, array 12 | 13 | ## Tuple Producers 14 | If your expected and actual functions need more than one input, pass them in as a tuple. 15 | @docs tuple, tuple3, tuple4, tuple5 16 | 17 | # Working with Producers 18 | @docs Producer, filter, convert, map 19 | 20 | # Uncommon Producers 21 | @docs unit, order 22 | 23 | ## Character Producers 24 | @docs char, upperCaseChar, lowerCaseChar, ascii, unicode 25 | 26 | -} 27 | 28 | import Array exposing (Array) 29 | import Shrink exposing (Shrinker) 30 | import Random exposing (Generator) 31 | import Random.Extra 32 | import Random.Order 33 | import Random.Char 34 | import Random.String 35 | import Random.Array 36 | 37 | 38 | {-| An Producer type is a 39 | [Random](http://package.elm-lang.org/packages/elm-lang/core/latest/Random) 40 | `Generator` paired with a shrinking strategy, or `Shrinker`. Shrinkers are defined 41 | in [`elm-community/shrink`](http://package.elm-lang.org/packages/elm-community/shrink/latest/). 42 | You will need to be familiar with both libraries to write custom producers for your own types. 43 | Here is an example for a record: 44 | 45 | type alias Position = 46 | { x : Int, y : Int } 47 | 48 | 49 | position : Producer Position 50 | position = 51 | Producer 52 | (Random.map2 Position (Random.int 0 1919) (Random.int 0 1079)) 53 | (\{ x, y } -> Shrink.map Position (Shrink.int x) `Shrink.andMap` (Shrink.int y)) 54 | 55 | Here is an example for a union type: 56 | 57 | type Question 58 | = Name String 59 | | Age Int 60 | 61 | 62 | question = 63 | let 64 | generator = 65 | Random.bool `Random.andThen` (\b -> 66 | if b then 67 | Random.map Name string.generator 68 | else 69 | Random.map Age (Random.int 0 120) 70 | ) 71 | 72 | shrinker question = 73 | case question of 74 | Name n -> 75 | Shrink.string n |> Shrink.map Name 76 | 77 | Age i -> 78 | Shrink.int i |> Shrink.map Age 79 | in 80 | Producer generator shrinker 81 | -} 82 | type alias Producer a = 83 | { generator : Generator a 84 | , shrinker : Shrinker a 85 | } 86 | 87 | 88 | {-| A producer for the unit value. Unit is a type with only one value, commonly 89 | used as a placeholder. 90 | -} 91 | unit : Producer () 92 | unit = 93 | Producer (Random.Extra.constant ()) Shrink.noShrink 94 | 95 | 96 | {-| A producer for bool values. 97 | -} 98 | bool : Producer Bool 99 | bool = 100 | Producer (Random.bool) Shrink.bool 101 | 102 | 103 | {-| A producer for order values. 104 | -} 105 | order : Producer Order 106 | order = 107 | Producer (Random.Order.order) Shrink.order 108 | 109 | 110 | {-| A producer for int values. 111 | -} 112 | int : Producer Int 113 | int = 114 | let 115 | generator = 116 | Random.Extra.frequency 117 | [ ( 3, Random.int -50 50 ) 118 | , ( 0.2, Random.Extra.constant 0 ) 119 | , ( 1, Random.int 0 (Random.maxInt - Random.minInt) ) 120 | , ( 1, Random.int (Random.minInt - Random.maxInt) 0 ) 121 | ] 122 | in 123 | Producer generator Shrink.int 124 | 125 | 126 | {-| A producer for int values within between a given minimum and maximum value, 127 | inclusive. Shrunken values will also be within the range. 128 | -} 129 | rangeInt : Int -> Int -> Producer Int 130 | rangeInt min max = 131 | Producer (Random.int min max) 132 | (Shrink.keepIf (\i -> i >= min && i <= max) Shrink.int) 133 | 134 | 135 | {-| A producer for float values. It will never produce `NaN`, `Infinity`, or `-Infinity`. 136 | -} 137 | float : Producer Float 138 | float = 139 | let 140 | generator = 141 | Random.Extra.frequency 142 | [ ( 3, Random.float -50 50 ) 143 | , ( 0.5, Random.Extra.constant 0 ) 144 | , ( 1, Random.float -1 1 ) 145 | , ( 1, Random.float 0 (toFloat <| Random.maxInt - Random.minInt) ) 146 | , ( 1, Random.float (toFloat <| Random.minInt - Random.maxInt) 0 ) 147 | ] 148 | in 149 | Producer generator Shrink.float 150 | 151 | 152 | {-| A producer for float values within between a given minimum and maximum 153 | value, inclusive. Shrunken values will also be within the range. 154 | -} 155 | rangeFloat : Float -> Float -> Producer Float 156 | rangeFloat min max = 157 | Producer (Random.float min max) 158 | (Shrink.keepIf (\i -> i >= min && i <= max) Shrink.float) 159 | 160 | 161 | {-| A producer for percentage values. Generates random floats between `0.0` and 162 | `1.0`. 163 | -} 164 | percentage : Producer Float 165 | percentage = 166 | let 167 | generator = 168 | Random.Extra.frequency 169 | [ ( 8, Random.float 0 1 ) 170 | , ( 1, Random.Extra.constant 0 ) 171 | , ( 1, Random.Extra.constant 1 ) 172 | ] 173 | in 174 | Producer generator Shrink.float 175 | 176 | 177 | {-| A producer for ASCII char values. 178 | -} 179 | ascii : Producer Char 180 | ascii = 181 | Producer Random.Char.ascii Shrink.char 182 | 183 | 184 | {-| A producer for char values. Generates random ascii chars disregarding the control 185 | characters. 186 | -} 187 | char : Producer Char 188 | char = 189 | Producer (Random.Char.char 32 126) Shrink.character 190 | 191 | 192 | {-| A producer for uppercase char values. 193 | -} 194 | upperCaseChar : Producer Char 195 | upperCaseChar = 196 | Producer Random.Char.upperCaseLatin Shrink.character 197 | 198 | 199 | {-| A producer for lowercase char values. 200 | -} 201 | lowerCaseChar : Producer Char 202 | lowerCaseChar = 203 | Producer Random.Char.lowerCaseLatin Shrink.character 204 | 205 | 206 | {-| A producer for unicode char values. 207 | -} 208 | unicode : Producer Char 209 | unicode = 210 | Producer Random.Char.unicode Shrink.char 211 | 212 | 213 | {-| A producer for string values. Generates random printable ascii strings whose 214 | length is between 0 and 10. 215 | -} 216 | string : Producer String 217 | string = 218 | Producer (Random.String.rangeLengthString 0 10 char.generator) 219 | Shrink.string 220 | 221 | 222 | {-| Given a producer of a type, create a producer of a maybe for that type. 223 | -} 224 | maybe : Producer a -> Producer (Maybe a) 225 | maybe prod = 226 | let 227 | genBool = 228 | Random.map not <| Random.Extra.oneIn 4 229 | in 230 | Producer (Random.Extra.maybe genBool prod.generator) (Shrink.maybe prod.shrinker) 231 | 232 | 233 | {-| Given producers for an error type and a success type, createa a producer for 234 | a result. 235 | -} 236 | result : Producer error -> Producer value -> Producer (Result error value) 237 | result errProd valProd = 238 | Producer (Random.Extra.result Random.bool errProd.generator valProd.generator) 239 | (Shrink.result errProd.shrinker valProd.shrinker) 240 | 241 | 242 | {-| Given a producer of a type, create a producer of a list of that type. 243 | Generates random lists of varying length, favoring shorter lists. 244 | -} 245 | list : Producer a -> Producer (List a) 246 | list prod = 247 | Producer 248 | (Random.Extra.frequency 249 | [ ( 1, Random.Extra.constant [] ) 250 | , ( 1, Random.map (\x -> [ x ]) prod.generator ) 251 | , ( 3, Random.Extra.rangeLengthList 2 10 prod.generator ) 252 | , ( 2, Random.Extra.rangeLengthList 10 100 prod.generator ) 253 | , ( 0.5, Random.Extra.rangeLengthList 100 400 prod.generator ) 254 | ] 255 | ) 256 | (Shrink.list prod.shrinker) 257 | 258 | 259 | {-| Given a producer of a type, create a producer of an array of that type. 260 | Generates random arrays of varying length, favoring shorter arrays. 261 | -} 262 | array : Producer a -> Producer (Array a) 263 | array prod = 264 | Producer 265 | (Random.Extra.frequency 266 | [ ( 1, Random.Extra.constant Array.empty ) 267 | , ( 1, Random.map (Array.repeat 1) prod.generator ) 268 | , ( 3, Random.Array.rangeLengthArray 2 10 prod.generator ) 269 | , ( 2, Random.Array.rangeLengthArray 10 100 prod.generator ) 270 | , ( 0.5, Random.Array.rangeLengthArray 100 400 prod.generator ) 271 | ] 272 | ) 273 | (Shrink.array prod.shrinker) 274 | 275 | 276 | {-| Turn a tuple of producers into a producer of tuples. 277 | -} 278 | tuple : ( Producer a, Producer b ) -> Producer ( a, b ) 279 | tuple ( prodA, prodB ) = 280 | Producer (Random.map2 (,) prodA.generator prodB.generator) 281 | (Shrink.tuple ( prodA.shrinker, prodB.shrinker )) 282 | 283 | 284 | {-| Turn a 3-tuple of producers into a producer of 3-tuples. 285 | -} 286 | tuple3 : ( Producer a, Producer b, Producer c ) -> Producer ( a, b, c ) 287 | tuple3 ( prodA, prodB, prodC ) = 288 | Producer (Random.map3 (,,) prodA.generator prodB.generator prodC.generator) 289 | (Shrink.tuple3 ( prodA.shrinker, prodB.shrinker, prodC.shrinker )) 290 | 291 | 292 | {-| Turn a 4-tuple of producers into a producer of 4-tuples. 293 | -} 294 | tuple4 : ( Producer a, Producer b, Producer c, Producer d ) -> Producer ( a, b, c, d ) 295 | tuple4 ( prodA, prodB, prodC, prodD ) = 296 | Producer (Random.map4 (,,,) prodA.generator prodB.generator prodC.generator prodD.generator) 297 | (Shrink.tuple4 ( prodA.shrinker, prodB.shrinker, prodC.shrinker, prodD.shrinker )) 298 | 299 | 300 | {-| Turn a 5-tuple of producers into a producer of 5-tuples. 301 | -} 302 | tuple5 : ( Producer a, Producer b, Producer c, Producer d, Producer e ) -> Producer ( a, b, c, d, e ) 303 | tuple5 ( prodA, prodB, prodC, prodD, prodE ) = 304 | Producer (Random.map5 (,,,,) prodA.generator prodB.generator prodC.generator prodD.generator prodE.generator) 305 | (Shrink.tuple5 ( prodA.shrinker, prodB.shrinker, prodC.shrinker, prodD.shrinker, prodE.shrinker )) 306 | 307 | 308 | {-| Filter the values from a Producer. The resulting Producer will only generate 309 | random test values or shrunken values that satisfy the predicate. The predicate 310 | must be satisfiable. 311 | -} 312 | filter : (a -> Bool) -> Producer a -> Producer a 313 | filter predicate prod = 314 | Producer (Random.Extra.filter predicate prod.generator) 315 | (Shrink.keepIf predicate prod.shrinker) 316 | 317 | 318 | {-| Convert the output of one producer to another type. This is useful if 319 | you're testing a function that expects a large model record, but you only need 320 | to randomize a few fields. You might do this several different ways for a single 321 | model, so you generate and shrink only the fields relevant to each test. 322 | 323 | type alias Person = 324 | { first : String, last : String, age : String } 325 | 326 | spy : Producer Person 327 | spy = convert (\age -> Person "James" "Bond" age) .age (rangeInt 0 120) 328 | 329 | In order for shrinking to work, you need to pass an inverse function of the 330 | function being mapped. 331 | -} 332 | convert : (a -> b) -> (b -> a) -> Producer a -> Producer b 333 | convert f g prod = 334 | Producer (Random.map f prod.generator) 335 | (Shrink.convert f g prod.shrinker) 336 | 337 | 338 | {-| Map a function over an producer. This works exactly like `convert`, 339 | except it does not require an inverse function, and consequently does no 340 | shrinking. 341 | -} 342 | map : (a -> b) -> Producer a -> Producer b 343 | map f prod = 344 | Producer (Random.map f prod.generator) 345 | Shrink.noShrink 346 | -------------------------------------------------------------------------------- /src/Check.elm: -------------------------------------------------------------------------------- 1 | module Check exposing (..) 2 | 3 | {-| 4 | 5 | A toolkit for writing property-based tests, which take the form of `Claim`s. A 6 | `Claim` is made using the provided domain-specific language (DSL). A single 7 | `Claim` can be written in one of these ways: 8 | 9 | 1. claim - (string) - that - (actual) - is - (expected) - for - (producer) 10 | 2. claim - (string) - true - (predicate) - for - (producer) 11 | 3. claim - (string) - false - (predicate) - for - (producer) 12 | 13 | 14 | For example, 15 | 16 | claim_multiplication_identity = 17 | claim 18 | "Multiplying by one does not change a number" 19 | `that` 20 | (\n -> n * 1) 21 | `is` 22 | identity 23 | `for` 24 | int 25 | 26 | See the README for more information. 27 | 28 | *Warning: The DSL follows a very strict format. Deviating from this format will 29 | yield potentially unintelligible type errors. The following functions have 30 | horrendous type signatures and you are better off ignoring them.* 31 | 32 | @docs claim, that, is, for, true, false 33 | 34 | # Group Claims 35 | @docs suite 36 | 37 | # Check a Claim 38 | @docs quickCheck, check 39 | 40 | # Types 41 | @docs Claim 42 | 43 | ## Evidence 44 | The results of checking a claim are given back in the types defined here. You 45 | can examine them yourself, or see `Check.Test` to convert them into tests to use 46 | with `elm-check`'s runners. 47 | @docs Evidence, UnitEvidence, SuccessOptions, FailureOptions 48 | -} 49 | 50 | import Lazy.List exposing (LazyList) 51 | import Random exposing (Seed, Generator) 52 | import Trampoline exposing (Trampoline(..)) 53 | import Check.Producer exposing (Producer) 54 | 55 | 56 | {-| A Claim is an object that makes a claim of truth about a system. 57 | A claim is either a function which yields evidence regarding the claim 58 | or a list of such claims. 59 | -} 60 | type Claim 61 | = Claim String (Int -> Seed -> Evidence) 62 | | Suite String (List Claim) 63 | 64 | 65 | {-| Evidence is the output from checking a claim or multiple claims. 66 | -} 67 | type Evidence 68 | = Unit UnitEvidence 69 | | Multiple String (List Evidence) 70 | 71 | 72 | {-| UnitEvidence is the concrete type returned by checking a single claim. 73 | A UnitEvidence can easily be converted to an assertion or can be considered 74 | as the result of an assertion. 75 | -} 76 | type alias UnitEvidence = 77 | Result FailureOptions SuccessOptions 78 | 79 | 80 | {-| SuccessOptions is the concrete type returned in case there is no evidence 81 | found disproving a Claim. 82 | 83 | SuccessOptions contains: 84 | 1. the `name` of the claim 85 | 2. the number of checks performed 86 | 3. the `seed` used in order to reproduce the check. 87 | -} 88 | type alias SuccessOptions = 89 | { name : String 90 | , seed : Seed 91 | , numberOfChecks : Int 92 | } 93 | 94 | 95 | {-| FailureOptions is the concrete type returned in case evidence was found 96 | disproving a Claim. 97 | 98 | FailureOptions contains: 99 | 1. the `name` of the claim 100 | 2. the minimal `counterExample` which serves as evidence that the claim is false 101 | 3. the value `expected` to be returned by the claim 102 | 4. the `actual` value returned by the claim 103 | 5. the `seed` used in order to reproduce the results 104 | 6. the number of checks performed 105 | 7. the number of shrinking operations performed 106 | 8. the original `counterExample`, `actual`, and `expected` values found prior 107 | to performing the shrinking operations. 108 | -} 109 | type alias FailureOptions = 110 | { name : String 111 | , counterExample : String 112 | , actual : String 113 | , expected : String 114 | , original : 115 | { counterExample : String 116 | , actual : String 117 | , expected : String 118 | } 119 | , seed : Seed 120 | , numberOfChecks : Int 121 | , numberOfShrinks : Int 122 | } 123 | 124 | 125 | {-| 126 | -} 127 | claim : String -> (a -> b) -> (a -> b) -> Producer a -> Claim 128 | claim name actualStatement expectedStatement producer = 129 | ------------------------------------------------------------------- 130 | -- QuickCheck Algorithm with Shrinking : 131 | -- 1. Find a counter example within a given number of checks 132 | -- 2. If there is no such counter example, return a success 133 | -- 3. Else, shrink the counter example to a minimal representation 134 | -- 4. Return a failure. 135 | ------------------------------------------------------------------- 136 | Claim name <| 137 | -- A Claim is just a function that takes a number of checks 138 | -- and a random seed and returns an `Evidence` object 139 | \numberOfChecks seed -> 140 | -- `numberOfChecks` is the given number of checks which is usually 141 | -- passed in by the `check` function. This sets an upper bound on 142 | -- the number of checks performed in order to find a counter example 143 | -- `seed` is the random seed which is usually passed in by the `check` 144 | -- function. Explictly passing random seeds allow the user to reproduce 145 | -- checks in order to re-run old checks on newer, presumably less buggy, 146 | -- code. 147 | let 148 | -- Find the original counter example. The original counter example 149 | -- is the first counter example found that disproves the claim. 150 | -- This counter example, if found, will later be shrunk into a more 151 | -- minimal version, hence "original". 152 | -- Note that since finding a counter example is a recursive process, 153 | -- trampolines are used. `findOriginalCounterExample` returns a 154 | -- trampoline. 155 | findOriginalCounterExample : Seed -> Int -> Trampoline (Result ( a, b, b, Seed, Int ) Int) 156 | findOriginalCounterExample seed currentNumberOfChecks = 157 | if currentNumberOfChecks >= numberOfChecks then 158 | ------------------------------------------------------------------ 159 | -- Stopping Condition: 160 | -- If we have checked the claim at least `numberOfChecks` times 161 | -- Then we simple return `Ok` with the number of checks signifying 162 | -- that we have failed to find a counter example. 163 | ------------------------------------------------------------------ 164 | Trampoline.done (Ok numberOfChecks) 165 | else 166 | let 167 | -------------------------------------------------------------- 168 | -- Body of loop: 169 | -- 1. We generate a new random value and the next seed using 170 | -- the producer's random generator and the previous seed. 171 | -- 2. We calculate the actual outcome and the expected 172 | -- outcome from the given `actualStatement` and 173 | -- `expectedStatement` respectively 174 | -- 3. We compare the actual and the expected 175 | -- 4. If actual equals expected, we continue the loop with 176 | -- the next seed and incrementing the current number of 177 | -- checks 178 | -- 5. Else, we have found our counter example. 179 | -------------------------------------------------------------- 180 | ( value, nextSeed ) = 181 | Random.step producer.generator seed 182 | 183 | actual = 184 | actualStatement value 185 | 186 | expected = 187 | expectedStatement value 188 | in 189 | if actual == expected then 190 | Trampoline.jump (\() -> findOriginalCounterExample nextSeed (currentNumberOfChecks + 1)) 191 | else 192 | Trampoline.done (Err ( value, actual, expected, nextSeed, currentNumberOfChecks + 1 )) 193 | 194 | originalCounterExample : Result ( a, b, b, Seed, Int ) Int 195 | originalCounterExample = 196 | Trampoline.evaluate (findOriginalCounterExample seed 0) 197 | in 198 | case originalCounterExample of 199 | ------------------------------------------------------------ 200 | -- Case: No counter examples were found 201 | -- We simply return the name of the claim, the seed, and the 202 | -- number of checks performed. 203 | ------------------------------------------------------------ 204 | Ok numberOfChecks -> 205 | Unit <| 206 | Ok 207 | { name = name 208 | , seed = seed 209 | , numberOfChecks = max 0 numberOfChecks 210 | } 211 | 212 | ------------------------------------------------------------ 213 | -- Case : A counter example was found 214 | -- We proceed to shrink the counter example to a more minimal 215 | -- representation which still disproves the claim. 216 | ------------------------------------------------------------ 217 | Err ( originalCounterExample, originalActual, originalExpected, seed, numberOfChecks ) -> 218 | let 219 | ------------------------------------------------------------------ 220 | -- Find the minimal counter example: 221 | -- 1. Given a counter example, we produce a list of values 222 | -- considered more minimal (i.e. we shrink the counter example) 223 | -- 2. We keep only the shrunken values that disprove the claim. 224 | -- 3. If there are no such shrunken value, then we consider the 225 | -- given counter example to be minimal and report the number 226 | -- of shrinking operations performed. 227 | -- 4. Else, we recurse, passing in the new shrunken value 228 | -- and incrementing the current number of shrinks counter. 229 | ------------------------------------------------------------------ 230 | -- Note that since finding the minimal counter example is a 231 | -- recursive process, trampolines are used. `shrink` returns 232 | -- a trampoline. 233 | -- shrink : a -> Int -> Trampoline (a, Int) 234 | shrink counterExample currentNumberOfShrinks = 235 | let 236 | -- Produce a list of values considered more minimal that 237 | -- the given `counterExample`. 238 | shrunkenCounterExamples : LazyList a 239 | shrunkenCounterExamples = 240 | producer.shrinker counterExample 241 | 242 | -- Keep only the counter examples that disprove the claim. 243 | -- (i.e. they violate `actual == expected`) 244 | failingShrunkenCounterExamples : LazyList a 245 | failingShrunkenCounterExamples = 246 | Lazy.List.keepIf 247 | (\shrunk -> 248 | not (actualStatement shrunk == expectedStatement shrunk) 249 | ) 250 | shrunkenCounterExamples 251 | in 252 | case Lazy.List.head failingShrunkenCounterExamples of 253 | Nothing -> 254 | -------------------------------------------------------- 255 | -- Stopping Condition : 256 | -- If there are no further shrunken counter examples 257 | -- we simply return the given counter example and report 258 | -- the number of shrinking operations performed. 259 | -------------------------------------------------------- 260 | Trampoline.done ( counterExample, currentNumberOfShrinks ) 261 | 262 | Just failing -> 263 | -------------------------------------------------------- 264 | -- Body of Loop : 265 | -- We simply recurse with the first shrunken counter 266 | -- example we can get our hands on and incrementing the 267 | -- current number of shrinking operations counter 268 | -------------------------------------------------------- 269 | Trampoline.jump (\() -> shrink failing (currentNumberOfShrinks + 1)) 270 | 271 | -- minimal : a 272 | -- numberOfShrinks : Int 273 | ( minimal, numberOfShrinks ) = 274 | Trampoline.evaluate (shrink originalCounterExample 0) 275 | 276 | actual : b 277 | actual = 278 | actualStatement minimal 279 | 280 | expected : b 281 | expected = 282 | expectedStatement minimal 283 | in 284 | -- Here, we return an `Err` signifying that a counter example was 285 | -- found. The returned record contains a number of fields and 286 | -- values useful for diagnostics, such as the counter example, 287 | -- the expected and the actual values, as well the original 288 | -- unshrunk versions, the name of the claim, the seed used to 289 | -- find the counter example, the number of checks performed to find 290 | -- the counter example, and the number of shrinking operations 291 | -- performed. 292 | Unit <| 293 | Err 294 | { name = name 295 | , seed = seed 296 | , counterExample = toString minimal 297 | , expected = toString expected 298 | , actual = toString actual 299 | , original = 300 | { counterExample = toString originalCounterExample 301 | , actual = toString originalActual 302 | , expected = toString originalExpected 303 | } 304 | , numberOfChecks = numberOfChecks 305 | , numberOfShrinks = numberOfShrinks 306 | } 307 | 308 | 309 | {-| Check a claim and produce evidence. 310 | 311 | To check a claim, you need to provide the number of checks to perform, and a 312 | random seed. You can set up a CI server to run through a large number of checks 313 | with a randomized seed. 314 | 315 | aggressiveCheck : Claim -> Evidence 316 | aggressiveCheck = 317 | check 2000 (Random.initialSeed 0xFFFF) 318 | -} 319 | check : Int -> Seed -> Claim -> Evidence 320 | check n seed claim = 321 | case claim of 322 | Claim name f -> 323 | f n seed 324 | 325 | Suite name claims -> 326 | Multiple name (List.map (check n seed) claims) 327 | 328 | 329 | {-| Quickly check a claim. 330 | 331 | This function is very useful when checking claims in local development. 332 | `quickCheck` will perform 100 checks and use `Random.initialSeed 1` as the 333 | random seed. 334 | -} 335 | quickCheck : Claim -> Evidence 336 | quickCheck = 337 | check 100 (Random.initialSeed 1) 338 | 339 | 340 | {-| Group a list of claims into a suite. This is very useful in order to 341 | group similar claims together. 342 | 343 | suite nameOfSuite listOfClaims 344 | 345 | Suites can be nested as deep as you like. 346 | 347 | suite "All tests" 348 | [ someClaim 349 | , suite "Regression tests" listOfClaims 350 | ] 351 | -} 352 | suite : String -> List Claim -> Claim 353 | suite name claims = 354 | Suite name claims 355 | 356 | 357 | {-| -} 358 | that : ((a -> b) -> (a -> b) -> Producer a -> Claim) -> (a -> b) -> ((a -> b) -> Producer a -> Claim) 359 | that f x = 360 | f x 361 | 362 | 363 | {-| -} 364 | is : ((a -> b) -> Producer a -> Claim) -> (a -> b) -> (Producer a -> Claim) 365 | is f x = 366 | f x 367 | 368 | 369 | {-| -} 370 | for : (Producer a -> Claim) -> Producer a -> Claim 371 | for f x = 372 | f x 373 | 374 | 375 | {-| -} 376 | true : ((a -> Bool) -> (a -> Bool) -> Producer a -> Claim) -> (a -> Bool) -> (Producer a -> Claim) 377 | true f pred = 378 | f pred (always True) 379 | 380 | 381 | {-| -} 382 | false : ((a -> Bool) -> (a -> Bool) -> Producer a -> Claim) -> (a -> Bool) -> (Producer a -> Claim) 383 | false f pred = 384 | f pred (always False) 385 | --------------------------------------------------------------------------------