├── .gitignore ├── LICENSE ├── README.md ├── priv └── templates │ ├── prop.erl.tpl │ ├── prop_fsm.erl.tpl │ ├── prop_statem.erl.tpl │ ├── proper.template │ ├── proper_fsm.template │ └── proper_statem.template ├── rebar.config ├── rebar.lock └── src ├── rebar3_proper.app.src ├── rebar3_proper.erl └── rebar3_proper_prv.erl /.gitignore: -------------------------------------------------------------------------------- 1 | .rebar3 2 | _* 3 | .eunit 4 | *.o 5 | *.beam 6 | *.plt 7 | *.swp 8 | *.swo 9 | .erlang.cookie 10 | ebin 11 | log 12 | erl_crash.dump 13 | .rebar 14 | _rel 15 | _deps 16 | _plugins 17 | _tdeps 18 | logs 19 | _build -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Fred Hebert, All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors 14 | may be used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Rebar3 Proper Plugin 2 | ===== 3 | 4 | Run PropEr test suites. 5 | 6 | By default, will look for all modules starting in `prop_` in the `test/` 7 | directories of a rebar3 project, and running all properties (functions of arity 8 | 0 with a `prop_` prefix) in them. 9 | 10 | Todo/Gotchas 11 | ---- 12 | 13 | - No automated tests yet since this repo runs tests for a living 14 | 15 | Use 16 | --- 17 | 18 | Add the plugin to your rebar config: 19 | 20 | %% the plugin itself 21 | {project_plugins, [rebar3_proper]}. 22 | %% The PropEr dependency is required to compile the test cases 23 | %% and will be used to run the tests as well. 24 | {profiles, 25 | [{test, [ 26 | {deps, [ 27 | %% hex 28 | {proper, "1.3.0"} 29 | %% newest from master 30 | {proper, {git, "https://github.com/proper-testing/proper.git", 31 | {branch, "master"}}} 32 | ]} 33 | ]} 34 | ]}. 35 | 36 | Then just call your plugin directly in an existing application: 37 | 38 | Usage: rebar3 proper [-d ] [-m ] [-p ] 39 | [-n ] [-v ] [-c []] 40 | [-w ] [-t ] 41 | [--retry []] [--regressions []] 42 | [--store []] [--long_result ] 43 | [--start_size ] [--max_size ] 44 | [--max_shrinks ] 45 | [--noshrink ] 46 | [--constraint_tries ] 47 | [--spec_timeout ] 48 | [--any_to_integer ] 49 | [--stop_nodes ] 50 | 51 | -d, --dir directory where the property tests are located 52 | (defaults to "test"). The directory also needs to be 53 | declared in extra_src_dirs. 54 | -m, --module name of one or more modules to test (comma-separated) 55 | -p, --prop name of properties to test within a specified module 56 | (comma-separated) 57 | -n, --numtests number of tests to run when testing a given property 58 | -s, --search_steps number of searches to run when testing a given 59 | targeted property 60 | -v, --verbose each property tested shows its output or not 61 | (defaults to true) 62 | -c, --cover generate cover data [default: false] 63 | -w, --workers number of workers to use when parallelizing property 64 | tests 65 | -t, --type this is only used when running parallel PropEr: 66 | indicates the type of the property to test, it can 67 | either be "pure" when it is side-effect and has no 68 | state, or "impure" when it does 69 | --retry If failing test case counterexamples have been 70 | stored, they are retried [default: false] 71 | --regressions replays the test cases stored in the regression 72 | file. [default: false] 73 | --store stores the last counterexample into the regression 74 | file. [default: false] 75 | --long_result enables long-result mode, displaying 76 | counter-examples on failure rather than just false 77 | --start_size specifies the initial value of the size parameter 78 | --max_size specifies the maximum value of the size parameter 79 | --max_shrinks specifies the maximum number of times a failing test 80 | case should be shrunk before returning 81 | --noshrink instructs PropEr to not attempt to shrink any 82 | failing test cases 83 | --constraint_tries specifies the maximum number of tries before the 84 | generator subsystem gives up on producing an 85 | instance that satisfies a ?SUCHTHAT constraint 86 | --spec_timeout duration, in milliseconds, after which PropEr 87 | considers an input to be failing 88 | --any_to_integer converts instances of the any() type to integers in 89 | order to speed up execution 90 | --stop_nodes this is only used when running parallel PropEr: 91 | indicates whether PropEr should restart the nodes 92 | for each impure property, when testing them in 93 | parallel, or not 94 | 95 | All of [PropEr's standard configurations](http://proper.softlab.ntua.gr/doc/proper.html#Options) 96 | that can be put in a consult file can be put in `{proper_opts, [Options]}.` in your rebar.config file. 97 | 98 | Workflow 99 | --- 100 | 101 | A workflow to handle errors and do development is being experimented with: 102 | 103 | 1. Run any properties with `rebar3 proper` 104 | 2. On a test failure, replay the last failing cases with `rebar3 proper --retry` 105 | 3. Call `rebar3 proper --store` if the cases are interesting and you want to keep them for the future. The entries will be appended in a `proper-regressions.consult` file in your configured test directory. Check in that file or edit it as you wish. 106 | 4. Use `rebar3 proper --regressions` to prevent regressions from happening by testing your code against all stored counterexamples 107 | 108 | Per-Properties Meta functions 109 | --- 110 | 111 | This plugin allows you to export additional meta functions to add per-property options and documentation. For example, in the following code: 112 | 113 | ```erlang 114 | -module(prop_demo). 115 | -include_lib("proper/include/proper.hrl"). 116 | -export([prop_demo/1]). % NOT auto-exported by PropEr, we must do it ourselves 117 | 118 | prop_demo(doc) -> 119 | %% Docs are shown when the test property fails 120 | "only properties that return `true' are seen as passing"; 121 | prop_demo(opts) -> 122 | %% Override CLI and rebar.config option for `numtests' only 123 | [{numtests, 500}]. 124 | 125 | prop_demo() -> % auto-exported by Proper 126 | ?FORALL(_N, integer(), false). % always fail 127 | 128 | prop_works() -> 129 | ?FORALL(_N, integer(), true). 130 | 131 | prop_fails() -> 132 | ?FORALL(_N, integer(), false). % fails also 133 | ``` 134 | 135 | When run, the `prop_demo/0` property will _always_ run 500 times (if it does not fail), and on failure, properties with a doc value have it displayed: 136 | 137 | ``` 138 | ... 139 | 1/3 properties passed, 2 failed 140 | ===> Failed test cases: 141 | prop_demo:prop_demo() -> false (only properties that return `true' are seen as passing) 142 | prop_demo:prop_fails() -> false 143 | ``` 144 | 145 | The meta function may be omitted entirely. 146 | 147 | 148 | Changelog 149 | ---- 150 | 151 | - 0.12.1: fix debug message to match newer rebar3 standards, fixes unintuitive handling of non-compiled directories. 152 | - 0.12.0: drop compile phase since newer rebar3 versions handle all of that for us out of the box. Eliminates old deprecation warning. 153 | - 0.11.1: fix unicode support in meta-functions output 154 | - 0.11.0: add option to set search steps for targeted properties 155 | - 0.10.4: add PropEr FSM template 156 | - 0.10.3: fix the template change, which was apparently rushed. 157 | - 0.10.2: create the regression file path if it doesn't exist; simplify prop_statem template 158 | - 0.10.1: support per-app `erl_opts` values rather than only root config 159 | - 0.10.0: support hooks for app and umbrella level; add per-property opts and docs via meta-functions; remove runtime dependency on PropEr and use the one specified by the app instead 160 | - 0.9.0: support for umbrella projects 161 | - 0.8.0: storage and replay of counterexamples 162 | - 0.7.2: rely on a non-beta PropEr version 163 | - 0.7.1: fix bug regarding lib and priv directories in code path 164 | - 0.7.0: fix bug with include paths of hrl files from parent apps, support counterexamples with --retry 165 | - 0.6.3: fix bug with cover-compiling in rebar 3.2.0 and above again 166 | - 0.6.2: fix bug with cover-compiling in rebar 3.2.0 and above 167 | - 0.6.1: fix bug on option parsing in config files 168 | - 0.5.0: switches to package dependencies 169 | - 0.4.0: switches license to BSD with templates 170 | - 0.3.0: code coverage supported 171 | - 0.2.0: basic functionality 172 | - 0.1.0: first commits 173 | -------------------------------------------------------------------------------- /priv/templates/prop.erl.tpl: -------------------------------------------------------------------------------- 1 | -module(prop_{{name}}). 2 | -include_lib("proper/include/proper.hrl"). 3 | 4 | %%%%%%%%%%%%%%%%%% 5 | %%% Properties %%% 6 | %%%%%%%%%%%%%%%%%% 7 | prop_test() -> 8 | ?FORALL(Type, mytype(), 9 | begin 10 | boolean(Type) 11 | end). 12 | 13 | %%%%%%%%%%%%%%% 14 | %%% Helpers %%% 15 | %%%%%%%%%%%%%%% 16 | boolean(_) -> true. 17 | 18 | %%%%%%%%%%%%%%%%%% 19 | %%% Generators %%% 20 | %%%%%%%%%%%%%%%%%% 21 | mytype() -> term(). 22 | -------------------------------------------------------------------------------- /priv/templates/prop_fsm.erl.tpl: -------------------------------------------------------------------------------- 1 | -module(prop_{{name}}). 2 | -include_lib("proper/include/proper.hrl"). 3 | 4 | -export([initial_state/0, initial_state_data/0, 5 | on/1, off/1, service/3, % State generators 6 | weight/3, precondition/4, postcondition/5, next_state_data/5]). 7 | 8 | prop_test() -> 9 | ?FORALL(Cmds, proper_fsm:commands(?MODULE), 10 | begin 11 | actual_system:start_link(), 12 | {History,State,Result} = proper_fsm:run_commands(?MODULE, Cmds), 13 | actual_system:stop(), 14 | ?WHENFAIL(io:format("History: ~p\nState: ~p\nResult: ~p\n", 15 | [History,State,Result]), 16 | aggregate(zip(proper_fsm:state_names(History), 17 | command_names(Cmds)), 18 | Result =:= ok)) 19 | end). 20 | 21 | -record(data, {}). 22 | 23 | %% Initial state for the state machine 24 | initial_state() -> on. 25 | %% Initial model data at the start. Should be deterministic. 26 | initial_state_data() -> #data{}. 27 | 28 | %% State commands generation 29 | on(_Data) -> [{off, {call, actual_system, some_call, [term(), term()]}}]. 30 | 31 | off(_Data) -> 32 | [{off, {call, actual_system, some_call, [term(), term()]}}, 33 | {history, {call, actual_system, some_call, [term(), term()]}}, 34 | { {service,sub,state}, {call, actual_system, some_call, [term()]}}]. 35 | 36 | service(_Sub, _State, _Data) -> 37 | [{on, {call, actual_system, some_call, [term(), term()]}}]. 38 | 39 | %% Optional callback, weight modification of transitions 40 | weight(_FromState, _ToState, _Call) -> 1. 41 | 42 | %% Picks whether a command should be valid. 43 | precondition(_From, _To, #data{}, {call, _Mod, _Fun, _Args}) -> true. 44 | 45 | %% Given the state states and data *prior* to the call 46 | %% `{call, Mod, Fun, Args}', determine if the result `Res' (coming 47 | %% from the actual system) makes sense. 48 | postcondition(_From, _To, _Data, {call, _Mod, _Fun, _Args}, _Res) -> true. 49 | 50 | %% Assuming the postcondition for a call was true, update the model 51 | %% accordingly for the test to proceed. 52 | next_state_data(_From, _To, Data, _Res, {call, _Mod, _Fun, _Args}) -> 53 | NewData = Data, 54 | NewData. 55 | -------------------------------------------------------------------------------- /priv/templates/prop_statem.erl.tpl: -------------------------------------------------------------------------------- 1 | -module(prop_{{name}}). 2 | -include_lib("proper/include/proper.hrl"). 3 | 4 | %% Model Callbacks 5 | -export([command/1, initial_state/0, next_state/3, 6 | precondition/2, postcondition/3]). 7 | 8 | %%%%%%%%%%%%%%%%%% 9 | %%% PROPERTIES %%% 10 | %%%%%%%%%%%%%%%%%% 11 | prop_test() -> 12 | ?FORALL(Cmds, commands(?MODULE), 13 | begin 14 | actual_system:start_link(), 15 | {History, State, Result} = run_commands(?MODULE, Cmds), 16 | actual_system:stop(), 17 | ?WHENFAIL(io:format("History: ~p\nState: ~p\nResult: ~p\n", 18 | [History,State,Result]), 19 | aggregate(command_names(Cmds), Result =:= ok)) 20 | end). 21 | 22 | %%%%%%%%%%%%% 23 | %%% MODEL %%% 24 | %%%%%%%%%%%%% 25 | %% @doc Initial model value at system start. Should be deterministic. 26 | initial_state() -> 27 | #{}. 28 | 29 | %% @doc List of possible commands to run against the system 30 | command(_State) -> 31 | oneof([ 32 | {call, actual_system, some_call, [term(), term()]} 33 | ]). 34 | 35 | %% @doc Determines whether a command should be valid under the 36 | %% current state. 37 | precondition(_State, {call, _Mod, _Fun, _Args}) -> 38 | true. 39 | 40 | %% @doc Given the state `State' *prior* to the call 41 | %% `{call, Mod, Fun, Args}', determine whether the result 42 | %% `Res' (coming from the actual system) makes sense. 43 | postcondition(_State, {call, _Mod, _Fun, _Args}, _Res) -> 44 | true. 45 | 46 | %% @doc Assuming the postcondition for a call was true, update the model 47 | %% accordingly for the test to proceed. 48 | next_state(State, _Res, {call, _Mod, _Fun, _Args}) -> 49 | NewState = State, 50 | NewState. 51 | -------------------------------------------------------------------------------- /priv/templates/proper.template: -------------------------------------------------------------------------------- 1 | {description, "Template for a regular PropEr suite"}. 2 | {variables, [ 3 | {name, "Name of the suite (prop_)"}, 4 | {test_dir, "test", "The directory where the test suite goes"} 5 | ]}. 6 | {template, "prop.erl.tpl", "{{test_dir}}/prop_{{name}}.erl"}. 7 | -------------------------------------------------------------------------------- /priv/templates/proper_fsm.template: -------------------------------------------------------------------------------- 1 | {description, "Template for a state machine PropEr suite"}. 2 | {variables, [ 3 | {name, "fsm", "Name of the suite (prop_)"}, 4 | {test_dir, "test", "The directory where the test suite goes"} 5 | ]}. 6 | {template, "prop_fsm.erl.tpl", "{{test_dir}}/prop_{{name}}.erl"}. 7 | 8 | 9 | -------------------------------------------------------------------------------- /priv/templates/proper_statem.template: -------------------------------------------------------------------------------- 1 | {description, "Template for a statem PropEr suite"}. 2 | {variables, [ 3 | {name, "stateful", "Name of the suite (prop_)"}, 4 | {test_dir, "test", "The directory where the test suite goes"} 5 | ]}. 6 | {template, "prop_statem.erl.tpl", "{{test_dir}}/prop_{{name}}.erl"}. 7 | 8 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% PropEr not actually needed as a dep as long as the 2 | %% parent app has it as a dependency. 3 | % {deps, [ 4 | % {proper, "1.2.0"} 5 | % ]}. 6 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | []. 2 | -------------------------------------------------------------------------------- /src/rebar3_proper.app.src: -------------------------------------------------------------------------------- 1 | {application, rebar3_proper, 2 | [{description, "Run PropEr test suites"}, 3 | {vsn, "0.12.1"}, 4 | {registered, []}, 5 | {applications, 6 | [kernel, 7 | stdlib, 8 | proper 9 | ]}, 10 | {env,[]}, 11 | {modules, []}, 12 | 13 | {licenses, ["BSD"]}, 14 | {links, [{"Github", "https://github.com/ferd/rebar3_proper"}, 15 | {"PropEr", "https://proper-testing.github.io/"}]} 16 | ]}. 17 | -------------------------------------------------------------------------------- /src/rebar3_proper.erl: -------------------------------------------------------------------------------- 1 | -module(rebar3_proper). 2 | 3 | -export([init/1]). 4 | 5 | -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. 6 | init(State) -> 7 | {ok, State1} = rebar3_proper_prv:init(State), 8 | {ok, State1}. 9 | -------------------------------------------------------------------------------- /src/rebar3_proper_prv.erl: -------------------------------------------------------------------------------- 1 | -module(rebar3_proper_prv). 2 | 3 | -export([init/1, do/1, format_error/1]). 4 | 5 | -define(PROVIDER, proper). 6 | -define(DEPS, [compile]). 7 | -define(PRV_ERROR(Reason), {error, {?MODULE, Reason}}). 8 | -define(COUNTEREXAMPLE_FILE, "rebar3_proper-counterexamples.consult"). 9 | -define(REGRESSION_FILE, "proper-regressions.consult"). 10 | 11 | %% =================================================================== 12 | %% Public API 13 | %% =================================================================== 14 | -spec init(rebar_state:t()) -> {ok, rebar_state:t()}. 15 | init(State) -> 16 | Provider = providers:create([ 17 | {name, ?PROVIDER}, % The 'user friendly' name of the task 18 | {module, ?MODULE}, % The module implementation of the task 19 | {profiles, [test]}, 20 | {bare, true}, % The task can be run by the user, always true 21 | {deps, ?DEPS}, % The list of dependencies 22 | {example, "rebar3 proper"}, % How to use the plugin 23 | {opts, proper_opts()}, % list of options understood by the plugin 24 | {short_desc, "Run PropEr test suites"}, 25 | {desc, "Run PropEr test suites"} 26 | ]), 27 | {ok, rebar_state:add_provider(State, Provider)}. 28 | 29 | 30 | -spec do(rebar_state:t()) -> {ok, rebar_state:t()} | {error, string()}. 31 | do(State) -> 32 | run_pre_hooks(State), 33 | {Opts, ProperOpts} = handle_opts(State), 34 | rebar_api:debug("{proper_opts,\n\t% general options:~n\t~p~n\t++~n" 35 | "\t% proper-specific options:~n\t~p}", 36 | [Opts, ProperOpts]), 37 | rebar_utils:update_code(rebar_state:code_paths(State, all_deps), [soft_purge]), 38 | maybe_cover_compile(State), 39 | %% needed in 3.2.0 and after -- this reloads the code paths required to 40 | %% compile everything *after* cover-compiling has cleaned up after itself 41 | %% (which incidentally clears up *our* environment too), but skips reloading 42 | %% the top level apps' own code paths since those would overwrite the cover- 43 | %% compiled code sitting in memory. The top app's path is re-added to the 44 | %% code path but not pre-loaded in memory, though. 45 | TopAppsPaths = app_paths(State), 46 | rebar_utils:update_code(rebar_state:code_paths(State, all_deps)--TopAppsPaths, [soft_purge]), 47 | FlatPaths = TopAppsPaths ++ (code:get_path() -- TopAppsPaths), 48 | true = code:set_path(FlatPaths), 49 | 50 | ensure_proper(), 51 | SysConfigs = sys_config_list(ProperOpts, Opts), 52 | Configs = lists:flatmap(fun(Filename) -> 53 | rebar_file_utils:consult_config(State, Filename) 54 | end, SysConfigs), 55 | [application:load(Application) || Config <- SysConfigs, {Application, _} <- Config], 56 | rebar_utils:reread_config(Configs), 57 | 58 | 59 | Res = case run_type(Opts) of 60 | quickcheck -> do_quickcheck(State, Opts, ProperOpts); 61 | retry -> do_retry(State, Opts, ProperOpts); 62 | regressions -> do_regressions(State, Opts, ProperOpts); 63 | store -> do_store(State, Opts, ProperOpts) 64 | end, 65 | run_post_hooks(State, Res), 66 | Res. 67 | 68 | run_type(Opts) -> 69 | case {proplists:get_value(retry, Opts, false), 70 | proplists:get_value(regressions, Opts, false), 71 | proplists:get_value(store, Opts, false)} of 72 | {true, _, _} -> retry; 73 | {_, true, _} -> regressions; 74 | {_, _, true} -> store; 75 | _ -> quickcheck 76 | end. 77 | 78 | do_quickcheck(State, Opts, ProperOpts) -> 79 | try find_properties(State, Opts) of 80 | Props -> 81 | rebar_api:debug("properties: ~p", [Props]), 82 | Failed = [{Mod, Fun, Res, proper:counterexample()} 83 | || {Mod, Fun} <- Props, 84 | Res <- [catch check(Mod, Fun, ProperOpts)], 85 | Res =/= true], 86 | rebar_api:debug("Failing Results: ~p", [Failed]), 87 | maybe_write_coverdata(State), 88 | rebar_utils:cleanup_code_path(rebar_state:code_paths(State, default)), 89 | case Failed of 90 | [] -> 91 | Tot = length(Props), 92 | rebar_api:info("~n~p/~p properties passed", [Tot, Tot]), 93 | {ok, State}; 94 | [_|_] -> 95 | Tot = length(Props), 96 | FailedCount = length(Failed), 97 | Passed = Tot - FailedCount, 98 | rebar_api:error("~n~p/~p properties passed, ~p failed", [Passed, Tot, FailedCount]), 99 | store_counterexamples(State, Failed), 100 | ?PRV_ERROR({failed, Failed}) 101 | end 102 | catch 103 | throw:{module_not_found,_Mod,_Props}=Error -> ?PRV_ERROR(Error); 104 | throw:{property_not_found,_Prop,_Mods}=Error -> ?PRV_ERROR(Error) 105 | end. 106 | 107 | do_retry(State, Opts, ProperOpts) -> 108 | Base = rebar_dir:base_dir(State), 109 | FilePath = filename:join([Base, ?COUNTEREXAMPLE_FILE]), 110 | case file:consult(FilePath) of 111 | {ok, Data} -> 112 | run_retries(State, Opts, ProperOpts, Data); 113 | {error, _} -> 114 | rebar_api:info("no counterexamples to run.", []), 115 | {ok, State} 116 | end. 117 | 118 | do_regressions(State, Opts, ProperOpts) -> 119 | Dir = proplists:get_value(dir, Opts, "test"), 120 | FilePath = filename:join([Dir, ?REGRESSION_FILE]), 121 | case file:consult(FilePath) of 122 | {ok, Data} -> 123 | run_retries(State, Opts, ProperOpts, Data); 124 | {error, _} -> 125 | rebar_api:info("no regression tests to run.", []), 126 | {ok, State} 127 | end. 128 | 129 | do_store(State, Opts, _ProperOpts) -> 130 | Base = rebar_dir:base_dir(State), 131 | FilePath = filename:join([Base, ?COUNTEREXAMPLE_FILE]), 132 | case file:consult(FilePath) of 133 | {ok, Data} -> 134 | Dir = proplists:get_value(dir, Opts, "test"), 135 | RegressionPath = filename:join([Dir, ?REGRESSION_FILE]), 136 | filelib:ensure_dir(RegressionPath), 137 | {ok, Io} = file:open(RegressionPath, [append, {encoding, utf8}]), 138 | rebar_api:debug("Storing counterexamples to ~s", [RegressionPath]), 139 | {ok, Prior} = file:consult(RegressionPath), 140 | [io:format(Io, "~n~p.~n", [{Mod,Fun,CounterEx}]) 141 | || {Mod,Fun,CounterEx} <- Data, 142 | CounterEx =/= undefined, 143 | not lists:member({Mod,Fun,CounterEx}, Prior)], % dedupe 144 | file:close(Io), 145 | {ok, State}; 146 | {error, ConsultErr} -> 147 | rebar_api:debug("counterexample file consult result: ~p", [ConsultErr]), 148 | rebar_api:info("no counterexamples to store.", []), 149 | {ok, State} 150 | end. 151 | 152 | 153 | run_retries(State, Opts, ProperOpts, CounterExamples) -> 154 | Dir = proplists:get_value(dir, Opts, "test"), 155 | {Mods, Props} = lists:unzip([{atom_to_list(M), atom_to_list(F)} 156 | || {M, F, _Args} <- CounterExamples]), 157 | try find_properties(State, Dir, Mods, Props) of 158 | Found -> 159 | ExpectedLen = length(CounterExamples), 160 | FoundLen = length(Found), 161 | rebar_api:info("Running ~p counterexamples out of ~p properties", 162 | [ExpectedLen, FoundLen]), 163 | Failed = [{M, F, Result, Args} 164 | || {M,F,Args} <- CounterExamples, 165 | lists:member({M,F}, Found), 166 | Result <- [catch retry(M, F, Args, ProperOpts)], 167 | Result =/= true], 168 | FailedCount = length(Failed), 169 | Passed = ExpectedLen - FailedCount, 170 | case Failed of 171 | [] -> 172 | rebar_api:info("~n~p/~p counterexamples passed", [Passed, ExpectedLen]), 173 | {ok, State}; 174 | [_|_] -> 175 | rebar_api:error("~n~p/~p counterexamples passed, ~p failed", 176 | [Passed, ExpectedLen, FailedCount]), 177 | ?PRV_ERROR({failed, Failed}) 178 | end 179 | catch 180 | throw:{module_not_found,_Mod,_Props}=Error -> ?PRV_ERROR(Error); 181 | throw:{property_not_found,_Prop,_Mods}=Error -> ?PRV_ERROR(Error) 182 | end. 183 | 184 | 185 | 186 | -spec format_error(any()) -> iolist(). 187 | format_error({failed, Failed}) -> 188 | ["Failed test cases:", 189 | [io_lib:format("~n~p:~p() -> ~p~ts", 190 | [M,F,Res,format_doc(M,F)]) || {M,F,Res,_} <- Failed]]; 191 | format_error({module_not_found, Mod, any}) -> 192 | io_lib:format("Module ~p does not exist or exports no properties", [Mod]); 193 | format_error({module_not_found, Mod, _}) -> 194 | io_lib:format("Module ~p does not exist", [Mod]); 195 | format_error({property_not_found, Prop, []}) -> 196 | io_lib:format("Property ~p does not belong to any module", [Prop]); 197 | format_error({property_not_found, Prop, Mods}) -> 198 | io_lib:format("Property ~p does not belong to any module in ~p", [Prop, Mods]); 199 | format_error(Reason) -> 200 | io_lib:format("~p", [Reason]). 201 | 202 | %% =================================================================== 203 | %% Private 204 | %% =================================================================== 205 | maybe_cover_compile(State) -> 206 | {RawOpts, _} = rebar_state:command_parsed_args(State), 207 | State1 = case proplists:get_value(cover, RawOpts, false) of 208 | true -> rebar_state:set(State, cover_enabled, true); 209 | false -> State 210 | end, 211 | rebar_prv_cover:maybe_cover_compile(State1). 212 | 213 | maybe_write_coverdata(State) -> 214 | {RawOpts, _} = rebar_state:command_parsed_args(State), 215 | State1 = case proplists:get_value(cover, RawOpts, false) of 216 | true -> rebar_state:set(State, cover_enabled, true); 217 | false -> State 218 | end, 219 | rebar_prv_cover:maybe_write_coverdata(State1, ?PROVIDER). 220 | 221 | ensure_proper() -> 222 | try proper:module_info() of 223 | _ -> ok 224 | catch 225 | error:undef -> 226 | rebar_api:abort("PropEr not found. Add it as a dependency of " 227 | "the application you are testing.", []) 228 | end. 229 | 230 | check(Mod, Fun, Opts) -> 231 | rebar_api:info("Testing ~p:~p()", [Mod, Fun]), 232 | NewOpts = fetch_opts(Mod, Fun, Opts), 233 | proper:quickcheck(Mod:Fun(), NewOpts). 234 | 235 | retry(Mod, Fun, Args, Opts) -> 236 | rebar_api:info("Retrying ~p:~p()", [Mod, Fun]), 237 | NewOpts = fetch_opts(Mod, Fun, Opts), 238 | proper:check(Mod:Fun(), Args, NewOpts). 239 | 240 | fetch_opts(Mod, Fun, Opts) -> 241 | try Mod:Fun(opts) of 242 | TestOpts -> 243 | rebar_api:debug("Custom test options found for ~p:~p():~n\t~p", 244 | [Mod, Fun, TestOpts]), 245 | TestOpts ++ Opts 246 | catch 247 | error:E when E == undef; E == function_clause -> 248 | rebar_api:debug("~p:~p(opts) not found; using predefined options", 249 | [Mod, Fun]), 250 | Opts 251 | end. 252 | 253 | store_counterexamples(State, Failed) -> 254 | Base = rebar_dir:base_dir(State), 255 | FilePath = filename:join([Base, ?COUNTEREXAMPLE_FILE]), 256 | {ok, Io} = file:open(FilePath, [write, {encoding, utf8}]), 257 | rebar_api:debug("Writing counterexamples to ~s", [FilePath]), 258 | %% Then run as proper:check(Mod:Fun(), CounterEx) 259 | [io:format(Io, "~p.~n", [{Mod,Fun,CounterEx}]) || {Mod,Fun,_,CounterEx} <- Failed, 260 | CounterEx =/= undefined], 261 | file:close(Io), 262 | ok. 263 | 264 | find_properties(State, Opts) -> 265 | Dir = proplists:get_value(dir, Opts, "test"), 266 | Mods = proplists:get_value(module, Opts, any), 267 | Props = proplists:get_value(properties, Opts, any), 268 | Found = find_properties(State, Dir, Mods, Props), 269 | rebar_api:debug("Found: ~p", [Found]), 270 | {ModsFound0, PropsFound0} = lists:unzip(Found), 271 | ModsFound = [atom_to_list(Mod) || Mod <- ModsFound0], 272 | PropsFound = [atom_to_list(Prop) || Prop <- PropsFound0], 273 | Props =/= any andalso 274 | [throw({property_not_found, Prop, Mods}) 275 | || Prop <- Props, not lists:member(Prop, PropsFound)], 276 | Mods =/= any andalso 277 | [throw({module_not_found, Mod, Props}) 278 | || Mod <- Mods, not lists:member(Mod, ModsFound)], 279 | Found. 280 | 281 | find_properties(State, Dir, Mods, Props) -> 282 | %% Fetch directories and app configs 283 | RawDirs = [{{rebar_app_info:name(App), 284 | filename:join([rebar_app_info:out_dir(App), Dir])}, 285 | filename:join(rebar_app_info:dir(App), Dir)} 286 | || App <- rebar_state:project_apps(State), 287 | not rebar_app_info:is_checkout(App)], 288 | %% Pick a root test directory for umbrella apps 289 | UmbrellaDir = 290 | [{{<<"root">>, 291 | filename:join(rebar_dir:base_dir(State), "prop_"++Dir)}, 292 | P} || P <- [make_absolute_path(filename:join([".", Dir]))], 293 | not lists:member(P, [D || {_,D} <- RawDirs])], 294 | TestDirs = RawDirs ++ UmbrellaDir, 295 | rebar_api:debug("SearchDirs: ~p", [TestDirs]), 296 | %% Keep directories with properties in them 297 | Dirs = [{App, TestDir} 298 | || {App, TestDir} <- TestDirs, 299 | {ok, Files} <- [file:list_dir(TestDir)], 300 | lists:any(fun(File) -> prop_suite(Mods, File) end, Files)], 301 | [Prop || {_, TestDir} <- Dirs, 302 | {ok, Files} <- [file:list_dir(TestDir)], 303 | File <- Files, 304 | prop_suite(Mods, File), 305 | mod_compiled(module(File), TestDir), 306 | Prop <- properties(Props, module(File))]. 307 | 308 | prop_suite(Mods, File) -> 309 | Mod = filename:basename(File, ".erl"), 310 | filename:extension(File) =:= ".erl" 311 | andalso 312 | ((Mods =:= any andalso lists:prefix("prop_", Mod)) 313 | orelse 314 | (Mods =/= any andalso lists:member(Mod, Mods))). 315 | 316 | module(File) -> 317 | list_to_atom(filename:basename(File, ".erl")). 318 | 319 | mod_compiled(Mod, TestDir) -> 320 | try Mod:module_info() of 321 | _ -> true 322 | catch 323 | error:undef when TestDir =:= "test" -> 324 | rebar_api:debug("Skipping module ~p since it was not compiled.", 325 | [Mod]), 326 | false; 327 | error:undef -> 328 | rebar_api:debug("Skipping module ~p since it was not compiled. " 329 | "Verify presence in extra_src_dirs", [Mod]), 330 | false 331 | end. 332 | 333 | properties(any, Mod) -> 334 | [{Mod, Prop} || {Prop,0} <- Mod:module_info(exports), 335 | prop_prefix(Prop)]; 336 | properties(Props, Mod) -> 337 | [{Mod, Prop} || {Prop,0} <- Mod:module_info(exports), 338 | lists:member(atom_to_list(Prop), Props)]. 339 | 340 | prop_prefix(Atom) -> 341 | lists:prefix("prop_", atom_to_list(Atom)). 342 | 343 | proper_opts() -> 344 | [{dir, $d, "dir", string, 345 | "directory where the property tests are located (defaults to \"test\"). " 346 | "The directory also needs to be declared in extra_src_dirs."}, 347 | {module, $m, "module", string, 348 | "name of one or more modules to test (comma-separated)"}, 349 | {properties, $p, "prop", string, 350 | "name of properties to test within a specified module (comma-separated)"}, 351 | {numtests, $n, "numtests", integer, 352 | "number of tests to run when testing a given property"}, 353 | {search_steps, $s, "search_steps", integer, 354 | "number of searches to run when testing a given targeted property"}, 355 | {verbose, $v, "verbose", boolean, 356 | "each property tested shows its output or not (defaults to true)"}, 357 | {cover, $c, "cover", {boolean, false}, 358 | "generate cover data"}, 359 | {numworkers, $w, "workers", integer, 360 | "number of workers to use when parallelizing property tests"}, 361 | {type, $t, "type", atom, 362 | "this is only used when running parallel PropEr: indicates the " 363 | "type of the property to test, it can either be \"pure\" when it is side-effect " 364 | "and has no state, or \"impure\" when it does"}, 365 | %% no short format for these buddies 366 | {retry, undefined, "retry", {boolean, false}, 367 | "If failing test case counterexamples have been stored, " 368 | "they are retried"}, 369 | {regressions, undefined, "regressions", {boolean, false}, 370 | "replays the test cases stored in the regression file."}, 371 | {store, undefined, "store", {boolean, false}, 372 | "stores the last counterexample into the regression file."}, 373 | {long_result, undefined, "long_result", boolean, 374 | "enables long-result mode, displaying counter-examples on failure " 375 | "rather than just false"}, 376 | {start_size, undefined, "start_size", integer, 377 | "specifies the initial value of the size parameter"}, 378 | {max_size, undefined, "max_size", integer, 379 | "specifies the maximum value of the size parameter"}, 380 | {max_shrinks, undefined, "max_shrinks", integer, 381 | "specifies the maximum number of times a failing test case should be " 382 | "shrunk before returning"}, 383 | {noshrink, undefined, "noshrink", boolean, 384 | "instructs PropEr to not attempt to shrink any failing test cases"}, 385 | {constraint_tries, undefined, "constraint_tries", integer, 386 | "specifies the maximum number of tries before the generator subsystem " 387 | "gives up on producing an instance that satisfies a ?SUCHTHAT " 388 | "constraint"}, 389 | {spec_timeout, undefined, "spec_timeout", integer, 390 | "duration, in milliseconds, after which PropEr considers an input " 391 | "to be failing"}, 392 | {any_to_integer, undefined, "any_to_integer", boolean, 393 | "converts instances of the any() type to integers in order to speed " 394 | "up execution"}, 395 | {on_output, undefined, "on_output", string, 396 | "specifies a binary function '{Mod,Fun}', similar to io:format/2, " 397 | "to be used for all output printing"}, 398 | {sys_config, undefined, "sys_config", string, 399 | "config file to load before starting tests"}, 400 | {stop_nodes, undefined, "stop_nodes", boolean, 401 | "this is only used when running parallel PropEr: indicates whether " 402 | "PropEr should restart the nodes for each impure property, " 403 | "when testing them in parallel, or not"} 404 | ]. 405 | 406 | handle_opts(State) -> 407 | {CliOpts, _} = rebar_state:command_parsed_args(State), 408 | ConfigOpts = rebar_state:get(State, proper_opts, []), 409 | {fill_defaults(rebar3_opts(merge_opts(ConfigOpts, CliOpts))), 410 | proper_opts(merge_opts(ConfigOpts, proper_opts(CliOpts)))}. 411 | 412 | fill_defaults(Opts) -> 413 | [{dir, "test"} || proplists:get_value(dir, Opts) =:= undefined] ++ 414 | [{mods, any} || proplists:get_value(mods, Opts) =:= undefined] ++ 415 | [{properties, any} || proplists:get_value(properties, Opts) =:= undefined] 416 | ++ Opts. 417 | 418 | rebar3_opts([]) -> 419 | []; 420 | rebar3_opts([{dir, Dir} | T]) -> 421 | [{dir, Dir} | rebar3_opts(T)]; 422 | rebar3_opts([{module, Mods} | T]) -> 423 | [{module, maybe_parse_csv(Mods)} | rebar3_opts(T)]; 424 | rebar3_opts([{properties, Props} | T]) -> 425 | [{properties, maybe_parse_csv(Props)} | rebar3_opts(T)]; 426 | rebar3_opts([{retry, Retry} | T]) -> 427 | [{retry, Retry} | rebar3_opts(T)]; 428 | rebar3_opts([{regressions, Retry} | T]) -> 429 | [{regressions, Retry} | rebar3_opts(T)]; 430 | rebar3_opts([{sys_config, Config} | T]) -> 431 | [{sys_config, Config} | rebar3_opts(T)]; 432 | rebar3_opts([{store, Retry} | T]) -> 433 | [{store, Retry} | rebar3_opts(T)]; 434 | rebar3_opts([_ | T]) -> 435 | rebar3_opts(T). 436 | 437 | proper_opts([]) -> []; 438 | proper_opts([{verbose, true} | T]) -> [verbose | proper_opts(T)]; 439 | proper_opts([{verbose, false} | T]) -> [quiet | proper_opts(T)]; 440 | proper_opts([{long_result, true} | T]) -> [long_result | proper_opts(T)]; 441 | proper_opts([{long_result, false} | T]) -> proper_opts(T); 442 | proper_opts([{noshrink, true} | T]) -> [noshrink | proper_opts(T)]; 443 | proper_opts([{noshrink, false} | T]) -> proper_opts(T); 444 | proper_opts([{any_to_integer, true} | T]) -> [any_to_integer | proper_opts(T)]; 445 | proper_opts([{any_to_integer, false} | T]) -> proper_opts(T); 446 | proper_opts([{on_output, {Mod, Fun}} | T]) -> 447 | [{on_output, fun Mod:Fun/2} | proper_opts(T)]; 448 | proper_opts([{on_output, MFStr} | T]) when is_list(MFStr) -> 449 | case on_output(MFStr) of 450 | undefined -> proper_opts(T); 451 | Fun -> [{on_output, Fun} | proper_opts(T)] 452 | end; 453 | proper_opts([{type, Type} | T]) -> [Type | proper_opts(T)]; 454 | %% those are rebar3-only options 455 | proper_opts([{dir,_} | T]) -> proper_opts(T); 456 | proper_opts([{module,_} | T]) -> proper_opts(T); 457 | proper_opts([{properties,_} | T]) -> proper_opts(T); 458 | proper_opts([{cover,_} | T]) -> proper_opts(T); 459 | proper_opts([{retry,_} | T]) -> proper_opts(T); 460 | proper_opts([{regressions,_} | T]) -> proper_opts(T); 461 | proper_opts([{sys_config,_} | T]) -> proper_opts(T); 462 | proper_opts([{store,_} | T]) -> proper_opts(T); 463 | %% fall-through 464 | proper_opts([H|T]) -> [H | proper_opts(T)]. 465 | 466 | merge_opts(Old, New) -> 467 | rebar_utils:tup_umerge(New, Old). 468 | 469 | maybe_parse_csv(Data) -> 470 | case is_atom_list(Data) of 471 | true -> [atom_to_list(D) || D <- Data]; 472 | false -> parse_csv(Data) 473 | end. 474 | 475 | is_atom_list([]) -> true; 476 | is_atom_list([H|T]) when is_atom(H) -> is_atom_list(T); 477 | is_atom_list(_) -> false. 478 | 479 | parse_csv(IoData) -> 480 | re:split(IoData, ", *", [{return, list}]). 481 | 482 | -spec on_output(MFStr :: string()) -> 'undefined' | Fun when 483 | Fun :: fun((Format :: io:format(), Data :: [term()]) -> ok). 484 | on_output(MFStr) -> 485 | case erl_scan:string(MFStr ++ ".") of 486 | {ok, Tokens, _EndLocation} -> 487 | case erl_parse:parse_term(Tokens) of 488 | {ok, {Mod, Fun}} -> fun Mod:Fun/2; 489 | _ -> undefined 490 | end; 491 | _ -> undefined 492 | end. 493 | 494 | app_paths(State) -> 495 | Apps = rebar_state:project_apps(State), 496 | [rebar_app_info:ebin_dir(App) || App <- Apps, 497 | not rebar_app_info:is_checkout(App)]. 498 | 499 | sys_config_list(CmdOpts, CfgOpts) -> 500 | CmdSysConfigs = split_string(proplists:get_value(sys_config, CmdOpts, "")), 501 | case proplists:get_value(sys_config, CfgOpts, []) of 502 | [H | _]=Configs when is_list(H) -> 503 | Configs ++ CmdSysConfigs; 504 | [] -> 505 | CmdSysConfigs; 506 | Configs -> 507 | [Configs | CmdSysConfigs] 508 | end. 509 | 510 | split_string(String) -> 511 | string:tokens(String, [$,]). 512 | 513 | make_absolute_path(Path) -> 514 | case filename:pathtype(Path) of 515 | absolute -> 516 | Path; 517 | relative -> 518 | {ok, Dir} = file:get_cwd(), 519 | filename:join([Dir, Path]); 520 | volumerelative -> 521 | Volume = hd(filename:split(Path)), 522 | {ok, Dir} = file:get_cwd(Volume), 523 | filename:join([Dir, Path]) 524 | end. 525 | 526 | run_pre_hooks(State) -> 527 | Providers = rebar_state:providers(State), 528 | Cwd = rebar_dir:get_cwd(), 529 | rebar_hooks:run_project_and_app_hooks(Cwd, pre, ?PROVIDER, Providers, State). 530 | 531 | run_post_hooks(_, {ok, State}) -> run_post_hooks_(State); 532 | run_post_hooks(State, _) -> run_post_hooks_(State). 533 | 534 | run_post_hooks_(State) -> 535 | Providers = rebar_state:providers(State), 536 | Cwd = rebar_dir:get_cwd(), 537 | rebar_hooks:run_project_and_app_hooks(Cwd, post, ?PROVIDER, Providers, State), 538 | %% reset code paths for the plugin if we want to handle our own errors 539 | %% since the rebar3 hooks drop them by default 540 | PluginDepsPaths = lists:usort(rebar_state:code_paths(State, all_plugin_deps)), 541 | code:add_pathsa(PluginDepsPaths), 542 | ok. 543 | 544 | format_doc(Mod, Fun) -> 545 | try Mod:Fun(doc) of 546 | IoData -> [" (", IoData, $)] 547 | catch 548 | error:E when E == undef; E == function_clause -> 549 | rebar_api:debug("~p:~p(doc) not found; omitting docstring", 550 | [Mod,Fun]), 551 | [] 552 | end. 553 | --------------------------------------------------------------------------------