├── test ├── test_helper.exs └── true_story_test.exs ├── mix.lock ├── .gitignore ├── lib ├── true_story │ └── assertions.ex └── true_story.ex ├── config └── config.exs ├── mix.exs └── README.md /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"earmark": {:hex, :earmark, "1.0.1", "2c2cd903bfdc3de3f189bd9a8d4569a075b88a8981ded9a0d95672f6e2b63141", [], []}, 2 | "ex_doc": {:hex, :ex_doc, "0.13.0", "aa2f8fe4c6136a2f7cfc0a7e06805f82530e91df00e2bff4b4362002b43ada65", [], [{:earmark, "~> 1.0", [hex: :earmark, optional: false]}]}} 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc 12 | 13 | # If the VM crashes, it generates a dump, let's ignore it too. 14 | erl_crash.dump 15 | 16 | # Also ignore archive artifacts (built via "mix archive.build"). 17 | *.ez 18 | -------------------------------------------------------------------------------- /lib/true_story/assertions.ex: -------------------------------------------------------------------------------- 1 | defmodule TrueStory.Assertions do 2 | @doc false 3 | def __wrapper__(expr) do 4 | quote do 5 | try do 6 | unquote(expr) 7 | catch 8 | :error, %ExUnit.AssertionError{} = error -> 9 | stack = System.stacktrace 10 | if errors = Process.get(:true_story_errors) do 11 | Process.put(:true_story_errors, [{:error, error, stack}|errors]) 12 | else 13 | :erlang.raise(:error, error, stack) 14 | end 15 | end 16 | end 17 | end 18 | 19 | @doc false 20 | defmacro __wrapper_macro__(expr) do 21 | __wrapper__(expr) 22 | end 23 | 24 | defmacro assert(assertion) do 25 | __wrapper__(quote do: ExUnit.Assertions.assert(unquote(assertion))) 26 | end 27 | 28 | defmacro refute(assertion) do 29 | __wrapper__(quote do: ExUnit.Assertions.refute(unquote(assertion))) 30 | end 31 | 32 | def assert(value, message) do 33 | TrueStory.Assertions.__wrapper__(ExUnit.Assertions.assert(value, message)) 34 | end 35 | 36 | def refute(value, message) do 37 | TrueStory.Assertions.__wrapper__(ExUnit.Assertions.refute(value, message)) 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # 3rd-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure for your application as: 12 | # 13 | # config :true_story, key: :value 14 | # 15 | # And access this configuration in your application as: 16 | # 17 | # Application.get_env(:true_story, :key) 18 | # 19 | # Or configure a 3rd-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | # import_config "#{Mix.env}.exs" 31 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule TrueStory.Mixfile do 2 | use Mix.Project 3 | 4 | @version "0.1.0" 5 | 6 | def project do 7 | [app: :true_story, 8 | version: @version, 9 | elixir: "~> 1.3-dev", 10 | build_embedded: Mix.env == :prod, 11 | start_permanent: Mix.env == :prod, 12 | deps: deps(), 13 | description: description(), 14 | package: package(), 15 | name: "TrueStory", 16 | docs: [source_ref: "v#{@version}", 17 | source_url: "https://github.com/ericmj/true_story", 18 | main: "readme", extras: ["README.md"]]] 19 | end 20 | 21 | # Configuration for the OTP application 22 | # 23 | # Type "mix help compile.app" for more information 24 | def application do 25 | [applications: [:logger]] 26 | end 27 | 28 | defp description do 29 | "Make your tests tell a story" 30 | end 31 | 32 | defp package do 33 | [maintainers: ["Eric Meadows-Jönsson", "Bruce Tate"], 34 | licenses: ["Apache 2.0"], 35 | links: %{"GitHub" => "https://github.com/ericmj/true_story"}] 36 | end 37 | 38 | # Dependencies can be Hex packages: 39 | # 40 | # {:mydep, "~> 0.3.0"} 41 | # 42 | # Or git/path repositories: 43 | # 44 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1.0"} 45 | # 46 | # Type "mix help deps" for more examples and options 47 | defp deps do 48 | [{:ex_doc, ">= 0.0.0", only: :dev}] 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /test/true_story_test.exs: -------------------------------------------------------------------------------- 1 | defmodule TrueStoryTest do 2 | use ExUnit.Case 3 | use TrueStory 4 | 5 | defp add_to_map(c, key, value), 6 | do: Map.put(c, key, value) 7 | 8 | story "flat story" do 9 | assert :erlang.phash("", 1) == 1 10 | end 11 | 12 | story "adding to a map", c 13 | |> add_to_map(:key, :value), 14 | verify do 15 | assert c.key == :value 16 | refute c.key == :not_value 17 | end 18 | 19 | test "assign", c do 20 | c = assign c, key: :value 21 | assert c.key == :value 22 | 23 | _ = assign c, key: :value2 24 | refute c.key == :value2 25 | 26 | c = assign c, 27 | number1: 1, 28 | number2: c.number1+1 29 | 30 | assert c.number2 == 2 31 | end 32 | 33 | integration "adding and removing a key from a map" do 34 | story "adding to a map", c 35 | |> add_to_map(:key, :value), 36 | verify do 37 | # assert false 38 | assert c.key == :value 39 | refute c.key == :not_value 40 | end 41 | 42 | story "adding another key to the map", c 43 | |> add_to_map(:another_key, :value), 44 | verify do 45 | # assert false 46 | assert c.key == :value 47 | assert c.another_key == :value 48 | refute c.key == :not_value 49 | end 50 | end 51 | 52 | # FAILURES 53 | # story "single multi error", c 54 | # |> add_to_map(:key, :value), 55 | # verify do 56 | # refute c.key == :value 57 | # refute c.key == :not_value 58 | # end 59 | # 60 | # story "two multi errors", c 61 | # |> add_to_map(:key, :value), 62 | # verify do 63 | # refute c.key == :value 64 | # assert c.key == :not_value 65 | # end 66 | # 67 | # story "multi error with failure", c 68 | # |> add_to_map(:key, :value), 69 | # verify do 70 | # refute c.key == :value 71 | # assert c.key == :not_value 72 | # raise "exception" 73 | # end 74 | end 75 | -------------------------------------------------------------------------------- /lib/true_story.ex: -------------------------------------------------------------------------------- 1 | defmodule TrueStory do 2 | 3 | defmacro __using__(_opts) do 4 | quote do 5 | import unquote(__MODULE__), only: [story: 2, story: 4, assign: 2, integration: 2] 6 | import ExUnit.Assertions, except: [assert: 1, assert: 2, refute: 1, refute: 2] 7 | import TrueStory.Assertions 8 | @true_story_integration false 9 | end 10 | end 11 | 12 | defmacro integration(name, block) do 13 | context_var = quote do 14 | context 15 | end 16 | Module.put_attribute __CALLER__.module, :true_story_integration, true 17 | Module.put_attribute __CALLER__.module, :true_story_functions, [] 18 | Module.put_attribute __CALLER__.module, :integration_test_name, name 19 | 20 | # NOTE: This is a hack! 21 | Module.eval_quoted(__CALLER__.module, block, [], __CALLER__) 22 | 23 | Module.put_attribute __CALLER__.module, :true_story_integration, false 24 | 25 | quote do 26 | test unquote(name), unquote(context_var) do 27 | unquote(build_integration_test(context_var, __CALLER__.module)) 28 | end 29 | end 30 | end 31 | 32 | def build_integration_test(context, module) do 33 | functions = Module.get_attribute(module, :true_story_functions) |> Enum.reverse 34 | Enum.reduce(functions, context, fn(name, ast) -> 35 | quote do 36 | unquote(ast) |> unquote(name)() 37 | end 38 | end) 39 | end 40 | 41 | defmacro story(name, block) do 42 | quote do 43 | story(unquote(name), _c, verify, unquote(block)) 44 | end 45 | end 46 | 47 | defmacro story(name, setup, verify, block) do 48 | inside_integration_block = Module.get_attribute __CALLER__.module, :true_story_integration 49 | _story(inside_integration_block, name, setup, verify, block, __CALLER__.module) 50 | end 51 | 52 | defp _story(true, name, setup, verify, block, integration_test_module) do 53 | [{context_var, 0} | pipes] = Macro.unpipe(setup) 54 | setup = expand_setup(context_var, pipes) 55 | _verify = expand_verify(verify) 56 | block = expand_block(block) 57 | 58 | 59 | test_function_name = create_name name, integration_test_module 60 | existing_functions = Module.get_attribute(integration_test_module, :true_story_functions) 61 | Module.put_attribute integration_test_module, :true_story_functions, [test_function_name|existing_functions] 62 | 63 | quote do 64 | def unquote(test_function_name)( context ) do 65 | try do 66 | TrueStory.setup_pdict() 67 | unquote(setup) 68 | unquote(block) 69 | unquote(context_var) 70 | catch 71 | kind, error -> 72 | TrueStory.raise_multi([{kind, error, System.stacktrace}]) 73 | else 74 | context -> 75 | TrueStory.raise_multi([]) 76 | context 77 | end 78 | end 79 | end 80 | end 81 | 82 | defp _story(integrated, name, setup, verify, block, _) when integrated in [nil, false] do 83 | [{context_var, 0} | pipes] = Macro.unpipe(setup) 84 | setup = expand_setup(context_var, pipes) 85 | _verify = expand_verify(verify) 86 | block = expand_block(block) 87 | 88 | quote do 89 | test unquote(name), context do 90 | try do 91 | TrueStory.setup_pdict() 92 | unquote(setup) 93 | unquote(block) 94 | catch 95 | kind, error -> 96 | TrueStory.raise_multi([{kind, error, System.stacktrace}]) 97 | else 98 | value -> 99 | TrueStory.raise_multi([]) 100 | value 101 | end 102 | end 103 | end 104 | end 105 | 106 | # TODO capture pretty error. This fails if there are two integration tests of the same name 107 | def create_name(text, integration_test_module) do 108 | String.to_atom("#{Module.get_attribute(integration_test_module, :integration_test_name)} #{text}") 109 | end 110 | 111 | defp expand_setup(context_var, pipes) do 112 | acc = quote do: unquote(context_var) = context 113 | 114 | Enum.reduce(pipes, acc, fn {call, 0}, acc -> 115 | quote do 116 | unquote(acc) 117 | unquote(context_var) = unquote(context_var) |> unquote(call) 118 | end 119 | end) 120 | end 121 | 122 | defp expand_verify({:verify, _, context}) when is_atom(context), do: nil 123 | 124 | defp expand_block([do: block]), do: block 125 | 126 | @doc false 127 | def setup_pdict do 128 | Process.put(:true_story_errors, []) 129 | end 130 | 131 | @doc false 132 | def raise_multi(failure) do 133 | errors = Enum.reverse(failure ++ Process.get(:true_story_errors)) 134 | 135 | case errors do 136 | [] -> 137 | :ok 138 | [{kind, error, stack}] -> 139 | :erlang.raise(kind, error, stack) 140 | errors -> 141 | raise ExUnit.MultiError.exception(errors: errors) 142 | end 143 | end 144 | 145 | defmacro assign(context, assigns) do 146 | ast = quote do: context = unquote(context) 147 | Enum.reduce(assigns, ast, fn {key, expr}, ast -> 148 | expr = Macro.prewalk(expr, &translate_var(&1, context)) 149 | quote do 150 | unquote(ast) 151 | context = Map.put(context, unquote(key), unquote(expr)) 152 | end 153 | end) 154 | end 155 | 156 | defp translate_var({name, _, context}, {name, _, context}) do 157 | quote do: context 158 | end 159 | defp translate_var(expr, _context) do 160 | expr 161 | end 162 | end 163 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TrueStory 2 | 3 | _Make your tests tell a story._ 4 | 5 | ## Why? 6 | 7 | We've observed that well-written code and a well-structured API tells a good story. Writing single-purpose functions and improving setup composition improves tests. When you get the setup right, tests get simpler and the structure is easier to read and easier to follow. This thin DSL around ExUnit does exactly that. 8 | 9 | ## Quick Start 10 | 11 | To use TrueStory, just add as a dependency and write your tests. 12 | 13 | [Available in Hex](https://hex.pm/packages/true_story), the package can be installed as: 14 | 15 | ### Add Your Dependencies 16 | 17 | Add `true_story` to your list of dependencies in `mix.exs`: 18 | 19 | ```elixir 20 | def deps do 21 | [{:true_story, "~> 0.0.1", only: :test}] 22 | end 23 | ``` 24 | 25 | ### Write Tests 26 | 27 | First, you'll use `ExUnit.Case`, and also use `TrueStory`, like this: 28 | 29 | 30 | ```elixir 31 | defmodule MyTest do 32 | use ExUnit.Case 33 | use TrueStory 34 | 35 | # tests go here 36 | 37 | end 38 | ``` 39 | Next, you'll write your tests. Everything will compose, with each part of a story modifying a map, or context. To keep things brief, it's idiomatic to call the context `c`. 40 | 41 | ### Experiments (`story`) and measurements (`verify`) 42 | 43 | A TrueStory test has an experiment and measurements. The experiment changes the world, and the measurements evaluate the impact of the experiment. Experiments go in a `story` section and measurements go in a `verify` block. 44 | 45 | This story tests adding to a map. In the `story` block, you'll test 46 | 47 | ```elixir 48 | story "adding to a map", c 49 | |> Map.put(:key, :value), 50 | verify do 51 | assert c.key == :value 52 | refute c.key == :not_value 53 | end 54 | ``` 55 | 56 | Please note: *verify blocks can't be stateful, and they can't mutate the context!* Keeping verify blocks pure allows us to run all verifications for a single test at once. 57 | 58 | That's it. The `story` section has a name and a context pipe. The context pipe is a macro that allows basic piping, but also has some goodies for convenience. 59 | 60 | ### Building Your Story 61 | 62 | You can write composable functions that transform a test context to build up your experiments, piece by piece, like this: 63 | 64 | ```elixir 65 | defp add_to_map(c, key, value), 66 | do: Map.put(c, key, value) 67 | 68 | story "adding to a map", c 69 | |> add_to_map(:key, :value), 70 | verify do 71 | assert c.key == :value 72 | refute c.key == :not_value 73 | end 74 | 75 | story "overwriting a key", c 76 | |> add_to_map(:key, :old), 77 | |> add_to_map(:key, :new), 78 | verify do 79 | assert c.key == :new 80 | refute c.key == :old 81 | end 82 | ``` 83 | 84 | Most application tests are built in the setup. Piping together setup functions like this, you can build a growing library of setup functions for your application, and save your setup library in a common module. 85 | 86 | ### Linking Multiple Stories 87 | 88 | Maybe we would like to measure intermediate steps. To do so, you can run an integration test across tests, like this: 89 | 90 | ```elixir 91 | integrate "adding multiple keys" do 92 | story "adding to a map", c 93 | |> add_to_map(:key, :old), 94 | verify do 95 | assert c.key == :old 96 | end 97 | 98 | story "overwriting a key", c 99 | |> add_to_map(:key, :new), 100 | verify do 101 | assert c.key == :new 102 | end 103 | 104 | story "overwriting a key", c 105 | |> remove_from_map(:key), 106 | verify do 107 | refute c.key 108 | end 109 | end 110 | ``` 111 | This test expands to a single ExUnit test, so there's no concern about compatibility. 112 | 113 | Like the experiment steps, these stories compose, with the previous story piped into the next. 114 | 115 | ## Goodies for convenience 116 | 117 | ### The `story` pipe 118 | 119 | The pipe operator in the `story` macro allows you to access any key in the context placed there by an earlier pipe segment. For example, say you had some setup functions: 120 | 121 | ```elixir 122 | defp create_user(c), 123 | do: Map.put(c, :user, %User{ name: "Bubba" } 124 | 125 | defp create_blog(c, user), 126 | do: Map.put(c, :blog, %Blog{ name: "Fishin'", user: user } 127 | 128 | defp create_post(c, blog, options), do: Blog.create(blog, options) 129 | ``` 130 | 131 | In your story, you can access the context in earlier pipe segments, like this: 132 | 133 | ```elixir 134 | story "Creating a post", c 135 | |> create_user 136 | |> create_blog(c.user) 137 | |> create_post(c.blog, post: post_options), 138 | verify do 139 | ... 140 | end 141 | ``` 142 | 143 | Read the previous code carefully. Typically, the `user` would not be available from the `c` variable. By making it available with a 144 | macro, we make it easy to effortlessly build a simple composition of pipe segments, with the changes of each previous segment 145 | available to the next. Notice we're free to specify c.user and c.blog, which otherwise would be out of bounds. We can also take 146 | advantage of the same behavior in our setup functions with `assigns`, like this: 147 | 148 | ```elixir 149 | defp blog_with_post(user, title, post) do 150 | assign( 151 | user: user, 152 | blog:create_blog(c.user), 153 | post: create_post(c.blog, c.user) ) 154 | end 155 | ``` 156 | That macro makes composing this kind of data much cleaner. 157 | 158 | ### defplot (coming soon) 159 | 160 | Often, you want to add a single key to a test context. To make things easier for the person reading the test, you would like to make the name of the function in the story block and the key in the context the same. `defplot` makes this easy. You can build a single line of a story, called a `plot`, like this: 161 | 162 | ```elixir 163 | defplot user(name, email) do 164 | %User{ name: name || "Bubba", email: email || "gone@fishin.example.com" } 165 | end 166 | ``` 167 | 168 | Note that you can one-line simpler functions as well: 169 | 170 | ```elixir 171 | defplot user(name, email), do:%User{ name: name, email: email } 172 | ``` 173 | 174 | That expands to: 175 | 176 | ```elixir 177 | def user(c, name, email) do 178 | Map.put c, :user, %User{ name: name, email: email } 179 | end 180 | ``` 181 | 182 | Say you have a story block that looks like this: 183 | 184 | ```elixir 185 | story "Emailing a user", c 186 | |> user, 187 | |> application_function_emailing_a_user 188 | verify do 189 | assert c.user.email 190 | end 191 | ``` 192 | 193 | Now, it's clear that the `user` plot in the story populates the `:user` key in the context. Your stories are easier to read, and your plot lines are easier to write. Win/win. 194 | 195 | ## Expected Use 196 | 197 | In True Story, We change the way we think about tests a little bit. Follow these rules and you'll get better benefit out of the framework. 198 | 199 | ### One experiment, multiple measurements 200 | 201 | The `story` block contains an experiment. The `verify` block conains one or more measurements. You probably noticed that we're not afraid of multiple assertions in our `verify` block. We think that's ok, and it fits our metaphor. We're verifying a story, or measuring the result of an experiment. 202 | 203 | ### Separation of pure and impure. 204 | 205 | Anything that changes the context or the external world *always* goes into `story`. The `verify` is left to pure functions. That means we'll call our `story` blocks exactly once, and that's a huge win. The processing is simpler, and allows the best possible concurrency. 206 | 207 | ### Reusable Library 208 | 209 | Over the course of time, you'll accumulate reusable testing units in your story `library`. The way we're structured encourages this practice, and encourages users to build into composeable blocks. It's easy to roll up smaller library functions into bigger ones using nothing but piping and this is encouraged. 210 | 211 | ### Experiments raise errors, assertions return data. 212 | 213 | That means we don't have to stop for a failure. Since assertions/measurements are stateless, we don't have to worry about failures corrupting our tests, so these tests can continue to run. We get better cycle times because we can fix multiple tests for a single run while doing green field development or refactoring. 214 | 215 | ### Everything should compose. In True Story, an integration test is just one test that flows into the next. Story pipe segments are also just compositions on the context. 216 | 217 | ## Wins 218 | 219 | We didn't release True Story until we'd had six months of experience with it. We can confirm that these techniques work. Here's what we're finding. 220 | 221 | - *Tests are first class citizens.* The macros in this library are big wins for the organization of setup functions, and thus tests. 222 | - *One experiment, multiple measurements.* We find single purpose code gives us prettier tests, and more composable, reusable setups. 223 | - *Experiments can be stateful; measurements can't.* We can run each setup *once* so we get great performance. 224 | - *Experiments raise; measurements return fail data.* This means we can return multiple failures per test, shorting cycle times. 225 | - *Everything composes.* We find that most testing effort is in setup. If setup is simple, the rest of the testing is much easier. 226 | 227 | Enjoy. Let us know what you think. 228 | 229 | We're looking into better integration with Phoenix, and better integration with genstage. We're open to ideas and pull requests. 230 | --------------------------------------------------------------------------------