├── test ├── test_helper.exs ├── workflow │ ├── field_not_set_test.exs │ ├── docs_test.exs │ ├── visualization_test.exs │ ├── testing_test.exs │ └── workflow_instrumentation_test.exs ├── support │ └── pacer_doc_sample.ex ├── config │ └── config_test.exs └── workflow_test.exs ├── assets └── PACER.png ├── .formatter.exs ├── lib ├── workflow │ ├── error.ex │ ├── field_not_set.ex │ ├── visualization.ex │ ├── docs.ex │ ├── options.ex │ └── testing.ex ├── config │ └── config.ex └── workflow.ex ├── .gitignore ├── .github └── workflows │ └── elixir.yml ├── mix.exs ├── mix.lock ├── README.md └── LICENSE /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /assets/PACER.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/carsdotcom/pacer/HEAD/assets/PACER.png -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /lib/workflow/error.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.Error do 2 | @moduledoc false 3 | 4 | defexception [:message] 5 | end 6 | -------------------------------------------------------------------------------- /test/workflow/field_not_set_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.FieldNotSetTest do 2 | use ExUnit.Case, async: true 3 | 4 | describe "Inspect protocol implementation" do 5 | test "inspect" do 6 | assert "#Pacer.Workflow.FieldNotSet<>" == inspect(%Pacer.Workflow.FieldNotSet{}, []) 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/workflow/field_not_set.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.FieldNotSet do 2 | @moduledoc """ 3 | Struct set as the initial value for fields in 4 | a `Pacer.Workflow` definition when no explicit default 5 | value is provided. 6 | """ 7 | 8 | defstruct [] 9 | 10 | defimpl Inspect do 11 | def inspect(_not_set, _opts) do 12 | ~s(#Pacer.Workflow.FieldNotSet<>) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | pacer-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | 28 | /.tool-versions -------------------------------------------------------------------------------- /test/support/pacer_doc_sample.ex: -------------------------------------------------------------------------------- 1 | defmodule PacerDocSample do 2 | @moduledoc """ 3 | These are the existing moduledocs. 4 | """ 5 | use Pacer.Workflow 6 | 7 | graph do 8 | field(:a, doc: "this is a field that contains data about a thing") 9 | 10 | field(:undocumented_field, 11 | resolver: &__MODULE__.fetch/1, 12 | dependencies: [:a, :service_call], 13 | default: "a default" 14 | ) 15 | 16 | batch :api_requests do 17 | field(:service_call, 18 | default: nil, 19 | resolver: &__MODULE__.fetch/1, 20 | doc: "Fetches data from a service" 21 | ) 22 | 23 | field(:undocumented_service_call, default: nil, resolver: &__MODULE__.fetch/1) 24 | end 25 | end 26 | 27 | def fetch(_), do: :ok 28 | end 29 | -------------------------------------------------------------------------------- /lib/workflow/visualization.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.Visualization do 2 | @moduledoc """ 3 | A small wrapper around a Pacer Workflow's `visualization` metadata function. 4 | 5 | This stringed strict diagraph that is returned can be used in graphviz or some other graph visualization tool. 6 | """ 7 | 8 | @spec visualize(module(), Keyword.t()) :: :ok 9 | def visualize(workflow, options \\ []) do 10 | path = Keyword.get(options, :path, File.cwd!() <> "/") 11 | 12 | filename = 13 | Keyword.get( 14 | options, 15 | :filename, 16 | "#{inspect(workflow)}" <> 17 | "_strict_digraph_string_" <> "#{Calendar.strftime(NaiveDateTime.utc_now(), "%Y-%m-%d")}" 18 | ) 19 | 20 | {:ok, strict_digraph_string} = workflow.__graph__(:visualization) 21 | File.write!(path <> "#{filename}", strict_digraph_string) 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /.github/workflows/elixir.yml: -------------------------------------------------------------------------------- 1 | name: Elixir CI 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | 15 | name: Build and test 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Elixir 21 | uses: erlef/setup-beam@v1 22 | with: 23 | elixir-version: '1.15.6' # Define the elixir version [required] 24 | otp-version: '24.X' # Define the OTP version [required] 25 | - name: Restore dependencies cache 26 | uses: actions/cache@v3 27 | with: 28 | path: deps 29 | key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} 30 | restore-keys: ${{ runner.os }}-mix- 31 | - name: Install dependencies 32 | run: mix deps.get 33 | - name: Run tests 34 | run: mix test 35 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.MixProject do 2 | use Mix.Project 3 | 4 | @name "Pacer" 5 | @version "0.1.6" 6 | @source_url "https://github.com/carsdotcom/pacer" 7 | 8 | def project do 9 | [ 10 | app: :pacer, 11 | build_path: "../../_build", 12 | contributors: contributors(), 13 | deps: deps(), 14 | deps_path: "../../deps", 15 | description: description(), 16 | docs: docs(), 17 | elixir: "~> 1.14", 18 | elixirc_paths: elixirc_paths(Mix.env()), 19 | name: @name, 20 | source_url: @source_url, 21 | package: package(), 22 | version: @version 23 | ] 24 | end 25 | 26 | def application do 27 | [ 28 | extra_applications: [:logger] 29 | ] 30 | end 31 | 32 | defp elixirc_paths(:test), do: ["lib", "test/support"] 33 | defp elixirc_paths(_), do: ["lib"] 34 | 35 | def contributors() do 36 | [ 37 | {"Zack Kayser", "@zkayser"}, 38 | {"Stephanie Lane", "@stelane"} 39 | ] 40 | end 41 | 42 | defp description do 43 | "Dependency graphs for optimal function call ordering" 44 | end 45 | 46 | defp docs do 47 | [ 48 | main: "Pacer.Workflow", 49 | logo: __DIR__ <> "/assets/PACER.png", 50 | extras: ["README.md", "LICENSE"] 51 | ] 52 | end 53 | 54 | defp package do 55 | [ 56 | name: "pacer", 57 | licenses: ["Apache-2.0"], 58 | links: %{"GitHub" => @source_url} 59 | ] 60 | end 61 | 62 | defp deps do 63 | [ 64 | {:ex_doc, "~> 0.34", only: :dev, runtime: false}, 65 | {:libgraph, "~> 0.16.0"}, 66 | {:nimble_options, ">= 0.0.0"}, 67 | {:telemetry, "~> 1.2 or ~> 0.4"} 68 | ] 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/workflow/docs.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Docs do 2 | @moduledoc """ 3 | This module is responsible for extracting information provided to fields 4 | in a Pacer.Workflow `graph` definition and turning that information into 5 | docs that can be rendered into a module's documentation. 6 | """ 7 | alias Pacer.Workflow.FieldNotSet 8 | 9 | defmacro generate do 10 | quote do 11 | docs = 12 | Enum.reduce(@pacer_docs, "\n## Pacer Fields:\n\n", fn 13 | {field, doc}, markdown -> 14 | markdown <> 15 | "\n - `#{field}`\n#{Pacer.Docs.write_doc(doc)}#{Pacer.Docs.write_default(Keyword.get(@pacer_struct_fields, field))}#{Pacer.Docs.write_dependencies(Keyword.get(@pacer_dependencies, field))}" 16 | 17 | {field, batch, doc}, markdown -> 18 | field_dependencies = @pacer_batch_dependencies |> List.keyfind!(field, 1) |> elem(2) 19 | 20 | markdown <> 21 | "\n - `#{field}`\n#{Pacer.Docs.write_doc(doc)}\t - **Batch**: `#{batch}`\n#{Pacer.Docs.write_default(Keyword.get(@pacer_struct_fields, field))}#{Pacer.Docs.write_dependencies(field_dependencies)}" 22 | end) 23 | 24 | case Module.get_attribute(__MODULE__, :moduledoc) do 25 | {v, original_doc} -> 26 | Module.put_attribute( 27 | __MODULE__, 28 | :moduledoc, 29 | {v, original_doc <> docs} 30 | ) 31 | 32 | nil -> 33 | Module.put_attribute(__MODULE__, :moduledoc, {__ENV__.line, docs}) 34 | end 35 | end 36 | end 37 | 38 | def write_doc(""), do: "" 39 | def write_doc(doc), do: "\t - #{doc}\n" 40 | 41 | def write_default(%FieldNotSet{}), do: "" 42 | def write_default(default), do: "\t - **Default**: `#{inspect(default)}`\n" 43 | 44 | def write_dependencies([]), do: "\t - *No dependencies*\n" 45 | def write_dependencies(deps), do: "\t - **Depends on**: `#{inspect(deps)}`\n" 46 | end 47 | -------------------------------------------------------------------------------- /test/workflow/docs_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.DocsTest do 2 | use ExUnit.Case 3 | 4 | alias Pacer.Docs 5 | 6 | describe "generate/0 macro" do 7 | test "generates a string of markdown describing the fields" do 8 | {_, _, _, _, %{"en" => moduledocs}, _, _} = Code.fetch_docs(PacerDocSample) 9 | 10 | assert moduledocs == 11 | "These are the existing moduledocs.\n\n## Pacer Fields:\n\n\n - `undocumented_service_call`\n\t - **Batch**: `api_requests`\n\t - **Default**: `nil`\n\t - *No dependencies*\n\n - `service_call`\n\t - Fetches data from a service\n\t - **Batch**: `api_requests`\n\t - **Default**: `nil`\n\t - *No dependencies*\n\n - `undocumented_field`\n\t - **Default**: `\"a default\"`\n\t - **Depends on**: `[:a, :service_call]`\n\n - `a`\n\t - this is a field that contains data about a thing\n\t - *No dependencies*\n" 12 | end 13 | end 14 | 15 | describe "write_docs/1" do 16 | test "returns an empty string when given an empty string" do 17 | assert "" == Docs.write_doc("") 18 | end 19 | 20 | test "returns the docs passed with a tab and bullet prefix" do 21 | doc = "this is some documentation for a field" 22 | 23 | assert "\t - #{doc}\n" == Docs.write_doc(doc) 24 | end 25 | end 26 | 27 | describe "write_default/1" do 28 | test "returns an empty string if a field's default is not given" do 29 | assert "" == Docs.write_default(%Pacer.Workflow.FieldNotSet{}) 30 | end 31 | 32 | test "returns the default value with some additional markdown prefixing" do 33 | default = [this: "is just a default"] 34 | 35 | assert "\t - **Default**: `#{inspect(default)}`\n" == Docs.write_default(default) 36 | end 37 | end 38 | 39 | describe "write_dependencies/1" do 40 | test "returns markdown indicating there are no dependencies when given an empty list" do 41 | assert "\t - *No dependencies*\n" == Docs.write_dependencies([]) 42 | end 43 | 44 | test "returns the list of field dependencies in markdown" do 45 | deps = [:some_dep_one, :some_dep_two] 46 | 47 | assert "\t - **Depends on**: `#{inspect(deps)}`\n" == Docs.write_dependencies(deps) 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /test/workflow/visualization_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.VisualizationTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Pacer.Workflow.Visualization 5 | 6 | defmodule TestGraph do 7 | use Pacer.Workflow 8 | 9 | graph do 10 | field(:custom_field) 11 | field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 12 | field(:field_with_default, virtual?: true, default: "this is a default value") 13 | 14 | batch :http_requests do 15 | field(:request_1, 16 | resolver: &__MODULE__.do_work/1, 17 | dependencies: [:custom_field], 18 | default: 2 19 | ) 20 | 21 | field(:request_2, 22 | resolver: &__MODULE__.do_work/1, 23 | dependencies: [:custom_field, :field_a], 24 | default: "this is a default value for request2" 25 | ) 26 | end 27 | end 28 | 29 | def do_work(_), do: :ok 30 | end 31 | 32 | defmodule EmptyGraph do 33 | use Pacer.Workflow 34 | 35 | graph do 36 | end 37 | end 38 | 39 | describe "visualize/2" do 40 | test "with not options provided, sets default path and filename" do 41 | date = Calendar.strftime(NaiveDateTime.utc_now(), "%Y-%m-%d") 42 | 43 | filename_path = 44 | File.cwd!() <> 45 | "/" <> "Pacer.Workflow.VisualizationTest.TestGraph_strict_digraph_string_" <> "#{date}" 46 | 47 | assert :ok == Visualization.visualize(TestGraph) 48 | assert File.exists?(filename_path) 49 | 50 | File.rm!(filename_path) 51 | end 52 | 53 | test "when workflow is empty, still creates file" do 54 | date = Calendar.strftime(NaiveDateTime.utc_now(), "%Y-%m-%d") 55 | 56 | filename_path = 57 | File.cwd!() <> 58 | "/" <> "Pacer.Workflow.VisualizationTest.EmptyGraph_strict_digraph_string_" <> "#{date}" 59 | 60 | assert :ok == Visualization.visualize(EmptyGraph) 61 | assert File.exists?(filename_path) 62 | 63 | File.rm!(filename_path) 64 | end 65 | 66 | test "when options are provided, set path and filename to passed in options" do 67 | File.mkdir_p("tmp/") 68 | path = "tmp/" 69 | filename = "filename_here" 70 | 71 | filename_path = path <> filename 72 | assert :ok == Visualization.visualize(TestGraph, path: path, filename: filename) 73 | assert File.exists?(filename_path) 74 | 75 | File.rm_rf!("tmp") 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, 3 | "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, 4 | "libgraph": {:hex, :libgraph, "0.16.0", "3936f3eca6ef826e08880230f806bfea13193e49bf153f93edcf0239d4fd1d07", [:mix], [], "hexpm", "41ca92240e8a4138c30a7e06466acc709b0cbb795c643e9e17174a178982d6bf"}, 5 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 6 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 7 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, 8 | "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, 9 | "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, 10 | "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, 11 | } 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | Pacer 3 | 4 | 5 | # Pacer: An Elixir Library for Dependency Graph-Based Workflows With Robust Compile Time Safety & Guarantees 6 | 7 | ## Overview 8 | 9 | `Pacer.Workflow` is an abstraction designed for complex workflows where many interdependent data points need to be 10 | stitched together to provide a final result, specifically workflows where each data point needs to be 11 | loaded and/or calculated using discrete, application-specific logic. 12 | 13 | See the moduledocs for `Pacer.Workflow` for detailed information about how to use Pacer and define your own 14 | workflows, along with a detailed list of options and ideas that underlie Pacer. 15 | 16 | ## Installation 17 | 18 | Add :pacer to the list of dependencies in `mix.exs`: 19 | 20 | ```elixir 21 | [ 22 | {:pacer, "~> 0.1"} 23 | ] 24 | ``` 25 | 26 | ## Anti-patterns 27 | 28 | - Using Pacer when your dependency graph is a line, or if you just have a 29 | handful of data points. You would be better off using a pipeline of function 30 | calls. It would be easier to write, clearer to read, and faster to execute. 31 | - Untested Workflow. The main public interface of your workflow is 32 | calling `Pacer.Workflow.execute/1` on it. Having a test file for a 33 | Pacer workflow that does not do this, is like having any other module 34 | where the main public interface is not exercised in the tests. 35 | 36 | [Here is a repo with some practical examples of Pacer Anti-patterns](https://github.com/dewetblomerus/pacer_anti_patterns) 37 | 38 | ## Contributing 39 | 40 | We welcome everyone to contribute to Pacer -- whether it is documentation updates, proposing and/or implementing new features, or contributing bugfixes. 41 | 42 | Please feel free to create issues on the repo if you notice any bugs or if you would like to propose new features or implementations. 43 | 44 | When contributing to the codebase, please: 45 | 46 | 1. Run the test suite locally with `mix test` 47 | 2. Verify Dialyzer still passes with `mix dialyzer` 48 | 3. Run `mix credo --strict` 49 | 4. Make sure your code has been formatted with `mix format` 50 | 51 | In your PRs please provide the following detailed information as you see fit, especially for larger proposed changes: 52 | 53 | 1. What does your PR aim to do? 54 | 2. The reason/why for the changes 55 | 3. Validation and verification instructions (how can we verify that your changes are working as expected; if possible please provide one or two code samples that demonstrate the behavior) 56 | 4. Additional commentary if necessary -- tradeoffs, background context, etc. 57 | 58 | We will try to provide responses and feedback in a timely manner. Please feel free to ping us if you have a PR or issue that has not been responded to. 59 | -------------------------------------------------------------------------------- /lib/config/config.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Config do 2 | @moduledoc """ 3 | The #{inspect(__MODULE__)} module provides functions for 4 | extracting user-provided configuration values specific 5 | to Pacer. 6 | """ 7 | 8 | @doc """ 9 | Fetches configuration options for extending the metadata provided in the 10 | `[:pacer, :execute_vertex, :start | :stop | :exception]` events for batched resolvers. 11 | 12 | The configuration must be set under the key `:batch_telemetry_options` at the application 13 | level (i.e., `Application.get_env(:pacer, :batch_telemetry_options)`) or when defining the 14 | workflow itself (`use Pacer.Workflow, batch_telemetry_options: `). 15 | 16 | The batch_telemetry_options defined by the user must be either: 17 | - a keyword list, or 18 | - a {module, function, args} mfa tuple; when invoked, this function must return a keyword list 19 | 20 | The keyword list of values returned by the mfa-style config, or the hardcoded keyword list, is 21 | fetched and converted into a map that gets merged into the telemetry event metadata for batched resolvers. 22 | """ 23 | @spec batch_telemetry_options(module()) :: keyword() | {module(), atom(), list()} 24 | def batch_telemetry_options(workflow_module) do 25 | case :persistent_term.get({__MODULE__, workflow_module, :batch_telemetry_options}, :unset) do 26 | :unset -> fetch_and_write({workflow_module, :batch_telemetry_options}) 27 | config -> config 28 | end 29 | end 30 | 31 | @doc """ 32 | Takes the batch_telemetry_options configuration, invoking mfa-style config if available, 33 | and converts the batch_telemetry_options keyword list into a map that gets merged into 34 | the metadata for the `[:pacer, :execute_vertex, :start | :stop | :exception]` events for 35 | batched resolvers. 36 | """ 37 | @spec fetch_batch_telemetry_options(module()) :: map() 38 | def fetch_batch_telemetry_options(workflow_module) do 39 | case batch_telemetry_options(workflow_module) do 40 | {mod, fun, args} -> Map.new(apply(mod, fun, args)) 41 | opts -> Map.new(opts) 42 | end 43 | end 44 | 45 | @spec fetch_and_write({workflow_module, :batch_telemetry_options}) :: 46 | keyword() | {module(), atom(), list()} 47 | when workflow_module: module() 48 | defp fetch_and_write({workflow_module, :batch_telemetry_options = key}) do 49 | global_config = Application.get_env(:pacer, key) || [] 50 | module_config = workflow_module.__config__(key) || [] 51 | 52 | cond do 53 | is_list(global_config) && is_list(module_config) -> 54 | global_config 55 | |> Keyword.merge(module_config) 56 | |> tap(&:persistent_term.put({__MODULE__, workflow_module, key}, &1)) 57 | 58 | match?({_m, _f, _a}, module_config) || is_list(module_config) -> 59 | tap(module_config, &:persistent_term.put({__MODULE__, workflow_module, key}, &1)) 60 | 61 | match?({_m, _f, _a}, global_config) || is_list(global_config) -> 62 | tap(global_config, &:persistent_term.put({__MODULE__, workflow_module, key}, &1)) 63 | 64 | true -> 65 | tap([], &:persistent_term.put({__MODULE__, workflow_module, key}, &1)) 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /test/workflow/testing_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.TestingTest do 2 | use ExUnit.Case 3 | use Pacer.Workflow.Testing 4 | 5 | defmodule TestGraph do 6 | use Pacer.Workflow 7 | 8 | graph do 9 | field(:field_a_dependency) 10 | 11 | field(:field_a, 12 | resolver: &__MODULE__.do_work/1, 13 | dependencies: [:field_a_dependency] 14 | ) 15 | 16 | field(:field_with_default, 17 | virtual?: true, 18 | default: "this is a default value" 19 | ) 20 | 21 | batch :http_requests do 22 | field(:request_1, 23 | resolver: &__MODULE__.do_work/1, 24 | dependencies: [:field_a_dependency], 25 | default: 2 26 | ) 27 | 28 | field(:request_2, 29 | resolver: &__MODULE__.do_work/1, 30 | dependencies: [:field_a_dependency, :field_a], 31 | default: "this is a default value for request2" 32 | ) 33 | end 34 | end 35 | 36 | def do_work(_), do: :ok 37 | end 38 | 39 | defmodule EmptyGraph do 40 | use Pacer.Workflow 41 | 42 | graph do 43 | end 44 | end 45 | 46 | describe "assert_dependencies/3" do 47 | test "when provided dependencies for field matches graph definition dependencies for field" do 48 | assert_dependencies(TestGraph, :field_a, [ 49 | :field_a_dependency 50 | ]) 51 | end 52 | 53 | test "provided field has dependencies but do not match input dependencies" do 54 | assert_dependencies(TestGraph, :field_a, []) 55 | rescue 56 | error in [ExUnit.AssertionError] -> 57 | assert error.message == 58 | """ 59 | Expected :field_a to have dependencies: []. 60 | 61 | Instead found: [:field_a_dependency]. 62 | """ 63 | end 64 | 65 | test "field does not have dependencies" do 66 | assert_dependencies(TestGraph, :field_with_default, [ 67 | :definitely_a_dependency 68 | ]) 69 | rescue 70 | error in [ExUnit.AssertionError] -> 71 | assert error.message == 72 | """ 73 | Expected :field_with_default to have dependencies: [:definitely_a_dependency]. 74 | 75 | Instead found: []. 76 | """ 77 | end 78 | end 79 | 80 | describe "assert_batch_dependencies/3" do 81 | test "when provided dependencies for field matches graph definition dependencies for field" do 82 | assert_batch_dependencies(TestGraph, :request_2, [ 83 | :field_a_dependency, 84 | :field_a 85 | ]) 86 | end 87 | 88 | test "provided field has dependencies but do not match input dependencies" do 89 | assert_batch_dependencies(TestGraph, :request_2, []) 90 | rescue 91 | error in [ExUnit.AssertionError] -> 92 | assert error.message == 93 | """ 94 | Expected batched field: :request_2 to have dependencies: []. 95 | 96 | Instead found: [:field_a_dependency, :field_a]. 97 | """ 98 | end 99 | end 100 | 101 | describe "assert_fields/2" do 102 | test "provided fields match fields from workflow" do 103 | assert_fields(TestGraph, [ 104 | :field_a_dependency, 105 | :field_a, 106 | :field_with_default, 107 | :request_1, 108 | :request_2 109 | ]) 110 | end 111 | 112 | test "provided fields do not match graph fields" do 113 | assert_fields(EmptyGraph, [:field_1]) 114 | rescue 115 | error in [ExUnit.AssertionError] -> 116 | assert error.message == 117 | """ 118 | Expected Pacer.Workflow.TestingTest.EmptyGraph to have fields: [:field_1]. 119 | 120 | Instead found: []. 121 | """ 122 | end 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /test/config/config_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.ConfigTest do 2 | use ExUnit.Case, async: false 3 | 4 | alias Pacer.ConfigTest.NoOptions 5 | alias Pacer.Config 6 | 7 | setup do 8 | default = Application.get_env(:pacer, :batch_telemetry_options) 9 | 10 | on_exit(fn -> 11 | :persistent_term.erase({Config, NoOptions, :batch_telemetry_options}) 12 | :persistent_term.erase({Config, TestBatchConfig, :batch_telemetry_options}) 13 | Application.put_env(:pacer, :batch_telemetry_options, default) 14 | end) 15 | 16 | :ok 17 | end 18 | 19 | describe "batch_telemetry_options/1" do 20 | defmodule NoOptions do 21 | use Pacer.Workflow 22 | 23 | graph do 24 | field(:foo) 25 | end 26 | end 27 | 28 | test "returns an empty map if no user-provided options are available" do 29 | assert Config.batch_telemetry_options(NoOptions) == [] 30 | end 31 | 32 | test "returns global batch_telemetry_options if no module-level options are provided" do 33 | Application.put_env(:pacer, :batch_telemetry_options, foo: "bar") 34 | assert Config.batch_telemetry_options(NoOptions) == [foo: "bar"] 35 | end 36 | 37 | test "accepts {module, function, args} tuples for batch_telemetry_options from global config" do 38 | Application.put_env(:pacer, :batch_telemetry_options, {MyTelemetryOptions, :run, []}) 39 | assert Config.batch_telemetry_options(NoOptions) == {MyTelemetryOptions, :run, []} 40 | end 41 | 42 | defmodule TestBatchConfig do 43 | use Pacer.Workflow, batch_telemetry_options: [batched: "config"] 44 | 45 | graph do 46 | field(:foo) 47 | end 48 | end 49 | 50 | test "returns module-level options when provided" do 51 | assert Config.batch_telemetry_options(TestBatchConfig) == [batched: "config"] 52 | end 53 | 54 | test "module-level batch_telemetry_options overrides global batch_telemetry_options" do 55 | Application.put_env(:pacer, :batch_telemetry_options, 56 | batched: "this value should be overridden" 57 | ) 58 | 59 | assert Config.batch_telemetry_options(TestBatchConfig) == [batched: "config"] 60 | end 61 | 62 | defmodule TestConfigWithMFA do 63 | use Pacer.Workflow, batch_telemetry_options: {__MODULE__, :batch_telemetry_opts, []} 64 | 65 | graph do 66 | field(:foo) 67 | end 68 | end 69 | 70 | test "returns {module, function, args} options stored in module config" do 71 | assert Config.batch_telemetry_options(TestConfigWithMFA) == 72 | {TestConfigWithMFA, :batch_telemetry_opts, []} 73 | end 74 | 75 | test "module config overrides global config when both are present and use {module, function, args} style config" do 76 | Application.put_env(:pacer, :batch_telemetry_options, {PacerGlobal, :default_options, []}) 77 | 78 | assert Config.batch_telemetry_options(TestConfigWithMFA) == 79 | {TestConfigWithMFA, :batch_telemetry_opts, []} 80 | end 81 | end 82 | 83 | describe "fetch_batch_telemetry_options/1" do 84 | defmodule MyWorkflowExample do 85 | use Pacer.Workflow 86 | 87 | graph do 88 | field(:foo) 89 | end 90 | 91 | def default_options do 92 | [ 93 | foo: "bar", 94 | baz: "quux" 95 | ] 96 | end 97 | end 98 | 99 | test "invokes {module, fun, args} style config when present and converts the keyword list returned into a map" do 100 | Application.put_env( 101 | :pacer, 102 | :batch_telemetry_options, 103 | {MyWorkflowExample, :default_options, []} 104 | ) 105 | 106 | assert Config.fetch_batch_telemetry_options(MyWorkflowExample) == %{foo: "bar", baz: "quux"} 107 | end 108 | 109 | test "converts keyword list style configuration into a map" do 110 | Application.put_env(:pacer, :batch_telemetry_options, foo: "bar", baz: "quux") 111 | 112 | assert Config.fetch_batch_telemetry_options(MyWorkflowExample) == %{foo: "bar", baz: "quux"} 113 | end 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /test/workflow/workflow_instrumentation_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.WorkflowInstrumentationTest do 2 | use ExUnit.Case 3 | 4 | alias Pacer.Workflow 5 | 6 | defmodule InstrumentationTestGraph do 7 | use Pacer.Workflow 8 | 9 | graph do 10 | field(:custom_field) 11 | field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 12 | field(:field_with_default, default: "this is a default value") 13 | 14 | batch :http_requests do 15 | field(:request_1, 16 | resolver: &__MODULE__.do_work/1, 17 | dependencies: [:custom_field], 18 | default: 2 19 | ) 20 | 21 | field(:request_2, 22 | resolver: &__MODULE__.do_work/1, 23 | dependencies: [:custom_field, :field_a], 24 | default: "this is a default value for request2" 25 | ) 26 | end 27 | end 28 | 29 | def do_work(_), do: :ok 30 | end 31 | 32 | defmodule TelemetryHandlers do 33 | def handle_event(event, measurements, metadata, config) do 34 | receiver_pid = 35 | case config do 36 | %{receiver_pid: receiver_pid} -> receiver_pid 37 | _ -> self() 38 | end 39 | 40 | send(receiver_pid, {:event_handled, event, measurements, metadata, config}) 41 | end 42 | end 43 | 44 | describe "Pacer Workflow Telemetry" do 45 | setup context do 46 | on_exit(fn -> :telemetry.detach(context[:test]) end) 47 | :ok 48 | end 49 | 50 | test "emits [:pacer, :vertex_execute, :start/:stop] events for field resolvers executed sequentially", 51 | context do 52 | :telemetry.attach_many( 53 | context[:test], 54 | [[:pacer, :execute_vertex, :start], [:pacer, :execute_vertex, :stop]], 55 | &TelemetryHandlers.handle_event/4, 56 | nil 57 | ) 58 | 59 | assert %InstrumentationTestGraph{field_a: :ok} = 60 | Workflow.execute(%InstrumentationTestGraph{}) 61 | 62 | assert_receive {:event_handled, [:pacer, :execute_vertex, :start], _measurements, 63 | start_metadata, _config} 64 | 65 | assert_receive {:event_handled, [:pacer, :execute_vertex, :stop], _measurements, 66 | stop_metadata, _config} 67 | 68 | assert start_metadata[:field] == :field_a 69 | assert stop_metadata[:field] == :field_a 70 | assert start_metadata[:workflow] == InstrumentationTestGraph 71 | assert stop_metadata[:workflow] == InstrumentationTestGraph 72 | end 73 | 74 | test "emits [:pacer, :vertex_execute, :start/:stop] events for field resolvers executed concurrently", 75 | context do 76 | test_pid = self() 77 | config = %{receiver_pid: test_pid} 78 | 79 | :telemetry.attach_many( 80 | context[:test], 81 | [[:pacer, :execute_vertex, :start], [:pacer, :execute_vertex, :stop]], 82 | &TelemetryHandlers.handle_event/4, 83 | config 84 | ) 85 | 86 | assert %InstrumentationTestGraph{request_1: _, request_2: _} = 87 | Workflow.execute(%InstrumentationTestGraph{}) 88 | 89 | assert_receive {:event_handled, [:pacer, :execute_vertex, :start], _measurements, 90 | %{field: :request_1, parent_pid: ^test_pid}, _} 91 | 92 | assert_receive {:event_handled, [:pacer, :execute_vertex, :stop], _measurements, 93 | %{parent_pid: ^test_pid}, _} 94 | 95 | assert_receive {:event_handled, [:pacer, :execute_vertex, :start], _measurements, 96 | %{field: :request_2, parent_pid: ^test_pid}, _} 97 | 98 | assert_receive {:event_handled, [:pacer, :execute_vertex, :stop], _measurements, 99 | %{parent_pid: ^test_pid}, _} 100 | end 101 | 102 | defmodule InstrumentedGraphWithError do 103 | use Pacer.Workflow 104 | 105 | graph do 106 | field(:initial_field, default: "a") 107 | field(:a, resolver: &__MODULE__.resolver/1, dependencies: [:initial_field]) 108 | end 109 | 110 | def resolver(_) do 111 | raise "oops" 112 | end 113 | end 114 | 115 | test "emits [:pacer, :execute_vertex, :exception] events when an exception is raised on a sequential field resolver", 116 | context do 117 | :telemetry.attach( 118 | context[:test], 119 | [:pacer, :execute_vertex, :exception], 120 | &TelemetryHandlers.handle_event/4, 121 | nil 122 | ) 123 | 124 | assert_raise RuntimeError, fn -> 125 | Workflow.execute(InstrumentedGraphWithError) 126 | end 127 | 128 | assert_receive {:event_handled, [:pacer, :execute_vertex, :exception], _measurements, 129 | metadata, _config} 130 | 131 | assert %{ 132 | field: _, 133 | kind: :error, 134 | reason: %RuntimeError{}, 135 | stacktrace: stacktrace, 136 | workflow: InstrumentedGraphWithError 137 | } = metadata 138 | 139 | assert is_list(stacktrace) 140 | end 141 | end 142 | 143 | defmodule GraphWithBatchErrors do 144 | use Pacer.Workflow 145 | 146 | graph do 147 | field(:a, default: "this is a default value") 148 | 149 | batch :requests do 150 | field(:one, 151 | resolver: &__MODULE__.bad_resolver/1, 152 | default: "this is a fallback", 153 | dependencies: [:a] 154 | ) 155 | 156 | field(:two, 157 | resolver: &__MODULE__.bad_resolver/1, 158 | default: "another fallback", 159 | dependencies: [:a] 160 | ) 161 | end 162 | end 163 | 164 | def bad_resolver(_) do 165 | raise "oh no, this is not good" 166 | end 167 | end 168 | 169 | test "emits a [:pacer, :execute_vertex, :exception] event when an exception is raised from a batched field resolver", 170 | context do 171 | test_pid = self() 172 | config = %{receiver_pid: test_pid} 173 | 174 | :telemetry.attach( 175 | context[:test], 176 | [:pacer, :execute_vertex, :exception], 177 | &TelemetryHandlers.handle_event/4, 178 | config 179 | ) 180 | 181 | _ = Workflow.execute(%GraphWithBatchErrors{}) 182 | 183 | assert_receive {:event_handled, [:pacer, :execute_vertex, :exception], _measurements, 184 | metadata, _config} 185 | 186 | assert %{ 187 | field: :one, 188 | kind: :error, 189 | parent_pid: ^test_pid, 190 | reason: %RuntimeError{}, 191 | stacktrace: _, 192 | workflow: GraphWithBatchErrors 193 | } = metadata 194 | end 195 | end 196 | -------------------------------------------------------------------------------- /lib/workflow/options.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.Options do 2 | @moduledoc false 3 | 4 | fields_schema = [ 5 | dependencies: [ 6 | default: [], 7 | type: {:list, :atom}, 8 | doc: """ 9 | A list of dependencies from the graph. 10 | Dependencies are specified as atoms, and each dependency must 11 | be another field in the same graph. 12 | 13 | Remember that cyclical dependencies are strictly not allowed, 14 | so fields cannot declare dependencies on themselves nor on 15 | any other field that has already declared a dependency on the 16 | current field. 17 | 18 | If the `dependencies` option is not given, it defaults to an 19 | empty list, indicating that the field has no dependencies. 20 | This will be the case if the field is a constant or can be 21 | constructed from values already available in the environment. 22 | """ 23 | ], 24 | doc: [ 25 | required: false, 26 | type: :string, 27 | doc: """ 28 | Allows users to document the field and provide background and/or context 29 | on what the field is intended to be used for, what kind of data the field 30 | contains, and how the data for the field is constructed. 31 | """ 32 | ], 33 | resolver: [ 34 | type: {:fun, 1}, 35 | doc: """ 36 | A resolver is a 1-arity function that specifies how to calculate 37 | the value for a field. 38 | 39 | The argument passed to the function will be a map that contains 40 | all of the field's declared dependencies. 41 | 42 | For example, if we have a field like this: 43 | 44 | ```elixir 45 | field(:request, resolver: &RequestHandler.resolve/1, dependencies: [:api_key, :url]) 46 | ``` 47 | 48 | The resolver `RequestHandler.resolve/1` would be passed a map that looks like this: 49 | ```elixir 50 | %{api_key: "", url: "https://some.endpoint.com"} 51 | ``` 52 | 53 | If the field has no dependencies, the resolver will receive an empty map. Note though 54 | that resolvers are only required for fields with no dependencies if the field is inside 55 | of a batch. If your field has no dependencies and is not inside a batch, you can skip 56 | defining a resolver and initialize your graph struct with a value that is either constant 57 | or readily available where you are constructing the struct. 58 | 59 | The result of the resolver will be placed on the graph struct under the field's key. 60 | 61 | For the above, assuming a graph that looks like this: 62 | 63 | ```elixir 64 | defmodule MyGraph do 65 | use Pacer.Workflow 66 | 67 | graph do 68 | field(:api_key) 69 | field(:url) 70 | field(:request, resolver: &RequestHandler.resolve/1, dependencies: [:api_key, :url]) 71 | end 72 | end 73 | ``` 74 | 75 | Then when the `RequestHandler.resolve/1` runs an returns a value of, let's say, `%{response: "important response"}`, 76 | your graph struct would look like this: 77 | 78 | ```elixir 79 | %MyGraph{ 80 | api_key: "", 81 | url: "https://some.endpoint.com", 82 | request: %{response: "important response"} 83 | } 84 | ``` 85 | """ 86 | ], 87 | default: [ 88 | type: :any, 89 | doc: """ 90 | The default value for the field. If no default is given, the default value becomes 91 | `#Pacer.Workflow.FieldNotSet<>`. 92 | """ 93 | ], 94 | virtual?: [ 95 | default: false, 96 | type: :boolean, 97 | doc: """ 98 | A virtual field is used for intermediate or transient computation steps during the workflow and becomes a 99 | node in the workflow's graph, but does not get returned in the results of the workflow execution. 100 | 101 | In other words, virtual keys will not be included in the map returned by calling `Pacer.Workflow.execute/1`. 102 | 103 | The intent of a virtual field is to allow a spot for intermediate and/or transient calculation steps but 104 | to avoid the extra memory overhead that would be associated with carrying these values downstream if, for example, 105 | the map returned from `Pacer.Workflow.execute/1` is stored in a long-lived process state; intermediate or transient 106 | values can cause unnecessary memory bloat if they are carried into process state where they are not neeeded. 107 | """ 108 | ] 109 | ] 110 | 111 | @default_batch_timeout :timer.seconds(1) 112 | @default_batch_options [timeout: @default_batch_timeout] 113 | 114 | batch_options_schema = [ 115 | on_timeout: [ 116 | default: :kill_task, 117 | type: :atom, 118 | required: true, 119 | doc: """ 120 | The task that is timed out is killed and returns {:exit, :timeout}. 121 | This :kill_task option only exits the task process that fails and not the process that spawned the task. 122 | """ 123 | ], 124 | timeout: [ 125 | default: @default_batch_timeout, 126 | type: :non_neg_integer, 127 | required: true, 128 | doc: """ 129 | The time in milliseconds that the batch is allowed to run for. 130 | Defaults to 1,000 (1 second). 131 | """ 132 | ] 133 | ] 134 | 135 | batch_fields_schema = 136 | fields_schema 137 | |> Keyword.update!(:resolver, &Keyword.put(&1, :required, true)) 138 | |> Keyword.update!(:default, &Keyword.put(&1, :required, true)) 139 | |> Keyword.put(:guard, 140 | type: {:fun, 1}, 141 | required: false, 142 | doc: """ 143 | A guard is a 1-arity function that takes in a map with the field's dependencies and returns either true or false. 144 | If the function returns `false`, it means there is no work to do and thus no reason to spin up another process 145 | to run the resolver function. In this case, the field's default value is returned. 146 | If the function returns `true`, the field's resolver will run in a separate process. 147 | """ 148 | ) 149 | 150 | graph_schema = [ 151 | generate_docs?: [ 152 | required: false, 153 | type: :boolean, 154 | default: true, 155 | doc: """ 156 | By invoking `use Pacer.Workflow`, Pacer will automatically generate module documentation for you. It will create a section 157 | titled `Pacer Fields` in your moduledoc, either by creating a moduledoc for you dynamically or appending this section to 158 | any existing module documentation you have already provided. 159 | 160 | To opt out of this feature, when you `use Pacer.Workflow`, set this option to false. 161 | """ 162 | ] 163 | ] 164 | 165 | @fields_schema NimbleOptions.new!(fields_schema) 166 | @batch_fields_schema NimbleOptions.new!(batch_fields_schema) 167 | @batch_schema NimbleOptions.new!(batch_options_schema) 168 | @graph_schema NimbleOptions.new!(graph_schema) 169 | 170 | @spec fields_definition() :: NimbleOptions.t() 171 | def fields_definition, do: @fields_schema 172 | 173 | @spec batch_fields_definition() :: NimbleOptions.t() 174 | def batch_fields_definition, do: @batch_fields_schema 175 | 176 | @spec batch_definition() :: NimbleOptions.t() 177 | def batch_definition, do: @batch_schema 178 | 179 | @spec graph_options() :: NimbleOptions.t() 180 | def graph_options, do: @graph_schema 181 | 182 | def default_batch_options, do: @default_batch_options 183 | end 184 | -------------------------------------------------------------------------------- /lib/workflow/testing.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow.Testing do 2 | @moduledoc """ 3 | This module provides testing helpers for dependencies and fields for Pacer.Workflow. 4 | 5 | ## How to use in tests 6 | 7 | `use` this module in tests: 8 | 9 | use Pacer.Workflow.Testing 10 | 11 | This will allow you to use the helper assertions in your test module. 12 | 13 | Examples: 14 | ```elixir 15 | defmodule SomeWorkflowTest do 16 | use Pacer.Workflow.Testing 17 | 18 | defmodule TestGraph do 19 | use Pacer.Workflow 20 | 21 | graph do 22 | field(:field_a) 23 | 24 | field(:field_with_default, 25 | virtual?: true, 26 | default: "this is a default value" 27 | ) 28 | 29 | batch :http_requests do 30 | field(:request_1, 31 | resolver: &__MODULE__.do_work/1, 32 | dependencies: [:field_a], 33 | default: 2 34 | ) 35 | 36 | field(:request_2, 37 | resolver: &__MODULE__.do_work/1, 38 | dependencies: [:field_a], 39 | default: "this is a default value for request2" 40 | ) 41 | end 42 | end 43 | 44 | def do_work(_), do: :ok 45 | end 46 | 47 | describe "testing graph dependencies and fields" do 48 | test "assert dependency of :field_a has no dependencies" do 49 | assert_dependencies(TestGraph, :field_a, []) 50 | end 51 | 52 | test "assert batch dependency of :request_2" do 53 | assert_batch_dependencies(TestGraph, :request_2, [:field_a]) 54 | end 55 | 56 | test "provided fields match fields from workflow" do 57 | assert_fields(TestGraph, [ 58 | :field_a, 59 | :field_with_default, 60 | :request_1, 61 | :request_2 62 | ]) 63 | end 64 | end 65 | end 66 | ``` 67 | """ 68 | 69 | import ExUnit.Assertions, only: [assert: 2] 70 | 71 | defmacro __using__(_) do 72 | quote do 73 | alias Pacer.Workflow.Testing 74 | 75 | def assert_dependencies(workflow, field, dependencies) do 76 | Testing.assert_dependencies(workflow, field, dependencies) 77 | end 78 | 79 | def assert_batch_dependencies(workflow, batch_field, dependencies) do 80 | Testing.assert_batch_dependencies(workflow, batch_field, dependencies) 81 | end 82 | 83 | def assert_fields(workflow, fields) do 84 | Testing.assert_fields(workflow, fields) 85 | end 86 | end 87 | end 88 | 89 | @doc """ 90 | Test helper to assert on non-batch field dependencies. 91 | First argument is your workflow, second is the field as an atom that you want to test, 92 | third are the dependencies you are passing in to be dependencies for the field passed in as the second argument. 93 | Note, the third argument should be a list of atoms or an empty list in the case of no dependencies. 94 | 95 | ## Example 96 | 97 | If we have a workflow defined as: 98 | ```elixir 99 | defmodule TestGraph do 100 | use Pacer.Workflow 101 | 102 | graph do 103 | field(:field_a) 104 | 105 | field(:field_b, 106 | resolver: &__MODULE__.do_work/1, 107 | dependencies: [:field_a] 108 | ) 109 | end 110 | end 111 | ``` 112 | 113 | We can test the dependencies of both fields: 114 | 115 | ```elixir 116 | test "assert :field_a has no dependencies" do 117 | assert_dependencies(TestGraph, :field_a, []) 118 | end 119 | 120 | test "assert dependencies of :field_b" do 121 | assert_dependencies(TestGraph, :field_b, [:field_a]) 122 | end 123 | ``` 124 | """ 125 | @spec assert_dependencies(module(), atom(), list(atom())) :: true | no_return() 126 | def assert_dependencies(workflow, field, dependencies) do 127 | found_dependencies = find_dependencies(workflow, field) 128 | 129 | error_message = """ 130 | Expected #{inspect(field)} to have dependencies: #{inspect(dependencies)}. 131 | 132 | Instead found: #{inspect(found_dependencies)}. 133 | """ 134 | 135 | assert dependencies_for_field(dependencies, found_dependencies), 136 | error_message 137 | end 138 | 139 | @doc """ 140 | Test helper to assert on batch field dependencies. 141 | First argument is your workflow, second is the batch field as an atom that you want to test, 142 | third are the dependencies you are passing in to be dependencies for the field passed in as the second argument. 143 | Note, the third argument should be a list of atoms or an empty list in the case of no dependencies. 144 | 145 | ## Example 146 | 147 | If we have a workflow defined as: 148 | ```elixir 149 | defmodule TestGraph do 150 | use Pacer.Workflow 151 | 152 | graph do 153 | field(:field_a) 154 | 155 | batch :http_requests do 156 | field(:request_1, 157 | resolver: &__MODULE__.do_work/1, 158 | dependencies: [:field_a], 159 | default: 2 160 | ) 161 | 162 | field(:request_2, 163 | resolver: &__MODULE__.do_work/1, 164 | dependencies: [:field_a], 165 | default: "this is a default value for request2" 166 | ) 167 | end 168 | end 169 | 170 | def do_work(_), do: :ok 171 | end 172 | ``` 173 | 174 | We can test the dependencies of both fields: 175 | 176 | ```elixir 177 | test "assert batch fields have dependencies on :field_a" do 178 | assert_batch_dependencies(TestGraph, :request_1, [:field_a]) 179 | assert_batch_dependencies(TestGraph, :request_2, [:field_a]) 180 | end 181 | ``` 182 | """ 183 | @spec assert_batch_dependencies(module(), atom(), list(atom())) :: true | no_return() 184 | def assert_batch_dependencies(workflow, batch_field, dependencies) do 185 | found_dependencies = find_batch_dependencies(workflow, batch_field) 186 | 187 | error_message = """ 188 | Expected batched field: #{inspect(batch_field)} to have dependencies: #{inspect(dependencies)}. 189 | 190 | Instead found: #{inspect(found_dependencies)}. 191 | """ 192 | 193 | assert dependencies_for_field(dependencies, found_dependencies), 194 | error_message 195 | end 196 | 197 | @doc """ 198 | Test helper to assert on the field defined in the workflow. 199 | First argument is your workflow, second is a list of atoms that are fields in the workflow. 200 | Ordering does not matter in the list of atoms since we use MapSets to do the comparison. 201 | 202 | ## Example 203 | 204 | If we have a workflow defined as: 205 | ```elixir 206 | defmodule TestGraph do 207 | use Pacer.Workflow 208 | 209 | graph do 210 | field(:field_a) 211 | 212 | field(:field_b, 213 | resolver: &__MODULE__.do_work/1, 214 | dependencies: [:field_a] 215 | ) 216 | 217 | field(:field_with_default, 218 | virtual?: true, 219 | default: "this is a default value" 220 | ) 221 | 222 | batch :http_requests do 223 | field(:request_1, 224 | resolver: &__MODULE__.do_work/1, 225 | dependencies: [:field_a], 226 | default: 2 227 | ) 228 | 229 | field(:request_2, 230 | resolver: &__MODULE__.do_work/1, 231 | dependencies: [:field_a, :field_b], 232 | default: "this is a default value for request2" 233 | ) 234 | end 235 | end 236 | 237 | def do_work(_), do: :ok 238 | end 239 | ``` 240 | 241 | We can test the fields: 242 | ```elixir 243 | test "fields in the workflow" do 244 | assert_fields(TestGraph, [:field_a, :field_b, :request_2, :request_1]) 245 | end 246 | ``` 247 | """ 248 | @spec assert_fields(module(), list(atom())) :: true | no_return() 249 | def assert_fields(workflow, fields) do 250 | workflow_fields = workflow.__graph__(:fields) 251 | 252 | error_message = """ 253 | Expected #{inspect(workflow)} to have fields: #{inspect(fields)}. 254 | 255 | Instead found: #{inspect(workflow_fields)}. 256 | """ 257 | 258 | assert MapSet.equal?(MapSet.new(fields), MapSet.new(workflow_fields)), error_message 259 | end 260 | 261 | @spec find_dependencies(module(), atom()) :: list(atom()) | nil 262 | defp find_dependencies(workflow, field) do 263 | dependencies = workflow.__graph__(:dependencies) 264 | 265 | dependencies 266 | |> Enum.find(fn {found_field, _dependencies} -> 267 | found_field == field 268 | end) 269 | |> case do 270 | {_field, found_dependencies} -> found_dependencies 271 | nil -> nil 272 | end 273 | end 274 | 275 | @spec find_batch_dependencies(module(), atom()) :: list(atom()) | nil 276 | defp find_batch_dependencies(workflow, field) do 277 | dependencies = workflow.__graph__(:batch_dependencies) 278 | 279 | dependencies 280 | |> Enum.find(fn {_batch_name, found_field, _dependencies} -> 281 | found_field == field 282 | end) 283 | |> case do 284 | {_batch_name, _field, found_dependencies} -> found_dependencies 285 | nil -> nil 286 | end 287 | end 288 | 289 | @spec dependencies_for_field(list(atom()), list(atom()) | nil) :: boolean() 290 | defp dependencies_for_field(_dependencies, nil), do: false 291 | 292 | defp dependencies_for_field(dependencies, found_dependencies) do 293 | MapSet.equal?(MapSet.new(dependencies), MapSet.new(found_dependencies)) 294 | end 295 | end 296 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | (C) Copyright 2023 Cars.com Inc. all rights reserved. 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | -------------------------------------------------------------------------------- /test/workflow_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Pacer.WorkflowTest do 2 | use ExUnit.Case, async: true 3 | 4 | import ExUnit.CaptureLog 5 | 6 | alias Pacer.Workflow.Error 7 | alias Pacer.Workflow.FieldNotSet 8 | 9 | defmodule TestGraph do 10 | use Pacer.Workflow 11 | 12 | graph do 13 | field(:custom_field) 14 | field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 15 | field(:field_with_default, virtual?: true, default: "this is a default value") 16 | 17 | batch :http_requests do 18 | field(:request_1, 19 | resolver: &__MODULE__.do_work/1, 20 | dependencies: [:custom_field], 21 | default: 2 22 | ) 23 | 24 | field(:request_2, 25 | resolver: &__MODULE__.do_work/1, 26 | dependencies: [:custom_field, :field_a], 27 | default: "this is a default value for request2" 28 | ) 29 | 30 | field(:elixir_raise, 31 | resolver: &__MODULE__.elixir_raise/1, 32 | dependencies: [:custom_field], 33 | default: :elixir_raise 34 | ) 35 | 36 | field(:erlang_throw, 37 | resolver: &__MODULE__.erlang_throw/1, 38 | dependencies: [:custom_field], 39 | default: :erlang_throw 40 | ) 41 | 42 | field(:erlang_exit, 43 | resolver: &__MODULE__.erlang_exit/1, 44 | dependencies: [:custom_field], 45 | default: :erlang_exit 46 | ) 47 | 48 | field(:erlang_error, 49 | resolver: &__MODULE__.erlang_error/1, 50 | dependencies: [:custom_field], 51 | default: :erlang_error 52 | ) 53 | end 54 | end 55 | 56 | def do_work(_), do: :ok 57 | 58 | def elixir_raise(_), do: raise("boom baby!") 59 | def erlang_throw(_), do: :erlang.throw(:oops) 60 | def erlang_exit(_), do: :erlang.exit(:no_reason) 61 | def erlang_error(_), do: :erlang.error(:boom) 62 | end 63 | 64 | defmodule Resolvers do 65 | def resolve(_), do: :ok 66 | end 67 | 68 | @valid_graph_example """ 69 | Ex.: 70 | 71 | defmodule MyValidGraph do 72 | use Pacer.Workflow 73 | 74 | graph do 75 | field(:custom_field) 76 | field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 77 | field(:field_with_default, default: "this is a default value") 78 | 79 | batch :http_requests, timeout: :timer.seconds(1) do 80 | field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field], default: 5) 81 | 82 | field(:request_2, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field, :field_a], default: "this is the default for request2") 83 | field(:request_3, resolver: &__MODULE__.do_work/1, default: :this_default) 84 | end 85 | end 86 | 87 | def do_work(_), do: :ok 88 | end 89 | """ 90 | 91 | setup do 92 | on_exit(fn -> 93 | :persistent_term.erase({Pacer.Config, TestGraph, :batch_telemetry_options}) 94 | end) 95 | 96 | :ok 97 | end 98 | 99 | describe "telemetry" do 100 | test "execute/1 emits a [:pacer, :workflow, :start] and [:pacer, :workflow, :stop] event" do 101 | ref = 102 | :telemetry_test.attach_event_handlers(self(), [ 103 | [:pacer, :workflow, :start], 104 | [:pacer, :workflow, :stop] 105 | ]) 106 | 107 | Pacer.Workflow.execute(TestGraph) 108 | 109 | assert_received {[:pacer, :workflow, :start], ^ref, _, %{workflow: TestGraph}} 110 | assert_received {[:pacer, :workflow, :stop], ^ref, _, %{workflow: TestGraph}} 111 | end 112 | 113 | defmodule RaisingWorkflow do 114 | use Pacer.Workflow 115 | 116 | graph do 117 | field(:a, default: :ok) 118 | field(:exception_field, resolver: &__MODULE__.exception_raise/1, dependencies: [:a]) 119 | end 120 | 121 | def exception_raise(_args), do: raise("oops") 122 | end 123 | 124 | test "execute/1 emits a [:pacer, :workflow, :exception] event when the workflow execution raises" do 125 | ref = :telemetry_test.attach_event_handlers(self(), [[:pacer, :workflow, :exception]]) 126 | 127 | assert_raise(RuntimeError, fn -> 128 | Pacer.Workflow.execute(RaisingWorkflow) 129 | end) 130 | 131 | assert_received {[:pacer, :workflow, :exception], ^ref, _, %{workflow: RaisingWorkflow}} 132 | end 133 | 134 | test "batch resolvers inject user-provided telemetry config into metadata" do 135 | starting_config = Application.get_env(:pacer, :batch_telemetry_options) 136 | 137 | on_exit(fn -> 138 | Application.put_env(:pacer, :batch_telemetry_options, starting_config) 139 | end) 140 | 141 | ref = 142 | :telemetry_test.attach_event_handlers(self(), [ 143 | [:pacer, :execute_vertex, :start], 144 | [:pacer, :execute_vertex, :stop] 145 | ]) 146 | 147 | telemetry_options = [span_context: :rand.uniform()] 148 | Application.put_env(:pacer, :batch_telemetry_options, telemetry_options) 149 | 150 | Pacer.Workflow.execute(TestGraph) 151 | 152 | assert_receive {[:pacer, :execute_vertex, :start], ^ref, _, %{span_context: _}} 153 | assert_receive {[:pacer, :execute_vertex, :stop], ^ref, _, %{span_context: _}} 154 | end 155 | 156 | defmodule TestBatchConfigProvider do 157 | def telemetry_options do 158 | [span_context: :rand.uniform()] 159 | end 160 | end 161 | 162 | test "batch resolvers inject user-provided telemetry config into metadata when configured to use an MFA returning a keyword list" do 163 | starting_config = Application.get_env(:pacer, :batch_telemetry_options) 164 | 165 | on_exit(fn -> 166 | Application.put_env(:pacer, :batch_telemetry_options, starting_config) 167 | end) 168 | 169 | ref = 170 | :telemetry_test.attach_event_handlers(self(), [ 171 | [:pacer, :execute_vertex, :start], 172 | [:pacer, :execute_vertex, :stop] 173 | ]) 174 | 175 | Application.put_env( 176 | :pacer, 177 | :batch_telemetry_options, 178 | {TestBatchConfigProvider, :telemetry_options, []} 179 | ) 180 | 181 | Pacer.Workflow.execute(TestGraph) 182 | 183 | assert_receive {[:pacer, :execute_vertex, :start], ^ref, _, %{span_context: _}} 184 | assert_receive {[:pacer, :execute_vertex, :stop], ^ref, _, %{span_context: _}} 185 | end 186 | end 187 | 188 | test "graph metadata" do 189 | assert TestGraph.__graph__(:fields) == [ 190 | :custom_field, 191 | :field_a, 192 | :field_with_default, 193 | :request_1, 194 | :request_2, 195 | :elixir_raise, 196 | :erlang_throw, 197 | :erlang_exit, 198 | :erlang_error 199 | ] 200 | end 201 | 202 | test "dependency metadata" do 203 | assert TestGraph.__graph__(:dependencies, :field_a) == [:custom_field] 204 | assert TestGraph.__graph__(:dependencies, :http_requests) == [:custom_field, :field_a] 205 | end 206 | 207 | test "virtual field metadata" do 208 | assert TestGraph.__graph__(:virtual_fields) == [:field_with_default] 209 | end 210 | 211 | test "batch field dependency metadata" do 212 | assert TestGraph.__graph__(:batched_field_dependencies, :request_1) == [:custom_field] 213 | 214 | assert TestGraph.__graph__(:batched_field_dependencies, :request_2) == [ 215 | :custom_field, 216 | :field_a 217 | ] 218 | end 219 | 220 | test "batch metadata" do 221 | assert TestGraph.__graph__(:batch_fields, :http_requests) == [ 222 | :request_1, 223 | :request_2, 224 | :elixir_raise, 225 | :erlang_throw, 226 | :erlang_exit, 227 | :erlang_error 228 | ] 229 | 230 | assert TestGraph.__graph__(:http_requests, :options) == [ 231 | on_timeout: :kill_task, 232 | timeout: :timer.seconds(1) 233 | ] 234 | end 235 | 236 | test "batch options metadata with overrides" do 237 | defmodule BatchWithOptionOverrides do 238 | use Pacer.Workflow 239 | 240 | graph do 241 | field(:a) 242 | 243 | batch :requests, timeout: :timer.seconds(3) do 244 | field(:b, resolver: &__MODULE__.resolve/1, default: "default here") 245 | end 246 | end 247 | 248 | def resolve(_), do: :ok 249 | end 250 | 251 | assert BatchWithOptionOverrides.__graph__(:requests, :options) == [ 252 | on_timeout: :kill_task, 253 | timeout: :timer.seconds(3) 254 | ] 255 | end 256 | 257 | test "resolver metadata" do 258 | {:field, field_resolver} = TestGraph.__graph__(:resolver, :field_a) 259 | 260 | assert is_function(field_resolver, 1), 261 | "Expected resolver for field_a to be a 1-arity function" 262 | 263 | {:batch, batch_resolvers} = TestGraph.__graph__(:resolver, :http_requests) 264 | 265 | num_resolvers = Enum.count(batch_resolvers) 266 | 267 | assert num_resolvers == 6, 268 | "Expected http_requests batch node to have 3 resolvers. Got #{num_resolvers}:\n 269 | #{inspect(batch_resolvers)}" 270 | 271 | Enum.each(batch_resolvers, fn {field_name, resolver} -> 272 | assert field_name in [ 273 | :request_1, 274 | :request_2, 275 | :elixir_raise, 276 | :erlang_throw, 277 | :erlang_exit, 278 | :erlang_error 279 | ] 280 | 281 | assert is_function(resolver, 1) 282 | end) 283 | end 284 | 285 | test "evaluation_order returns all nodes in the graph with work to do in topologically sorted order" do 286 | [:field_a, :http_requests] = TestGraph.__graph__(:evaluation_order) 287 | end 288 | 289 | test "virtual fields are not returned in the results of Pacer.Workflow.execute/1" do 290 | assert %TestGraph{} = result = Pacer.Workflow.execute(TestGraph) 291 | 292 | refute Map.has_key?(result, :field_with_default) 293 | end 294 | 295 | test "resolver functions only receive their explicit dependencies and the current field when invoked" do 296 | defmodule ResolverInputSizeTest do 297 | use Pacer.Workflow 298 | 299 | graph do 300 | field(:a, default: 1) 301 | field(:b, resolver: &__MODULE__.resolve_b/1, dependencies: [:a]) 302 | field(:c, resolver: &__MODULE__.resolve_c/1, dependencies: [:a, :b]) 303 | end 304 | 305 | def resolve_b(args) do 306 | assert map_size(args) == 2 307 | assert Map.has_key?(args, :a) 308 | assert Map.has_key?(args, :b) 309 | end 310 | 311 | def resolve_c(args) do 312 | assert map_size(args) == 3 313 | assert Map.has_key?(args, :a) 314 | assert Map.has_key?(args, :b) 315 | assert Map.has_key?(args, :c) 316 | end 317 | end 318 | 319 | Pacer.Workflow.execute(ResolverInputSizeTest) 320 | end 321 | 322 | test "field defaults" do 323 | assert %TestGraph{ 324 | field_a: %FieldNotSet{}, 325 | custom_field: %FieldNotSet{}, 326 | request_1: 2, 327 | request_2: "this is a default value for request2", 328 | field_with_default: "this is a default value" 329 | } == %TestGraph{} 330 | end 331 | 332 | describe "cycle detection" do 333 | test "detects cycles in a graph definition" do 334 | module = """ 335 | defmodule GraphWithCycles do 336 | use Pacer.Workflow 337 | 338 | graph do 339 | field(:a, resolver: &__MODULE__.resolve/1, dependencies: [:b]) 340 | field(:b, resolver: &__MODULE__.resolve/1, dependencies: [:a]) 341 | field(:c) 342 | end 343 | 344 | def resolve(_), do: :ok 345 | end 346 | """ 347 | 348 | expected_error_message = """ 349 | Could not sort dependencies. 350 | The following dependencies form a cycle: 351 | 352 | a, b 353 | """ 354 | 355 | assert_raise Error, expected_error_message, fn -> 356 | Code.eval_string(module) 357 | end 358 | end 359 | 360 | test "detects reflexive cycles" do 361 | module = """ 362 | defmodule GraphWithReflexiveCycle do 363 | use Pacer.Workflow 364 | 365 | graph do 366 | field(:a, resolver: &__MODULE__.resolve/1, dependencies: [:a]) 367 | end 368 | 369 | def resolve(_), do: :ok 370 | end 371 | """ 372 | 373 | expected_error_message = """ 374 | Could not sort dependencies. 375 | The following dependencies form a cycle: 376 | 377 | Field `a` depends on itself. 378 | """ 379 | 380 | assert_raise Error, expected_error_message, fn -> 381 | Code.eval_string(module) 382 | end 383 | end 384 | end 385 | 386 | describe "workflow config" do 387 | defmodule WorkflowWithBatchOptions do 388 | use Pacer.Workflow, batch_telemetry_options: %{some_options: "foo"} 389 | 390 | graph do 391 | field(:bar) 392 | end 393 | end 394 | 395 | test "allows batch_telemetry_config option to be passed" do 396 | assert WorkflowWithBatchOptions.__config__(:batch_telemetry_options) == %{ 397 | some_options: "foo" 398 | } 399 | end 400 | 401 | defmodule WorkflowWithNoConfigOptions do 402 | use Pacer.Workflow 403 | 404 | graph do 405 | field(:foo) 406 | end 407 | end 408 | 409 | test "returns nil for missing or non-existent config values" do 410 | assert is_nil(WorkflowWithNoConfigOptions.__config__(:foo)) 411 | end 412 | end 413 | 414 | describe "graph validations" do 415 | test "options validations" do 416 | module = """ 417 | defmodule AllTheOptionsTooManyOptions do 418 | use Pacer.Workflow 419 | 420 | graph do 421 | field(:a, not_an_option: &Resolvers.resolve/1, invalid_option_again: :ohno) 422 | end 423 | end 424 | """ 425 | 426 | expected_error_message = """ 427 | unknown options [:not_an_option, :invalid_option_again], valid options are: [:dependencies, :doc, :resolver, :default, :virtual?] 428 | 429 | #{@valid_graph_example} 430 | """ 431 | 432 | assert_raise Error, expected_error_message, fn -> 433 | Code.eval_string(module) 434 | end 435 | end 436 | 437 | test "only one graph definition is allowed per module" do 438 | module = """ 439 | defmodule TwoGraphs do 440 | use Pacer.Workflow 441 | 442 | graph do 443 | field(:a) 444 | end 445 | 446 | graph do 447 | field(:b) 448 | end 449 | end 450 | """ 451 | 452 | expected_error_message = """ 453 | Module TwoGraphs already defines a graph on line 4 454 | """ 455 | 456 | assert_raise Error, expected_error_message, fn -> 457 | Code.eval_string(module) 458 | end 459 | end 460 | 461 | test "fields must be unique within a graph instance" do 462 | module = """ 463 | defmodule GraphWithDuplicateFields do 464 | use Pacer.Workflow 465 | 466 | graph do 467 | field(:a) 468 | field(:a) 469 | end 470 | end 471 | """ 472 | 473 | expected_error_message = 474 | "Found duplicate field in graph instance for GraphWithDuplicateFields: a" 475 | 476 | assert_raise Error, expected_error_message, fn -> 477 | Code.eval_string(module) 478 | end 479 | end 480 | 481 | test "dependencies must be a list if declared" do 482 | module = """ 483 | defmodule GraphWithBadDependencyField do 484 | use Pacer.Workflow 485 | 486 | graph do 487 | field(:a, resolver: &__MODULE__.resolve/1, dependencies: "strings are not valid values for dependencies") 488 | field(:b) 489 | end 490 | 491 | def resolve(_), do: :ok 492 | end 493 | """ 494 | 495 | expected_error_message = """ 496 | invalid value for :dependencies option: expected list, got: "strings are not valid values for dependencies" 497 | 498 | #{@valid_graph_example} 499 | """ 500 | 501 | assert_raise Error, expected_error_message, fn -> 502 | Code.eval_string(module) 503 | end 504 | end 505 | 506 | test "dependencies must be a list of atoms when declared and non-empty" do 507 | module = """ 508 | defmodule GraphWithStringsInDependencyList do 509 | use Pacer.Workflow 510 | 511 | graph do 512 | field(:a, resolver: &__MODULE__.resolve/1, dependencies: [:b, "strings", "are", "not", "valid", "here"]) 513 | field(:b, default: :ok) 514 | end 515 | 516 | def resolve(_), do: :ok 517 | end 518 | """ 519 | 520 | expected_error_message = """ 521 | invalid list in :dependencies option: invalid value for list element at position 1: expected atom, got: "strings" 522 | 523 | #{@valid_graph_example} 524 | """ 525 | 526 | assert_raise Error, expected_error_message, fn -> 527 | Code.eval_string(module) 528 | end 529 | end 530 | 531 | test "requires resolver functions to be a 1-arity function" do 532 | module = """ 533 | defmodule GraphWithBadValueForResolver do 534 | use Pacer.Workflow 535 | 536 | graph do 537 | field(:a, resolver: fn -> "0-arity functions are not valid resolvers" end, dependencies: [:b]) 538 | field(:b) 539 | end 540 | end 541 | """ 542 | 543 | expected_error_message = """ 544 | invalid value for :resolver option: expected function of arity 1, got: function of arity 0 545 | 546 | #{@valid_graph_example} 547 | """ 548 | 549 | assert_raise Error, expected_error_message, fn -> 550 | Code.eval_string(module) 551 | end 552 | end 553 | 554 | test "requires a resolver function for any field that declares dependencies" do 555 | module = """ 556 | defmodule NoResolverWithDepGraph do 557 | use Pacer.Workflow 558 | 559 | graph do 560 | field(:a, dependencies: [:b]) 561 | field(:b) 562 | end 563 | end 564 | """ 565 | 566 | expected_error_message = """ 567 | Field a in NoResolverWithDepGraph declared at least one dependency, but did not specify a resolver function. 568 | Any field that declares at least one dependency must also declare a resolver function. 569 | 570 | #{@valid_graph_example} 571 | """ 572 | 573 | assert_raise Error, expected_error_message, fn -> 574 | Code.eval_string(module) 575 | end 576 | end 577 | 578 | test "does not allow dependencies that are not also fields in the graph" do 579 | module = """ 580 | defmodule BadGraph do 581 | use Pacer.Workflow 582 | 583 | graph do 584 | field(:a, resolver: &__MODULE__.resolve/1, dependencies: [:not_a_field_in_this_graph]) 585 | end 586 | 587 | def resolve(_), do: :ok 588 | end 589 | """ 590 | 591 | expected_error_message = """ 592 | Found at least one invalid dependency in graph definiton for BadGraph 593 | Invalid dependencies: [:not_a_field_in_this_graph] 594 | """ 595 | 596 | assert_raise Error, expected_error_message, fn -> 597 | Code.eval_string(module) 598 | end 599 | end 600 | end 601 | 602 | describe "batch validations" do 603 | test "batch fields must define a 1-arity function as resolver" do 604 | module = """ 605 | defmodule BatchFieldWithInvalidResolverValue do 606 | use Pacer.Workflow 607 | 608 | graph do 609 | batch :requests do 610 | field(:a, resolver: "this is not valid", default: 5) 611 | end 612 | end 613 | end 614 | """ 615 | 616 | expected_error_message = """ 617 | invalid value for :resolver option: expected function of arity 1, got: "this is not valid" 618 | 619 | #{@valid_graph_example} 620 | """ 621 | 622 | assert_raise Error, expected_error_message, fn -> 623 | Code.eval_string(module) 624 | end 625 | end 626 | 627 | test "batch options validations" do 628 | module = """ 629 | defmodule BatchAllTheOptionsTooManyOptions do 630 | use Pacer.Workflow 631 | 632 | graph do 633 | field(:a, default: 1) 634 | batch :requests, invalid_option: :ohno, not_an_option: &Resolvers.resolve/1, timeout: 1000000, exit: :brutal_kill do 635 | field(:y, resolver: &Resolvers.resolve/1, dependencies: [:a]) 636 | field(:z, resolver: &Resolvers.resolve/1, dependencies: [:a]) 637 | end 638 | end 639 | end 640 | """ 641 | 642 | expected_error_message = """ 643 | unknown options [:invalid_option, :not_an_option, :exit], valid options are: [:on_timeout, :timeout] 644 | 645 | #{@valid_graph_example} 646 | """ 647 | 648 | assert_raise Error, expected_error_message, fn -> 649 | Code.eval_string(module) 650 | end 651 | end 652 | 653 | test "a field inside of a batch cannot have a dependency on a field in the same batch" do 654 | module = """ 655 | defmodule BatchFieldDepOnSameBatch do 656 | use Pacer.Workflow 657 | 658 | graph do 659 | batch :requests do 660 | field(:a, resolver: &__MODULE__.resolve/1, dependencies: [:b], default: 12) 661 | field(:b, resolver: &__MODULE__.resolve/1, dependencies: [:c], default: 3) 662 | end 663 | field(:c) 664 | end 665 | 666 | def resolve(_), do: :ok 667 | end 668 | """ 669 | 670 | expected_error_message = """ 671 | Found at least one invalid field dependency inside of a Pacer.Workflow batch. 672 | Invalid dependencies: [:b] 673 | Graph module: BatchFieldDepOnSameBatch 674 | 675 | Fields that are defined within a batch MUST not have dependencies on other 676 | fields in the same batch because their resolvers will run concurrently. 677 | 678 | You may need to rearrange an invalid field (or fields) out of your batch 679 | if the field does have a hard dependency on another field in the batch. 680 | """ 681 | 682 | assert_raise Error, expected_error_message, fn -> 683 | Code.eval_string(module) 684 | end 685 | end 686 | 687 | test "fields cannot be duplicated within a batch" do 688 | module = """ 689 | defmodule GraphWithDuplicateFieldsInBatch do 690 | use Pacer.Workflow 691 | 692 | graph do 693 | batch :requests do 694 | field(:a, resolver: &Resolvers.resolve/1, dependencies: [:b], default: :ok) 695 | field(:a, resolver: &Resolvers.resolve/1, dependencies: [:b], default: 123) 696 | end 697 | field(:b) 698 | end 699 | end 700 | """ 701 | 702 | expected_error_message = 703 | "Found duplicate field in graph instance for GraphWithDuplicateFieldsInBatch: a" 704 | 705 | assert_raise Error, expected_error_message, fn -> 706 | Code.eval_string(module) 707 | end 708 | end 709 | 710 | test "dependencies for fields inside of a batch must be other valid fields in the graph outside of the same batch" do 711 | module = """ 712 | defmodule InvalidDepInsideOfBatch do 713 | use Pacer.Workflow 714 | 715 | graph do 716 | batch :requests do 717 | field(:a, resolver: &Resolvers.resolve/1, dependencies: [:this_field_does_not_exist], default: 12) 718 | end 719 | end 720 | end 721 | """ 722 | 723 | expected_error_message = """ 724 | Found at least one invalid dependency in graph definiton for InvalidDepInsideOfBatch 725 | Invalid dependencies: [:this_field_does_not_exist] 726 | """ 727 | 728 | assert_raise Error, expected_error_message, fn -> 729 | Code.eval_string(module) 730 | end 731 | end 732 | 733 | test "fields inside of a batch MUST define a resolver, even if the field has no dependencies" do 734 | module = """ 735 | defmodule BatchWithResolverlessField do 736 | use Pacer.Workflow 737 | 738 | graph do 739 | batch :requests do 740 | field(:a) 741 | end 742 | end 743 | end 744 | """ 745 | 746 | expected_error_message = """ 747 | required :resolver option not found, received options: [:dependencies] 748 | 749 | #{@valid_graph_example} 750 | """ 751 | 752 | assert_raise Error, expected_error_message, fn -> 753 | Code.eval_string(module) 754 | end 755 | end 756 | 757 | test "duplicate batch names are not allowed" do 758 | module = """ 759 | defmodule DuplicateBatchNameGraph do 760 | use Pacer.Workflow 761 | 762 | graph do 763 | batch :dupe do 764 | field(:a, resolver: &Resolvers.resolve/1, default: :ok) 765 | end 766 | 767 | batch :dupe do 768 | field(:b, resolver: &Resolvers.resolve/1, default: :ok) 769 | end 770 | end 771 | end 772 | """ 773 | 774 | expected_error_message = """ 775 | Found duplicate batch name `dupe` in graph module DuplicateBatchNameGraph. 776 | Batch names within a single graph instance must be unique. 777 | """ 778 | 779 | assert_raise Error, expected_error_message, fn -> 780 | Code.eval_string(module) 781 | end 782 | end 783 | 784 | test "fields inside of a batch MUST define a default" do 785 | module = """ 786 | defmodule BatchFieldWithoutDefaultValue do 787 | use Pacer.Workflow 788 | 789 | graph do 790 | field(:a, default: 5) 791 | batch :requests do 792 | field(:c, resolver: &__MODULE__.resolve/1, dependencies: [:a]) 793 | end 794 | end 795 | 796 | def resolve(_), do: :ok 797 | end 798 | """ 799 | 800 | expected_error_message = """ 801 | required :default option not found, received options: [:resolver, :dependencies] 802 | 803 | #{@valid_graph_example} 804 | """ 805 | 806 | assert_raise Error, expected_error_message, fn -> 807 | Code.eval_string(module) 808 | end 809 | end 810 | end 811 | 812 | describe "execute/1, no batches" do 813 | defmodule TheTestWorkflow do 814 | use Pacer.Workflow 815 | 816 | graph do 817 | field(:a, default: 1) 818 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a]) 819 | end 820 | 821 | def calculate_b(%{a: a}), do: a + 1 822 | end 823 | 824 | test "runs each resolver and places the results of each resolver on the associated struct field" do 825 | assert %TheTestWorkflow{a: 1, b: 2} == Pacer.Workflow.execute(%TheTestWorkflow{}) 826 | end 827 | 828 | test "when given just the module name of a workflow, executes the workflow as if it were given an empty struct" do 829 | assert Pacer.Workflow.execute(TheTestWorkflow) == Pacer.Workflow.execute(%TheTestWorkflow{}) 830 | end 831 | 832 | test "runs each resolver with the values set on each dependent key, even when not pulling from defaults" do 833 | assert %TheTestWorkflow{a: 2, b: 3} == Pacer.Workflow.execute(%TheTestWorkflow{a: 2}) 834 | end 835 | 836 | defmodule NoBatchWorkflowWithResolverFailure do 837 | use Pacer.Workflow 838 | 839 | graph do 840 | field(:a, default: 1) 841 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a]) 842 | end 843 | 844 | def calculate_b(%{a: _a}), do: raise("OH NO") 845 | end 846 | 847 | test "when field's resolver raises, exits" do 848 | assert_raise RuntimeError, fn -> 849 | Pacer.Workflow.execute(%NoBatchWorkflowWithResolverFailure{}) 850 | end 851 | end 852 | end 853 | 854 | describe "execute/1 with batches" do 855 | defmodule TestWorkflowWithSingleBasicBatch do 856 | use Pacer.Workflow 857 | 858 | graph do 859 | field(:a, default: "the start") 860 | 861 | batch :requests do 862 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a], default: "here in b") 863 | field(:c, resolver: &__MODULE__.calculate_c/1, dependencies: [:a], default: "here in c") 864 | end 865 | end 866 | 867 | def calculate_b(%{a: a}), do: a <> " plus b" 868 | def calculate_c(%{a: a}), do: a <> " plus c" 869 | end 870 | 871 | test "runs batched resolvers and puts the results on the associated keys" do 872 | assert %TestWorkflowWithSingleBasicBatch{ 873 | a: "the start", 874 | b: "the start plus b", 875 | c: "the start plus c" 876 | } == Pacer.Workflow.execute(%TestWorkflowWithSingleBasicBatch{}) 877 | end 878 | 879 | defmodule TestWorkflowWithSingleBasicBatchAndQuickTimeout do 880 | use Pacer.Workflow 881 | 882 | graph do 883 | field(:a, default: "the start") 884 | 885 | batch :requests, timeout: 10 do 886 | field(:b, 887 | resolver: &__MODULE__.calculate_b/1, 888 | dependencies: [:a], 889 | default: "timed out so here in b" 890 | ) 891 | 892 | field(:c, 893 | resolver: &__MODULE__.calculate_c/1, 894 | dependencies: [:a], 895 | default: "timed out so here in c" 896 | ) 897 | end 898 | end 899 | 900 | def calculate_b(%{a: a}) do 901 | Process.sleep(1000) 902 | a <> " plus b" 903 | end 904 | 905 | def calculate_c(%{a: a}) do 906 | Process.sleep(1000) 907 | a <> " plus c" 908 | end 909 | end 910 | 911 | test "when batch field resolvers time out, return the defaults" do 912 | assert %TestWorkflowWithSingleBasicBatchAndQuickTimeout{ 913 | a: "the start", 914 | b: "timed out so here in b", 915 | c: "timed out so here in c" 916 | } == Pacer.Workflow.execute(%TestWorkflowWithSingleBasicBatchAndQuickTimeout{}) 917 | end 918 | 919 | defmodule WorkflowWithSingleFieldBatch do 920 | use Pacer.Workflow 921 | 922 | graph do 923 | field(:a, default: 1) 924 | 925 | batch :requests do 926 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a], default: 4) 927 | end 928 | end 929 | 930 | def calculate_b(%{a: a}) do 931 | send(self(), :calculating_b) 932 | a + 2 933 | end 934 | end 935 | 936 | test "a batch with a single field runs in the same process as the caller" do 937 | assert %WorkflowWithSingleFieldBatch{ 938 | a: 1, 939 | b: 3 940 | } == Pacer.Workflow.execute(%WorkflowWithSingleFieldBatch{}) 941 | 942 | assert_received :calculating_b 943 | end 944 | 945 | defmodule WorkflowWithTwoBatches do 946 | use Pacer.Workflow 947 | 948 | graph do 949 | field(:a, default: "the start") 950 | field(:b, default: "the end") 951 | 952 | batch :one do 953 | field(:c, 954 | resolver: &__MODULE__.batch_one_resolver/1, 955 | dependencies: [:a], 956 | default: "here in c" 957 | ) 958 | 959 | field(:d, 960 | resolver: &__MODULE__.batch_one_resolver/1, 961 | dependencies: [:a], 962 | default: "here in d" 963 | ) 964 | end 965 | 966 | batch :two do 967 | field(:e, 968 | resolver: &__MODULE__.batch_two_resolver/1, 969 | dependencies: [:b], 970 | default: "here in e" 971 | ) 972 | 973 | field(:f, 974 | resolver: &__MODULE__.batch_two_resolver/1, 975 | dependencies: [:b], 976 | default: "here in f" 977 | ) 978 | end 979 | end 980 | 981 | def batch_one_resolver(%{a: a}), do: a <> " plus a batch one resolver" 982 | def batch_two_resolver(%{b: b}), do: "a batch two resolver plus " <> b 983 | end 984 | 985 | test "with multiple batches, evaluates each resolver within batches and places the results on the associated keys" do 986 | assert %WorkflowWithTwoBatches{ 987 | a: "the start", 988 | b: "the end", 989 | c: "the start plus a batch one resolver", 990 | d: "the start plus a batch one resolver", 991 | e: "a batch two resolver plus the end", 992 | f: "a batch two resolver plus the end" 993 | } == Pacer.Workflow.execute(%WorkflowWithTwoBatches{}) 994 | end 995 | 996 | defmodule BatchWorkflowWithResolverFailure do 997 | use Pacer.Workflow 998 | 999 | graph do 1000 | field(:a, default: 1) 1001 | 1002 | batch :requests do 1003 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a], default: 6) 1004 | end 1005 | end 1006 | 1007 | def calculate_b(%{a: _a}), do: raise("OH NO") 1008 | end 1009 | 1010 | test "when batch field's resolver raises, returns default" do 1011 | assert %BatchWorkflowWithResolverFailure{a: 1, b: 6} == 1012 | Pacer.Workflow.execute(%BatchWorkflowWithResolverFailure{}) 1013 | end 1014 | 1015 | defmodule BatchWorkflowWithMultipleResolverIssues do 1016 | use Pacer.Workflow 1017 | 1018 | graph do 1019 | field(:a, default: 1) 1020 | 1021 | batch :requests, timeout: 1000 do 1022 | field(:b, 1023 | resolver: &__MODULE__.calculate_b/1, 1024 | dependencies: [:a], 1025 | default: "ddddefault" 1026 | ) 1027 | 1028 | field(:c, 1029 | resolver: &__MODULE__.calculate_c/1, 1030 | dependencies: [:a], 1031 | default: "timed out so default here for c" 1032 | ) 1033 | 1034 | field(:d, 1035 | resolver: &__MODULE__.calculate_d/1, 1036 | dependencies: [:a], 1037 | default: 6 1038 | ) 1039 | end 1040 | end 1041 | 1042 | def calculate_b(%{a: _a}), do: raise("OH NO") 1043 | 1044 | def calculate_c(%{a: _a}) do 1045 | Process.sleep(1500) 1046 | "cccccc" 1047 | end 1048 | 1049 | def calculate_d(%{a: a}), do: a + 23 1050 | end 1051 | 1052 | test "all sorts of batch issues, but returns defaults and or resolved results" do 1053 | assert %BatchWorkflowWithMultipleResolverIssues{ 1054 | a: 1, 1055 | b: "ddddefault", 1056 | c: "timed out so default here for c", 1057 | d: 24 1058 | } == 1059 | Pacer.Workflow.execute(%BatchWorkflowWithMultipleResolverIssues{}) 1060 | end 1061 | end 1062 | 1063 | describe "batch field guards" do 1064 | defmodule BatchWorkflowWithGuard do 1065 | use Pacer.Workflow 1066 | 1067 | graph do 1068 | field(:need_to_do_more_work?) 1069 | field(:other_field) 1070 | field(:test_pid) 1071 | 1072 | batch :test_batch do 1073 | field(:no_guard_field, 1074 | resolver: &__MODULE__.send_message/1, 1075 | dependencies: [:other_field, :test_pid], 1076 | default: :ok 1077 | ) 1078 | 1079 | field(:guard_field, 1080 | resolver: &__MODULE__.send_message/1, 1081 | guard: &__MODULE__.should_execute?/1, 1082 | dependencies: [:need_to_do_more_work?, :test_pid], 1083 | default: "simple default" 1084 | ) 1085 | end 1086 | end 1087 | 1088 | def send_message(%{test_pid: test_pid} = deps) do 1089 | send(test_pid, {:resolver_executed, Map.put(deps, :process_pid, self())}) 1090 | 1091 | :ok 1092 | end 1093 | 1094 | def should_execute?(%{need_to_do_more_work?: true}), do: true 1095 | def should_execute?(_), do: false 1096 | end 1097 | 1098 | test "when guard functions return false, no concurrent process is started to execute a batch field's resolver" do 1099 | test_pid = self() 1100 | 1101 | workflow = %BatchWorkflowWithGuard{ 1102 | need_to_do_more_work?: false, 1103 | other_field: "look for me in message mailbox", 1104 | test_pid: test_pid 1105 | } 1106 | 1107 | assert %BatchWorkflowWithGuard{guard_field: "simple default"} = 1108 | Pacer.Workflow.execute(workflow) 1109 | 1110 | # The non-guarded field depends on `other_field`, pattern match on that to assert message is received from other process 1111 | assert_receive {:resolver_executed, 1112 | %{other_field: "look for me in message mailbox", process_pid: process_pid}} 1113 | 1114 | refute process_pid == test_pid 1115 | 1116 | # The guarded field depends on need_to_do_more_work? and test_pid; refute this message was received because 1117 | # the resolver should not run 1118 | refute_receive {:resolver_executed, %{need_to_do_more_work?: false, test_pid: ^test_pid}} 1119 | end 1120 | 1121 | test "when guard functions return true, a concurrent process is started to execute a batch field's resolver" do 1122 | test_pid = self() 1123 | 1124 | workflow = %BatchWorkflowWithGuard{ 1125 | need_to_do_more_work?: true, 1126 | other_field: "look for me in message mailbox", 1127 | test_pid: test_pid 1128 | } 1129 | 1130 | assert %BatchWorkflowWithGuard{guard_field: :ok} = Pacer.Workflow.execute(workflow) 1131 | 1132 | assert_receive {:resolver_executed, 1133 | %{other_field: "look for me in message mailbox", process_pid: process_pid}} 1134 | 1135 | refute process_pid == test_pid 1136 | 1137 | assert_receive {:resolver_executed, 1138 | %{need_to_do_more_work?: true, process_pid: guarded_process_pid}} 1139 | 1140 | refute guarded_process_pid == test_pid 1141 | end 1142 | end 1143 | 1144 | describe "visualization" do 1145 | test "returns ok tuple with strict digraph as string" do 1146 | assert {:ok, stringed_digraph} = TestGraph.__graph__(:visualization) 1147 | 1148 | assert stringed_digraph =~ "strict digraph" 1149 | assert stringed_digraph =~ "label=\"http_requests\"" 1150 | assert stringed_digraph =~ "label=\"custom_field\"" 1151 | assert stringed_digraph =~ "label=\"field_with_default\"" 1152 | assert stringed_digraph =~ "label=\"field_a\"" 1153 | end 1154 | 1155 | test "when workflow is empty, return empty strict digraph as string" do 1156 | defmodule EmptyGraph do 1157 | use Pacer.Workflow 1158 | 1159 | graph do 1160 | end 1161 | end 1162 | 1163 | assert {:ok, "strict digraph {\n}\n"} = EmptyGraph.__graph__(:visualization) 1164 | end 1165 | end 1166 | 1167 | @batch_resolver_error_log """ 1168 | Resolver for Pacer.WorkflowTest.DebugModeTrueWorkflowWithResolverFailure.http_requests's resolver returned default. 1169 | Your resolver function failed for %RuntimeError{message: \"OH NO\"}. 1170 | 1171 | Returning default value of: :hello 1172 | """ 1173 | describe "execute with debug_mode on" do 1174 | defmodule DebugModeTrueWorkflowWithResolverFailure do 1175 | use Pacer.Workflow, debug_mode?: true 1176 | 1177 | graph do 1178 | field(:a, default: 1) 1179 | 1180 | batch :http_requests do 1181 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a], default: :hello) 1182 | end 1183 | end 1184 | 1185 | def calculate_b(%{a: _a}), do: raise("OH NO") 1186 | end 1187 | 1188 | test "when field's resolver raises, logs resolver failure message with error and default value" do 1189 | logs = 1190 | capture_log(fn -> 1191 | Pacer.Workflow.execute(%DebugModeTrueWorkflowWithResolverFailure{}) 1192 | end) 1193 | 1194 | assert logs =~ @batch_resolver_error_log 1195 | end 1196 | end 1197 | end 1198 | -------------------------------------------------------------------------------- /lib/workflow.ex: -------------------------------------------------------------------------------- 1 | defmodule Pacer.Workflow do 2 | @moduledoc """ 3 | Dependency Graph-Based Workflows With Robust Compile Time Safety & Guarantees 4 | 5 | ## Motivations 6 | 7 | `Pacer.Workflow` is designed for complex workflows where many interdependent data points need to be 8 | stitched together to provide a final result, specifically workflows where each data point needs to be 9 | loaded and/or calculated using discrete, application-specific logic. 10 | 11 | To create a struct backed by Pacer.Workflow, invoke `use Pacer.Workflow` at the top of your module and use 12 | the `graph/1` macro, which is explained in more detail in the docs below. 13 | 14 | Note that when using `Pacer.Workflow`, you can pass the following options: 15 | #{NimbleOptions.docs(Pacer.Workflow.Options.graph_options())} 16 | 17 | The following is a list of the main ideas and themes underlying `Pacer.Workflow` 18 | 19 | #### 1. `Workflow`s Are Dependency Graphs 20 | 21 | `Pacer.Workflow`s are backed by dependency graphs (specifically represented as directed acyclic graphs) that are constructed at compile-time. 22 | Your `Workflow`s will define a set of data points, represented as `field`s (see below); each `field` must explicitly define 23 | the dependencies it has on other fields in the `Workflow`. For example, if we have a workflow where we load 24 | a set of users and then fire off some requests to a 3rd party service to fetch some advertisements for those users, 25 | our `Workflow` might look something like this: 26 | 27 | ```elixir 28 | defmodule UserAdsWorkflow do 29 | use Pacer.Workflow 30 | 31 | graph do 32 | field(:users) 33 | field(:user_advertisements, resolver: &Ads.fetch_user_ads/1, dependencies: [:users]) 34 | end 35 | end 36 | ``` 37 | 38 | Why is the dependency graph idea important here? 39 | 40 | In the above, simplified example with only two fields, there may not be a need to define a dependency graph 41 | because we can look at the two fields and immediately realize that we first need to have the set of `users` 42 | before we can make the call to load `:user_advertisements`. 43 | 44 | However, in complex workflows with dozens or even hundreds of data points, if we were to try to manage what data points 45 | need to be loaded in which order manually, it would be a daunting and time-consuming task. Not only that, we would also 46 | run the risk of getting the ordering wrong, AND/OR when new data points are added or removed in the future, that we would 47 | need to manually rearrange things for each data point to be loaded in the correct order. 48 | 49 | The result is untenable for workflows of sufficient size. 50 | 51 | This is where dependency graphs come in to play. By forcing you to explicitly declare the other fields that 52 | you depend on in the workflow, `Pacer.Workflow` can build out a dependency graph and figure out how to schedule the 53 | execution of each of your resolver functions (see below for more details on resolvers) so that each function 54 | will only be called when its dependencies are ready. That eliminates the need to manually rearrange calls in 55 | your codebase, and also allows you to have discrete, single-purpose resolvers that can be rigorously unit-tested 56 | against a known, constrained set of inputs. 57 | 58 | #### 2. Batched, Parallel Requests to Disparate External Systems (3rd-party APIs, Database Calls, etc.) 59 | 60 | `Pacer.Workflow`s also allow users to fire off potentially high-latency calls in parallel to reduce overall 61 | latency of a `Workflow`. To do so, we can use the `batch/2` macro inside of your `graph` definition. One caveat 62 | to this, however, is that _fields inside of a batch definition must not have any dependencies on other fields 63 | inside the same batch_. 64 | 65 | Batches are nice to use when a workflow has multiple high-latency requests that need to be made. Batching the 66 | requests together, when possible, will fire off the requests in parallel. The requests can be to disparate, 67 | unrelated services, APIs, and external systems including databases and/or caches. 68 | 69 | Note: `batch`es should not be confused with `batch loading` data in the sense that, for example, GraphQL batches 70 | are used where users may provide a set of ids, etc., for related entities and the batch processing loads all of or 71 | as many of those entities in a single request rather than making a single request per entity. `Pacer.Workflow` batches 72 | can be used to do so in roughly the same way, but that choice is left up to the user and the implementation. 73 | The key idea of a `batch` here is that you have multiple (potentially) high-latency requests that you want to execute 74 | together (in parallel), rather than saying "I have a set of entities that I want to load as a batch request". 75 | 76 | For example, if we go back to the earlier example of a user-based workflow where we load a set of users and fetch 77 | advertisements for those users, if we add in another request to, say, an analytics service to get some more data on 78 | the set of users we have just loaded, we can do that in a batch as follows: 79 | 80 | ```elixir 81 | defmodule UserAdsWorkflow do 82 | use Pacer.Workflow 83 | 84 | graph do 85 | field(:users) 86 | batch :requests do 87 | field(:user_advertisements, resolver: &Ads.fetch_user_ads/1, dependencies: [:users]) 88 | field(:analytics, resolver: &Analytics.analyze_users/1, dependencies: [:users]) 89 | end 90 | end 91 | end 92 | ``` 93 | 94 | Now, rather than those two requests being fired sequentially (and thereby boosting the latency of the workflow to be 95 | equal to the latency of the ads request _plus_ the latency of the analytics request, 96 | the latency will instead be capped at the slowest of the two requests). 97 | 98 | #### 3. Compile-Time Safety And Guarantees 99 | 100 | The third motivating factor behind `Pacer.Workflow` is to provide a robust set of compile-time safety mechanisms. 101 | These include: 102 | - Detecting and preventing cyclical dependencies in the dependency graph defined by the workflow 103 | - Preventing "reflexive" dependencies, where a field depends on itself 104 | - Detecting invalid options on `field`s and `batch`es 105 | - Preventing a single module from defining more than one `Workflow` 106 | - Detecting duplicate field definitions in a graph 107 | - Ensuring that resolver definitions fit the contract required by `Pacer.Workflow` (a 1-arity function that takes a map) 108 | - Detecting dependencies on fields that do not exist in the graph definition 109 | - Requiring fields defined inside of a batch to have a resolver function defined 110 | 111 | `Pacer.Workflow` strives to provide helpful error messages to the user at compile time when it detects any issues 112 | and tries to direct the user on what went wrong, why, and how to fix the issue. 113 | 114 | The compile-time safety can prevent a whole class of issues at runtime, and also allows the dependency graph 115 | to be computed once at compile time. Building the dependency graph at compile time allows Pacer to cache the 116 | results of the graph and make those results accessible at runtime so your application does not have to incur 117 | the cost of building out the dependency graph at runtime. 118 | 119 | ## Summary 120 | 121 | `Pacer.Workflow` provides the ability to explicitly declare a dependency graph, where the nodes in the 122 | graph map to fields in a struct defined via the `graph/1` API. 123 | 124 | The key idea behind `Pacer.Workflow` is that it enables users to create Elixir structs that serve as containers 125 | of loosely-related fields, where the fields in the struct have dependencies between other fields in the struct. 126 | 127 | A "dependency", in the sense it is used here, means that one field relies on another field's value being readily 128 | available and loaded in memory before its own value can be computed or loaded. For example, if you have a struct 129 | `%MyStruct{field_a: 1, field_b: }`, `:field_b` is dependent on `:field_a`'s value already 130 | being present before it can be calculated. 131 | 132 | The example given above can be solved in a more straightforward way, by having a simple function to build out the 133 | entire struct given `:field_a`'s value as input, i.e.: 134 | 135 | ```elixir 136 | def build_my_struct(field_a_input) do 137 | %MyStruct{field_a: field_a_input, field_b: field_a_input + 1} 138 | end 139 | ``` 140 | 141 | While conceptually simple, this pattern becomes more difficult to maintain when additional fields are added with dependencies between each other. 142 | 143 | `Pacer.Workflow` addresses this problem by forcing users to explicitly declare the dependencies between fields up front, at compile time. 144 | Once the fields and dependencies have been declared, `Pacer.Workflow` can build a dependency graph, which allows the graph to solve for 145 | the problem of dependency resolution by answering the question: Which fields need to be available when and in what order do they need to be executed? 146 | 147 | There are a few key concepts to know in order to build out a `Pacer.Workflow`-backed struct: 148 | 149 | ## Fields 150 | 151 | A field can be defined within a graph definition with the `field/2` macro. A field 152 | maps one-to-one to keys on the struct generated by the graph definition. Fields are 153 | how you explicitly declare the dependencies each field has on other fields within the 154 | same graph. You do this by providing a list of dependencies as atoms to the `field/2` 155 | macro: 156 | 157 | ```elixir 158 | graph do 159 | field(:field_one) 160 | field(:field_two) 161 | field(:my_dependent_field, resolver: &MyResolver.resolve/1 dependencies: [:field_one, :field_two]) 162 | end 163 | ``` 164 | 165 | If the `:dependencies` option is not given, it defaults to an empty list and effectively means 166 | that the field has no dependencies. This may be the case when the value for the field meets one 167 | of the following conditions: 168 | 169 | - The value is a constant 170 | - The value is already available and accessible in memory when creating the struct 171 | 172 | Fields that do explicitly declare at least one dependency MUST also pass in a `:resolver` option. 173 | See the [Resolvers](#resolvers) section below for more details. 174 | 175 | Additionally, fields may declare a default value by passing a default to the `:default` option key: 176 | 177 | ```elixir 178 | graph do 179 | field(:my_field, default: 42) 180 | end 181 | ``` 182 | 183 | ## Resolvers 184 | 185 | Resolvers are 1-arity functions that take in the values from dependencies as input and return 186 | the value that should be placed on the struct key for the associated `field`. Resolvers are 187 | function definitions that `Pacer.Workflow` can use to incrementally compute all values needed. 188 | 189 | For example, for a graph definition that looks like this: 190 | 191 | ```elixir 192 | defmodule MyGraph do 193 | use Pacer.Workflow 194 | 195 | graph do 196 | field(:field_one) 197 | field(:dependent_field, resolver: &__MODULE__.resolve/1, dependencies: [:field_one]) 198 | end 199 | 200 | def resolve(inputs) do 201 | IO.inspect(inputs.field_one, label: "Received field_one's value") 202 | end 203 | end 204 | ``` 205 | 206 | Resolver functions will always be called with a map that contains the values for fields declared as dependencies. 207 | In the above example, that means if we have a struct `%MyGraph{field_one: 42}`, the resolver will be invoked with 208 | `%{field_one: 42}`. 209 | 210 | Keep in mind that if you declare any dependencies, you MUST also declare a resolver. 211 | 212 | 213 | ## Batches 214 | 215 | Batches can be defined using the `batch/3` macro. 216 | 217 | Batches allow users to group together a set of fields whose resolvers can and should be run in 218 | parallel. The main use-case for batches is to reduce running time for fields whose resolvers can 219 | have high-latencies. This generally means that batches are useful to group together calls that hit 220 | the network in some way. 221 | 222 | Batches do impose some more restrictive constraints on users, however. For example, all fields 223 | defined within a batch MUST NOT declare dependencies on any other field in the same batch. This 224 | is because the resolvers will run concurrently with one another, so there is no way to guarantee 225 | that a field within the same batch will have a value ready to use and pass to a separate resolver 226 | in the same batch. In scenarios where you find this happening, `Pacer.Workflow` will raise a compile time 227 | error and you will need to rearrange your batches, possibly creating two separate batches or forcing 228 | one field in the batch to run sequentially as a regular field outside of a `batch` block. 229 | 230 | Batches must also declare a name and fields within a batch must define a resolver. 231 | Batch names must also be unique within a single graph definition. Resolvers are required 232 | for fields within a batch regardless of whether or not the field has any dependencies. 233 | 234 | Ex.: 235 | 236 | ```elixir 237 | defmodule MyGraphWithBatches do 238 | use Pacer.Workflow 239 | 240 | graph do 241 | field(:regular_field) 242 | 243 | batch :http_requests do 244 | field(:request_one, resolver: &__MODULE__.resolve/1) 245 | field(:request_two, resolver: &__MODULE__.resolve/1, dependencies: [:regular_field]) 246 | end 247 | 248 | field(:another_field, resolver: &__MODULE__.simple_resolver/1, dependencies: [:request_two]) 249 | end 250 | 251 | def resolve(_) do 252 | IO.puts("Simulating HTTP request") 253 | end 254 | 255 | def simple_resolver(_), do: :ok 256 | end 257 | ``` 258 | 259 | Notes: 260 | 261 | The order fields are defined in within a `graph` definition does not matter. For example, if you have a field `:request_one` that depends 262 | on another field `:request_two`, the fields can be declared in any order. 263 | 264 | ## Telemetry 265 | 266 | Pacer provides two levels of granularity for workflow telemetry: one at the entire workflow level, and one at the resolver level. 267 | 268 | For workflow execution, Pacer will trigger the following telemetry events: 269 | 270 | - `[:pacer, :workflow, :start]` 271 | - Measurements include: `%{system_time: integer(), monotonic_time: integer()}` 272 | - Metadata provided: `%{telemetry_span_context: term(), workflow: module()}`, where the `workflow` key contains the module name for the workflow being executed 273 | - `[:pacer, :workflow, :stop]` 274 | - Measurements include: `%{duration: integer(), monotonic_time: integer()}` 275 | - Metadata provided: `%{telemetry_span_context: term(), workflow: module()}`, where the `workflow` key contains the module name for the workflow being executed 276 | - `[:pacer, :workflow, :exception]` 277 | - Measurements include: `%{duration: integer(), monotonic_time: integer()}` 278 | - Metadata provided: %{kind: :throw | :error | :exit, reason: term(), stacktrace: list(), telemetry_span_context: term(), workflow: module()}, where the `workflow` key contains the module name for the workflow being executed 279 | 280 | At the resolver level, Pacer will trigger the following telemetry events: 281 | 282 | - `[:pacer, :execute_vertex, :start]` 283 | - Measurements and metadata similar to `:workflow` start event, with the addition of the `%{field: atom()}` value passed in metadata. The `field` is the name of the field for which the resolver is being executed. 284 | - `[:pacer, :execute_vertex, :stop]` 285 | - Measurements and metadata similar to `:workflow` stop event, with the addition of the `%{field: atom()}` value passed in metadata. The `field` is the name of the field for which the resolver is being executed. 286 | - `[:pacer, :execute_vertex, :exception]` 287 | - Measurements and metadata similar to `:workflow` exception event, with the addition of the `%{field: atom()}` value passed in metadata. The `field` is the name of the field for which the resolver is being executed. 288 | 289 | Additionally, for `[:pacer, :execute_vertex]` events fired on batched resolvers (which will run in parallel processes), users can provide their own metadata through configuration. 290 | 291 | Users may provide either a keyword list of options which will be merged into the `:execute_vertex` event metadata, or an MFA `{mod, fun, args}` tuple that points to a function which 292 | returns a keyword list that will be merged into the `:execute_vertex` event metadata. 293 | 294 | There are two routes for configuring these telemetry options for batched resolvers: in the application environment using the `:pacer, :batch_telemetry_options` config key, or 295 | on the individual workflow modules themselves by passing `:batch_telemetry_options` when invoking `use Pacer.Workflow`. 296 | Configuration defined at the workflow module will override configuration defined in the application environment. 297 | 298 | Here are a couple of examples: 299 | 300 | ### User-Provided Telemetry Metadata for Batched Resolvers in Applicaton Config 301 | ```elixir 302 | # In config.exs (or whatever env config file you want to target): 303 | 304 | config :pacer, :batch_telemetry_options, application_name: MyApp 305 | 306 | ## When you invoke a workflow with batched resolvers now, you will get `%{application_name: MyApp}` merged into your 307 | ## event metadata in the `[:pacer, :execute_vertex, :start | :stop | :exception]` events. 308 | ``` 309 | 310 | ### User-Provided Telemetry Metadata for Batched Resolvers at the Workflow Level 311 | ```elixir 312 | defmodule MyWorkflow do 313 | use Pacer.Workflow, batch_telemetry_options: [extra_context: "some context from my application"] 314 | 315 | graph do 316 | field(:a) 317 | 318 | batch :long_running_requests do 319 | field(:b, dependencies: [:a], resolver: &Requests.trigger_b/1, default: nil) 320 | field(:c, dependencies: [:a], resolver: &Requests.trigger_c/1, default: nil) 321 | end 322 | end 323 | end 324 | 325 | ## Now when you invoke `Pacer.execute(MyWorkflow)`, you will get `%{extra_context: "some context from my application"}` 326 | ## merged into the metadata for the `[:pacer, :execute_vertex, :start | :stop | :exception]` events for fields `:b` and `:c` 327 | ``` 328 | 329 | Note that you can also provide an MFA tuple that points to a module/function that returns a keyword list of options to be 330 | injected into the metadata on `:execute_vertex` telemetry events for batched resolvers. This allows users to execute code at runtime 331 | to inject dynamic values into the metadata. Users may use this to inject things like span_context from the top-level workflow process 332 | into the parallel processes that run the batch resolvers. This lets you propagate context from, i.e., a process dictionary at the top-level 333 | into the sub-processes: 334 | 335 | ```elixir 336 | defmodule MyApp.BatchOptions do 337 | def inject_context do 338 | [span_context: MyTracingLibrary.Tracer.current_context()] 339 | end 340 | end 341 | 342 | ## Use this function to inject span context by configuring it at the workflow level or in the application environment 343 | 344 | ## In config.exs: 345 | 346 | config :pacer, :batch_telemetry_options, {MyApp.BatchOptions, :inject_context, []} 347 | ``` 348 | 349 | ### Using debug_mode config: 350 | 351 | An optional config for debug mode will log out caught errors from batch resolvers. 352 | This is helpful for local development as the workflow will catch the error and return 353 | the default to keep the workflow continuing. 354 | 355 | ```elixir 356 | defmodule MyWorkflow do 357 | use Pacer.Workfow, debug_mode?: true 358 | 359 | graph do 360 | field(:a, default: 1) 361 | 362 | batch :http_requests do 363 | field(:b, resolver: &__MODULE__.calculate_b/1, dependencies: [:a], default: :hello) 364 | end 365 | end 366 | 367 | def calculate_b(%{a: _a}), do: raise("OH NO") 368 | end 369 | ``` 370 | When running the workflow above, field b will silently raise as the default 371 | will be returned. In debug mode, you will also get a log telling you the error 372 | and the default returned. 373 | 374 | """ 375 | alias Pacer.Config 376 | alias Pacer.Workflow.Error 377 | alias Pacer.Workflow.FieldNotSet 378 | alias Pacer.Workflow.Options 379 | 380 | require Logger 381 | require Pacer.Docs 382 | 383 | @example_graph_message """ 384 | Ex.: 385 | 386 | defmodule MyValidGraph do 387 | use Pacer.Workflow 388 | 389 | graph do 390 | field(:custom_field) 391 | field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 392 | field(:field_with_default, default: "this is a default value") 393 | 394 | batch :http_requests, timeout: :timer.seconds(1) do 395 | field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field], default: 5) 396 | 397 | field(:request_2, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field, :field_a], default: "this is the default for request2") 398 | field(:request_3, resolver: &__MODULE__.do_work/1, default: :this_default) 399 | end 400 | end 401 | 402 | def do_work(_), do: :ok 403 | end 404 | """ 405 | 406 | @default_batch_options Options.default_batch_options() 407 | 408 | defmacro __using__(opts) do 409 | quote do 410 | import Pacer.Workflow, 411 | only: [ 412 | graph: 1, 413 | batch: 2, 414 | batch: 3, 415 | field: 1, 416 | field: 2, 417 | find_cycles: 1 418 | ] 419 | 420 | alias Pacer.Workflow.Options 421 | require Pacer.Docs 422 | 423 | Module.register_attribute(__MODULE__, :pacer_generate_docs?, accumulate: false) 424 | 425 | generate_docs? = Keyword.get(unquote(opts), :generate_docs?, true) 426 | 427 | Module.put_attribute( 428 | __MODULE__, 429 | :pacer_generate_docs?, 430 | generate_docs? 431 | ) 432 | 433 | Module.register_attribute(__MODULE__, :pacer_debug_mode?, accumulate: false) 434 | debug_mode? = Keyword.get(unquote(opts), :debug_mode?, false) 435 | Module.put_attribute(__MODULE__, :pacer_debug_mode?, debug_mode?) 436 | 437 | batch_telemetry_options = Keyword.get(unquote(opts), :batch_telemetry_options, %{}) 438 | 439 | Module.put_attribute(__MODULE__, :pacer_batch_telemetry_options, batch_telemetry_options) 440 | 441 | Module.register_attribute(__MODULE__, :pacer_docs, accumulate: true) 442 | Module.register_attribute(__MODULE__, :pacer_graph_vertices, accumulate: true) 443 | Module.register_attribute(__MODULE__, :pacer_field_to_batch_mapping, accumulate: false) 444 | Module.register_attribute(__MODULE__, :pacer_fields, accumulate: true) 445 | Module.register_attribute(__MODULE__, :pacer_struct_fields, accumulate: true) 446 | Module.register_attribute(__MODULE__, :pacer_dependencies, accumulate: true) 447 | Module.register_attribute(__MODULE__, :pacer_resolvers, accumulate: true) 448 | Module.register_attribute(__MODULE__, :pacer_batches, accumulate: true) 449 | Module.register_attribute(__MODULE__, :pacer_batch_dependencies, accumulate: true) 450 | Module.register_attribute(__MODULE__, :pacer_batch_guard_functions, accumulate: true) 451 | Module.register_attribute(__MODULE__, :pacer_batch_options, accumulate: true) 452 | Module.register_attribute(__MODULE__, :pacer_batch_resolvers, accumulate: true) 453 | Module.register_attribute(__MODULE__, :pacer_vertices, accumulate: true) 454 | Module.register_attribute(__MODULE__, :pacer_virtual_fields, accumulate: true) 455 | end 456 | end 457 | 458 | @doc """ 459 | The graph/1 macro is the main entrypoint into Pacer.Workflow to create a dependency graph struct. 460 | `use` the `Pacer.Workflow` macro at the top of your module and proceed to define your fields and/or batches. 461 | 462 | Example 463 | ```elixir 464 | defmodule MyValidGraph do 465 | use Pacer.Workflow 466 | 467 | graph do 468 | field(:custom_field) 469 | field(:field_a, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 470 | field(:field_with_default, default: "this is a default value") 471 | 472 | batch :http_requests do 473 | field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field, :field_a]) 474 | field(:request_2, resolver: &__MODULE__.do_work/1) 475 | end 476 | end 477 | 478 | def do_work(_), do: :ok 479 | end 480 | ``` 481 | 482 | Your module may only define ONE graph per module. 483 | 484 | The above example will also create a struct with all of the fields defined within the graph, as follows: 485 | 486 | ```elixir 487 | %MyValidGraph{ 488 | custom_field: nil, 489 | field_a: nil, 490 | field_with_default: "this is a default value", 491 | request_1: nil, 492 | request_2: nil 493 | } 494 | ``` 495 | 496 | 497 | The graph macro gives you access to some defined metadata functions, such as (using the above example graph): 498 | - `MyValidGraph.__graph__(:fields)` 499 | - `MyValidGraph.__graph__(:dependencies, :http_requests)` 500 | - `MyValidGraph.__graph__(:resolver, :field_a)` 501 | 502 | **Caution: These metadata functions are mostly intended for Pacer's internal use. Do not rely on their return values in runtime 503 | code as they may change as changes are made to the interface for Pacer. 504 | """ 505 | # credo:disable-for-lines:79 Credo.Check.Refactor.ABCSize 506 | # credo:disable-for-lines:79 Credo.Check.Refactor.CyclomaticComplexity 507 | defmacro graph(do: block) do 508 | caller = __CALLER__ 509 | 510 | prelude = 511 | quote do 512 | Module.put_attribute(__MODULE__, :pacer_field_to_batch_mapping, %{}) 513 | 514 | if line = Module.get_attribute(__MODULE__, :pacer_graph_defined) do 515 | raise Error, """ 516 | Module #{inspect(__MODULE__)} already defines a graph on line #{line} 517 | """ 518 | end 519 | 520 | @pacer_graph_defined unquote(caller.line) 521 | 522 | @after_compile Pacer.Workflow 523 | 524 | unquote(block) 525 | end 526 | 527 | # credo:disable-for-lines:155 Credo.Check.Refactor.LongQuoteBlocks 528 | postlude = 529 | quote unquote: false do 530 | batched_dependencies = 531 | Enum.reduce(@pacer_batch_dependencies, %{}, fn {batch_name, _field_name, deps}, 532 | batched_dependencies -> 533 | Map.update(batched_dependencies, batch_name, deps, fn existing_val -> 534 | Enum.uniq(Enum.concat(existing_val, deps)) 535 | end) 536 | end) 537 | 538 | batched_field_dependencies = 539 | Enum.reduce(@pacer_batch_dependencies, %{}, fn {_batch_name, field_name, deps}, 540 | batched_field_dependencies -> 541 | Map.put(batched_field_dependencies, field_name, deps) 542 | end) 543 | 544 | batched_fields = 545 | Enum.reduce(@pacer_batch_dependencies, %{}, fn {batch_name, field_name, _deps}, acc -> 546 | Map.update(acc, batch_name, [field_name], fn existing_val -> 547 | [field_name | existing_val] 548 | end) 549 | end) 550 | 551 | batched_resolvers = 552 | Enum.reduce(@pacer_batch_resolvers, %{}, fn {batch, field, resolver}, acc -> 553 | Map.update(acc, batch, [{field, resolver}], fn fields_and_resolvers -> 554 | [{field, resolver} | fields_and_resolvers] 555 | end) 556 | end) 557 | 558 | batch_guard_functions = 559 | Enum.reduce(@pacer_batch_guard_functions, %{}, fn {_batch, field, guard}, acc -> 560 | Map.put(acc, field, guard) 561 | end) 562 | 563 | @pacer_batch_dependencies 564 | |> Enum.reduce([], fn {batch, _, _}, batches -> [batch | batches] end) 565 | |> Enum.each(fn batch -> 566 | fields_for_batch = 567 | batched_fields 568 | |> Map.get(batch, []) 569 | |> MapSet.new() 570 | 571 | deps_for_batch = 572 | batched_dependencies 573 | |> Map.get(batch, []) 574 | |> MapSet.new() 575 | 576 | intersection = MapSet.intersection(fields_for_batch, deps_for_batch) 577 | 578 | unless Enum.empty?(intersection) do 579 | raise Error, 580 | """ 581 | Found at least one invalid field dependency inside of a Pacer.Workflow batch. 582 | Invalid dependencies: #{inspect(Enum.to_list(intersection))} 583 | Graph module: #{inspect(__MODULE__)} 584 | 585 | Fields that are defined within a batch MUST not have dependencies on other 586 | fields in the same batch because their resolvers will run concurrently. 587 | 588 | You may need to rearrange an invalid field (or fields) out of your batch 589 | if the field does have a hard dependency on another field in the batch. 590 | """ 591 | end 592 | end) 593 | 594 | # Instantiate the graph with the list of vertices derived from the graph definition 595 | initial_graph = Graph.add_vertices(Graph.new(), @pacer_graph_vertices) 596 | 597 | # Build the graph edges, where edges go from dependency -> dependent field OR batch. 598 | # We concat together `@pacer_batch_dependencies` with `@pacer_dependencies`: 599 | # - `@pacer_dependencies` is a list of `{field_name, list(field_dependencies)}` 600 | # - `@pacer_batch_dependencies` is a list of 3-tuples: `{batch_name, field_name, list(field_dependencies)}`. 601 | # The dependencies of all fields defined within a batch bubble up to the batch 602 | # itself. 603 | # Batch names become vertices in the graph, but the fields under 604 | # a batch are not represented in the graph as vertices: they collapse into the 605 | # vertex for the batch. This means that there is special treatment for batches. 606 | # Fields not within a batch ARE represented as vertices in the graph. So building 607 | # edges for fields not within a batch is relatively straighforward: take each dependency 608 | # the field defines and draw an edge from the dependency to the field. 609 | # Edges for batches require that we concat the dependencies for all fields within a batch, 610 | # then take that list of dependencies and iterate of each dependency. The edges then 611 | # become `dependency -> batch`. 612 | # Finally, if a dependency is itself part of a batch, we have to lookup the 613 | # batch it belongs to. Then, instead of drawing an edge from the dependency itself 614 | # to the field or batch that depends on it, we draw an edge from the dependency's batch 615 | # to the field or batch that depends on it. 616 | # The `@pacer_field_to_batch_mapping` is a map with plain fields as keys and the batch they 617 | # belong to as values. If a field does not belong to a batch, a lookup into the `@pacer_field_to_batch_mapping` 618 | # will return `nil`. We use this in the case statements within the `Enum.flat_map/2` call 619 | # to indicate whether or not we need to replace a field with the batch it belongs to when 620 | # drawing the edges. 621 | graph_edges = 622 | @pacer_batch_dependencies 623 | |> Enum.concat(@pacer_dependencies) 624 | |> Enum.flat_map(fn 625 | {batch, field, deps} -> 626 | for dep <- deps do 627 | case Map.get(@pacer_field_to_batch_mapping, dep) do 628 | nil -> Graph.Edge.new(dep, batch) 629 | dependency_batch -> Graph.Edge.new(dependency_batch, batch) 630 | end 631 | end 632 | 633 | {field, deps} -> 634 | for dep <- deps do 635 | case Map.get(@pacer_field_to_batch_mapping, dep) do 636 | nil -> Graph.Edge.new(dep, field) 637 | batch -> Graph.Edge.new(batch, field) 638 | end 639 | end 640 | end) 641 | 642 | # Update the graph with the graph edges 643 | graph = Graph.add_edges(initial_graph, graph_edges) 644 | 645 | visualization = Graph.to_dot(graph) 646 | # Technically, the call to `topsort/1` will fail (return `false`) if 647 | # there are any cycles in the graph, so we can indirectly derive whether or not 648 | # we have any cycles based on the return value from the call to `topsort/1` below. 649 | # However, we want to raise and show an error message that explicitly indicates 650 | # what vertices of the graph form the cycle so the user can more easily find and 651 | # fix any cycles they may have introduced. 652 | _ = find_cycles(graph) 653 | 654 | topsort = Graph.topsort(graph) 655 | 656 | if @pacer_generate_docs? do 657 | Pacer.Docs.generate() 658 | end 659 | 660 | # The call to `topsort/1` above returns all vertices in the graph. However, 661 | # not every vertex in the graph is going to have work to do, which we are deriving 662 | # based on whether or not the vertex has an associated resolver (for a field) or 663 | # list of resolvers (for batches). Any vertex that has NO associated resolvers 664 | # gets filtered out. 665 | # The result of this is what gets returned from the generated `def __graph__(:evaluation_order)` 666 | # function definition below. We will use this to iterate through the resolvers and execute them 667 | # in the order returned by the topological_sort. 668 | vertices_with_work_to_do = 669 | Enum.filter(topsort, fn vertex -> 670 | Keyword.get(@pacer_resolvers, vertex) || Map.get(batched_resolvers, vertex) 671 | end) 672 | 673 | defstruct Enum.reverse(@pacer_struct_fields) 674 | 675 | def __config__(:batch_telemetry_options), do: @pacer_batch_telemetry_options 676 | def __config__(:debug_mode?), do: @pacer_debug_mode? 677 | def __config__(_), do: nil 678 | 679 | def __graph__(:fields), do: Enum.reverse(@pacer_fields) 680 | def __graph__(:dependencies), do: Enum.reverse(@pacer_dependencies) 681 | def __graph__(:batch_dependencies), do: Enum.reverse(@pacer_batch_dependencies) 682 | def __graph__(:evaluation_order), do: unquote(vertices_with_work_to_do) 683 | def __graph__(:virtual_fields), do: Enum.reverse(@pacer_virtual_fields) 684 | def __graph__(:visualization), do: unquote(visualization) 685 | 686 | for {name, deps} <- @pacer_dependencies do 687 | def __graph__(:dependencies, unquote(name)), do: unquote(deps) 688 | end 689 | 690 | for {name, deps} <- batched_field_dependencies do 691 | def __graph__(:batched_field_dependencies, unquote(name)), do: unquote(deps) 692 | end 693 | 694 | for {name, guard} <- batch_guard_functions do 695 | def __graph__(:batched_field_guard_functions, unquote(name)), do: unquote(guard) 696 | end 697 | 698 | def __graph__(:batched_field_guard_functions, _field), do: nil 699 | 700 | for {batch_name, batch_options} <- @pacer_batch_options do 701 | def __graph__(unquote(batch_name), :options), do: unquote(batch_options) 702 | end 703 | 704 | for {batch_name, deps} <- batched_dependencies do 705 | def __graph__(:dependencies, unquote(batch_name)), do: unquote(deps) 706 | end 707 | 708 | for {batch_name, fields} <- batched_fields do 709 | def __graph__(:batch_fields, unquote(batch_name)), do: unquote(fields) 710 | end 711 | 712 | for {name, resolver} <- @pacer_resolvers do 713 | def __graph__(:resolver, unquote(name)), do: {:field, unquote(resolver)} 714 | end 715 | 716 | for {batch_name, fields_and_resolvers} <- batched_resolvers do 717 | def __graph__(:resolver, unquote(batch_name)), 718 | do: {:batch, unquote(fields_and_resolvers)} 719 | end 720 | end 721 | 722 | quote do 723 | unquote(prelude) 724 | unquote(postlude) 725 | end 726 | end 727 | 728 | @doc """ 729 | The batch/3 macro is to be invoked when grouping fields with resolvers that will run in parallel. 730 | 731 | Reminder: 732 | - The batch must be named and unique. 733 | - The fields within the batch must not have dependencies on one another since they will run concurrently. 734 | - The fields within the batch must each declare a resolver function. 735 | 736 | __NOTE__: In general, only batch fields whose resolvers contain potentially high-latency operations, such as network calls. 737 | 738 | Example 739 | ```elixir 740 | defmodule MyValidGraph do 741 | use Pacer.Workflow 742 | 743 | graph do 744 | field(:custom_field) 745 | 746 | batch :http_requests do 747 | field(:request_1, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 748 | field(:request_2, resolver: &__MODULE__.do_work/1, dependencies: [:custom_field]) 749 | field(:request_3, resolver: &__MODULE__.do_work/1) 750 | end 751 | end 752 | 753 | def do_work(_), do: :ok 754 | end 755 | ``` 756 | 757 | Field options for fields defined within a batch have one minor requirement difference 758 | from fields not defined within a batch: batched fields MUST always define a resolver function, 759 | regardless of whether or not they define any dependencies. 760 | 761 | ## Batch Field Options 762 | #{NimbleOptions.docs(Options.batch_fields_definition())} 763 | 764 | ## Batch options 765 | #{NimbleOptions.docs(Options.batch_definition())} 766 | """ 767 | defmacro batch(name, options \\ @default_batch_options, do: block) do 768 | prelude = 769 | quote do 770 | Module.put_attribute( 771 | __MODULE__, 772 | :pacer_batch_options, 773 | {unquote(name), 774 | unquote(options) 775 | |> Keyword.put(:on_timeout, :kill_task) 776 | |> Pacer.Workflow.validate_options(Options.batch_definition())} 777 | ) 778 | 779 | if unquote(name) in Module.get_attribute(__MODULE__, :pacer_batches) do 780 | raise Error, """ 781 | Found duplicate batch name `#{unquote(name)}` in graph module #{inspect(__MODULE__)}. 782 | Batch names within a single graph instance must be unique. 783 | """ 784 | else 785 | Module.put_attribute(__MODULE__, :pacer_batches, unquote(name)) 786 | end 787 | end 788 | 789 | postlude = 790 | Macro.postwalk(block, fn ast -> 791 | case ast do 792 | {:field, metadata, [field_name | args]} -> 793 | batched_ast = 794 | quote do 795 | {unquote(:batch), unquote(name), unquote(field_name)} 796 | end 797 | 798 | {:field, metadata, [batched_ast | args]} 799 | 800 | _ -> 801 | ast 802 | end 803 | end) 804 | 805 | quote do 806 | unquote(prelude) 807 | unquote(postlude) 808 | end 809 | end 810 | 811 | @doc """ 812 | The field/2 macro maps fields one-to-one to keys on the struct created via the graph definition. 813 | 814 | Fields must be unique within a graph instance. 815 | 816 | ## Options: 817 | There are specific options that are allowed to be passed in to the field macro, as indicated below: 818 | 819 | #{NimbleOptions.docs(Pacer.Workflow.Options.fields_definition())} 820 | """ 821 | defmacro field(name, options \\ []) do 822 | quote do 823 | Pacer.Workflow.__field__(__MODULE__, unquote(name), unquote(options)) 824 | end 825 | end 826 | 827 | @spec __field__(module(), atom(), Keyword.t()) :: :ok | no_return() 828 | def __field__(module, {:batch, batch_name, field_name}, options) do 829 | opts = validate_options(options, Options.batch_fields_definition()) 830 | 831 | _ = 832 | module 833 | |> Module.get_attribute(:pacer_field_to_batch_mapping) 834 | |> Map.put(field_name, batch_name) 835 | |> tap(fn mapping -> 836 | Module.put_attribute(module, :pacer_field_to_batch_mapping, mapping) 837 | end) 838 | 839 | Module.put_attribute( 840 | module, 841 | :pacer_batch_resolvers, 842 | {batch_name, field_name, Keyword.fetch!(opts, :resolver)} 843 | ) 844 | 845 | Module.put_attribute( 846 | module, 847 | :pacer_docs, 848 | {field_name, batch_name, Keyword.get(opts, :doc, "")} 849 | ) 850 | 851 | Module.put_attribute( 852 | module, 853 | :pacer_batch_dependencies, 854 | {batch_name, field_name, Keyword.get(opts, :dependencies, [])} 855 | ) 856 | 857 | if Keyword.get(opts, :guard) && is_function(Keyword.get(opts, :guard), 1) do 858 | Module.put_attribute( 859 | module, 860 | :pacer_batch_guard_functions, 861 | {batch_name, field_name, Keyword.fetch!(opts, :guard)} 862 | ) 863 | end 864 | 865 | put_field_attributes(module, field_name, opts) 866 | 867 | unless Enum.member?(Module.get_attribute(module, :pacer_graph_vertices), batch_name) do 868 | Module.put_attribute(module, :pacer_graph_vertices, batch_name) 869 | end 870 | end 871 | 872 | def __field__(module, name, options) do 873 | opts = validate_options(options, Options.fields_definition()) 874 | 875 | deps = Keyword.fetch!(opts, :dependencies) 876 | Module.put_attribute(module, :pacer_dependencies, {name, deps}) 877 | 878 | if Keyword.get(opts, :virtual?) do 879 | Module.put_attribute(module, :pacer_virtual_fields, name) 880 | end 881 | 882 | Module.put_attribute(module, :pacer_docs, {name, Keyword.get(opts, :doc, "")}) 883 | 884 | unless deps == [] do 885 | register_and_validate_resolver(module, name, opts) 886 | end 887 | 888 | put_field_attributes(module, name, opts) 889 | 890 | Module.put_attribute(module, :pacer_graph_vertices, name) 891 | end 892 | 893 | defp put_field_attributes(module, field_name, opts) do 894 | :ok = ensure_no_duplicate_fields(module, field_name) 895 | 896 | Module.put_attribute( 897 | module, 898 | :pacer_struct_fields, 899 | {field_name, Keyword.get(opts, :default, %FieldNotSet{})} 900 | ) 901 | 902 | Module.put_attribute(module, :pacer_fields, field_name) 903 | end 904 | 905 | @doc """ 906 | A Depth-First Search to find where is the dependency graph cycle 907 | and then display the cyclic dependencies back to the developer. 908 | """ 909 | @spec find_cycles(Graph.t()) :: nil 910 | def find_cycles(graph), do: find_cycles(graph, Graph.vertices(graph), MapSet.new()) 911 | 912 | defp find_cycles(_, [], _visited), do: nil 913 | 914 | defp find_cycles(graph, [v | vs], visited) do 915 | if v in visited, do: cycle_found(visited) 916 | 917 | find_cycles(graph, Graph.out_neighbors(graph, v), MapSet.put(visited, v)) 918 | find_cycles(graph, vs, visited) 919 | end 920 | 921 | @spec cycle_found(any()) :: no_return() 922 | defp cycle_found(visited) do 923 | message = 924 | case Enum.to_list(visited) do 925 | [reflexive_vertex] -> "Field `#{reflexive_vertex}` depends on itself." 926 | _ -> visited |> Enum.sort() |> Enum.join(", ") 927 | end 928 | 929 | raise Error, """ 930 | Could not sort dependencies. 931 | The following dependencies form a cycle: 932 | 933 | #{message} 934 | """ 935 | end 936 | 937 | @spec __after_compile__(Macro.Env.t(), binary()) :: :ok | no_return() 938 | def __after_compile__(%{module: module} = _env, _) do 939 | _ = validate_dependencies(module) 940 | 941 | :ok 942 | end 943 | 944 | @spec ensure_no_duplicate_fields(module(), atom()) :: :ok | no_return() 945 | defp ensure_no_duplicate_fields(module, field_name) do 946 | if Enum.member?(Module.get_attribute(module, :pacer_fields), field_name) do 947 | raise Error, "Found duplicate field in graph instance for #{inspect(module)}: #{field_name}" 948 | end 949 | 950 | :ok 951 | end 952 | 953 | @spec register_and_validate_resolver(module(), atom(), Keyword.t()) :: :ok | no_return() 954 | defp register_and_validate_resolver(module, name, options) do 955 | case Keyword.fetch(options, :resolver) do 956 | {:ok, resolver} -> 957 | Module.put_attribute(module, :pacer_resolvers, {name, resolver}) 958 | :ok 959 | 960 | :error -> 961 | error_message = """ 962 | Field #{name} in #{inspect(module)} declared at least one dependency, but did not specify a resolver function. 963 | Any field that declares at least one dependency must also declare a resolver function. 964 | 965 | #{@example_graph_message} 966 | """ 967 | 968 | raise Error, error_message 969 | end 970 | end 971 | 972 | @spec validate_options(Keyword.t(), NimbleOptions.t()) :: Keyword.t() 973 | def validate_options(options, schema) do 974 | case NimbleOptions.validate(options, schema) do 975 | {:ok, options} -> 976 | options 977 | 978 | {:error, %NimbleOptions.ValidationError{} = validation_error} -> 979 | raise Error, """ 980 | #{Exception.message(validation_error)} 981 | 982 | #{@example_graph_message} 983 | """ 984 | end 985 | end 986 | 987 | @spec validate_dependencies(module()) :: :ok | no_return() 988 | def validate_dependencies(module) do 989 | deps_set = 990 | :dependencies 991 | |> module.__graph__() 992 | |> Enum.concat(module.__graph__(:batch_dependencies)) 993 | |> Enum.flat_map(fn 994 | {_, deps} -> deps 995 | {_, _, deps} -> deps 996 | end) 997 | |> MapSet.new() 998 | 999 | field_set = MapSet.new(module.__graph__(:fields)) 1000 | 1001 | unless MapSet.subset?(deps_set, field_set) do 1002 | invalid_deps = MapSet.difference(deps_set, field_set) 1003 | 1004 | raise Error, 1005 | """ 1006 | Found at least one invalid dependency in graph definiton for #{inspect(module)} 1007 | Invalid dependencies: #{inspect(Enum.to_list(invalid_deps))} 1008 | """ 1009 | end 1010 | 1011 | :ok 1012 | end 1013 | 1014 | @doc """ 1015 | Takes a struct that has been defined via the `Pacer.Workflow.graph/1` macro. 1016 | `execute` will run/execute all of the resolvers defined in the definition of the 1017 | `graph` macro in an order that ensures all dependencies have been met before 1018 | the resolver runs. 1019 | 1020 | Resolvers that have been defined within batches will be executed in parallel. 1021 | """ 1022 | @spec execute(struct() | module()) :: struct() 1023 | def execute(workflow) when is_atom(workflow), do: execute(struct(workflow)) 1024 | 1025 | def execute(%module{} = workflow) do 1026 | :telemetry.span([:pacer, :workflow], %{workflow: module}, fn -> 1027 | workflow_result = 1028 | :evaluation_order 1029 | |> module.__graph__() 1030 | |> Enum.reduce(workflow, &execute_vertex/2) 1031 | 1032 | {Map.drop(workflow_result, module.__graph__(:virtual_fields)), %{workflow: module}} 1033 | end) 1034 | end 1035 | 1036 | defp execute_vertex(vertex, %module{} = workflow) do 1037 | case module.__graph__(:resolver, vertex) do 1038 | {:field, resolver} when is_function(resolver, 1) -> 1039 | metadata = %{field: vertex, workflow: workflow.__struct__} 1040 | resolver_args = Map.take(workflow, [vertex | module.__graph__(:dependencies, vertex)]) 1041 | 1042 | :telemetry.span([:pacer, :execute_vertex], metadata, fn -> 1043 | result = 1044 | Map.put( 1045 | workflow, 1046 | vertex, 1047 | resolver.(resolver_args) 1048 | ) 1049 | 1050 | {result, metadata} 1051 | end) 1052 | 1053 | {:batch, [{field, resolver}]} -> 1054 | metadata = %{field: field, workflow: workflow.__struct__} 1055 | 1056 | try do 1057 | resolver_args = 1058 | Map.take(workflow, [field | module.__graph__(:batched_field_dependencies, field)]) 1059 | 1060 | :telemetry.span([:pacer, :execute_vertex], metadata, fn -> 1061 | {Map.put( 1062 | workflow, 1063 | field, 1064 | resolver.(resolver_args) 1065 | ), metadata} 1066 | end) 1067 | catch 1068 | _kind, error -> 1069 | if module.__config__(:debug_mode?) do 1070 | Logger.error(""" 1071 | Resolver for #{inspect(module)}.#{vertex}'s resolver returned default. 1072 | Your resolver function failed for #{inspect(error)}. 1073 | 1074 | Returning default value of: #{inspect(Map.get(workflow, field))} 1075 | """) 1076 | end 1077 | 1078 | workflow 1079 | end 1080 | 1081 | {:batch, resolvers} when is_list(resolvers) -> 1082 | run_concurrently(%{resolvers: resolvers, vertex: vertex, workflow: workflow}) 1083 | end 1084 | end 1085 | 1086 | defp run_concurrently(%{resolvers: resolvers, vertex: vertex, workflow: %module{} = workflow}) do 1087 | task_options = module.__graph__(vertex, :options) 1088 | 1089 | parent_pid = self() 1090 | 1091 | user_provided_metadata = Config.fetch_batch_telemetry_options(module) 1092 | 1093 | resolvers 1094 | |> filter_guarded_resolvers(workflow) 1095 | |> Enum.map(fn {field, resolver} -> 1096 | dependencies = module.__graph__(:batched_field_dependencies, field) 1097 | # We want to also pass the current value (which is the default) of the field itself, 1098 | # so that we can use it in the fallback clause 1099 | partial_workflow = Map.take(workflow, [field | dependencies]) 1100 | {field, partial_workflow, resolver} 1101 | end) 1102 | |> Task.async_stream( 1103 | fn {field, partial_workflow, resolver} -> 1104 | metadata = 1105 | %{ 1106 | field: field, 1107 | workflow: module, 1108 | parent_pid: parent_pid 1109 | } 1110 | |> Map.merge(user_provided_metadata) 1111 | 1112 | try do 1113 | :telemetry.span( 1114 | [:pacer, :execute_vertex], 1115 | metadata, 1116 | fn -> 1117 | {{field, resolver.(partial_workflow)}, 1118 | Map.merge(%{parent_pid: parent_pid}, user_provided_metadata)} 1119 | end 1120 | ) 1121 | catch 1122 | _kind, _error -> 1123 | {field, Map.get(partial_workflow, field)} 1124 | end 1125 | end, 1126 | task_options 1127 | ) 1128 | |> Enum.reduce(workflow, fn result, workflow -> 1129 | case result do 1130 | {:ok, {field, resolver_result}} -> 1131 | Map.put(workflow, field, resolver_result) 1132 | 1133 | _ -> 1134 | workflow 1135 | end 1136 | end) 1137 | end 1138 | 1139 | defp filter_guarded_resolvers(resolvers, %module{} = workflow) do 1140 | Enum.filter(resolvers, fn {field, _resolver} -> 1141 | case module.__graph__(:batched_field_guard_functions, field) do 1142 | guard_function when is_function(guard_function, 1) -> 1143 | dependencies = module.__graph__(:batched_field_dependencies, field) 1144 | 1145 | guard_function.(Map.take(workflow, [field | dependencies])) 1146 | 1147 | _ -> 1148 | true 1149 | end 1150 | end) 1151 | end 1152 | end 1153 | --------------------------------------------------------------------------------