├── test ├── test_helper.exs ├── batch_loader_test.exs └── batch_loader │ ├── absinthe │ ├── middleware_test.exs │ └── plugin_test.exs │ ├── cache_test.exs │ ├── cache_store_test.exs │ └── absinthe_test.exs ├── .formatter.exs ├── .travis.yml ├── lib ├── batch_loader.ex └── batch_loader │ ├── absinthe │ ├── plugin.ex │ └── middleware.ex │ ├── cache.ex │ ├── cache_store.ex │ └── absinthe.ex ├── Makefile ├── .gitignore ├── mix.exs ├── CHANGELOG.md ├── mix.lock └── README.md /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | elixir: 3 | - '1.9.1' 4 | otp_release: 5 | - '22.0' 6 | env: 7 | - MIX_ENV=test 8 | script: mix coveralls.travis 9 | -------------------------------------------------------------------------------- /lib/batch_loader.ex: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader do 2 | @moduledoc """ 3 | A struct which is being used for batching. 4 | """ 5 | 6 | @enforce_keys [:item, :batch] 7 | defstruct [:item, :batch, opts: [default_value: nil, callback: nil]] 8 | 9 | def cache_key(batch_loader) do 10 | info = Function.info(batch_loader.batch) 11 | "#{info[:module]}-#{info[:name]}-#{inspect(info[:env])}" 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /test/batch_loader_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoaderTest do 2 | use ExUnit.Case 3 | 4 | describe "cache_key/1" do 5 | test "generates a cache key based on the batch function" do 6 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end} 7 | 8 | result = BatchLoader.cache_key(batch_loader) 9 | 10 | assert result == 11 | "Elixir.BatchLoaderTest--test cache_key/1 generates a cache key based on the batch function/1-fun-0--[]" 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | install: 4 | mix deps.get 5 | 6 | console: 7 | iex -S mix 8 | 9 | docs: 10 | mix docs 11 | 12 | release: 13 | sed -i '' -e 's/{:batch_loader, "~> [0-9\.]*"}/{:batch_loader, "~> $(VERSION)"}/' README.md && \ 14 | sed -i '' -e 's/@version "[0-9\.]*"/@version "$(VERSION)"/' mix.exs && \ 15 | git add --all && \ 16 | git commit -v -m "Release v$(VERSION)" && \ 17 | git tag "v$(VERSION)" 18 | 19 | publish: 20 | git push && \ 21 | git push --tags && \ 22 | mix hex.publish 23 | 24 | format: 25 | mix format && mix credo 26 | 27 | test: format 28 | iex -S mix test --trace 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | batch_loader-*.tar 24 | 25 | -------------------------------------------------------------------------------- /lib/batch_loader/absinthe/plugin.ex: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.Absinthe.Plugin do 2 | @moduledoc """ 3 | Absinthe Plugin which re-runs the delayed resolution and executes batching. 4 | """ 5 | 6 | @behaviour Absinthe.Plugin 7 | 8 | alias BatchLoader.Cache 9 | alias BatchLoader.CacheStore 10 | 11 | def before_resolution(res) do 12 | new_acc = CacheStore.clean(res.acc) 13 | %{res | acc: new_acc} 14 | end 15 | 16 | def after_resolution(res) do 17 | batched_caches = 18 | res.acc 19 | |> CacheStore.unbatched_caches() 20 | |> Enum.map(fn cache -> Cache.batch(cache) end) 21 | 22 | new_acc = CacheStore.replace_caches(res.acc, batched_caches) 23 | %{res | acc: new_acc} 24 | end 25 | 26 | def pipeline(pipeline, res) do 27 | if CacheStore.batched?(res.acc) do 28 | pipeline 29 | else 30 | [Absinthe.Phase.Document.Execution.Resolution | pipeline] 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/batch_loader/cache.ex: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.Cache do 2 | @moduledoc """ 3 | A struct which is used like cache per batch function. 4 | """ 5 | 6 | @enforce_keys [:batch] 7 | defstruct [:batch, items: [], value_by_item: %{}] 8 | 9 | def new(batch_loader) do 10 | %__MODULE__{batch: batch_loader.batch} 11 | end 12 | 13 | def batched?(cache) do 14 | !Enum.any?(cache.items) 15 | end 16 | 17 | def add_item(cache, batch_loader) do 18 | %{cache | items: [batch_loader.item] ++ cache.items} 19 | end 20 | 21 | def batch(cache) do 22 | value_by_item = cache.batch.(cache.items) |> Map.new() 23 | %{cache | value_by_item: value_by_item} 24 | end 25 | 26 | def clean(cache) do 27 | %{cache | items: []} 28 | end 29 | 30 | def value(cache, batch_loader) do 31 | case batch_loader.opts[:callback] do 32 | nil -> cache.value_by_item[batch_loader.item] || batch_loader.opts[:default_value] 33 | callback -> callback.(cache.value_by_item[batch_loader.item]) 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/batch_loader/absinthe/middleware.ex: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.Absinthe.Middleware do 2 | @moduledoc """ 3 | Absinthe Middleware which delays the resolution and then gets the result from CacheStore. 4 | """ 5 | 6 | @behaviour Absinthe.Middleware 7 | 8 | alias BatchLoader.Cache 9 | alias BatchLoader.CacheStore 10 | 11 | def call(%{state: :unresolved} = res, batch_loader) do 12 | new_cache = 13 | res.acc 14 | |> CacheStore.fetch_cache(batch_loader) 15 | |> Cache.add_item(batch_loader) 16 | 17 | new_acc = 18 | res.acc 19 | |> CacheStore.upsert_cache(new_cache) 20 | 21 | new_middleware = [{__MODULE__, batch_loader} | res.middleware] 22 | %{res | state: :suspended, acc: new_acc, middleware: new_middleware} 23 | end 24 | 25 | def call(%{state: :suspended} = res, batch_loader, resolution \\ Absinthe.Resolution) do 26 | result = 27 | res.acc 28 | |> CacheStore.fetch_cache(batch_loader) 29 | |> Cache.value(batch_loader) 30 | 31 | res |> resolution.put_result(result) 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/batch_loader/absinthe/middleware_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.Absinthe.MiddlewareTest do 2 | use ExUnit.Case 3 | 4 | alias BatchLoader.Cache 5 | alias BatchLoader.CacheStore 6 | alias BatchLoader.Absinthe.Middleware 7 | 8 | defmodule DummyResolution do 9 | def put_result(res, result) do 10 | Map.put(res, :result, result) 11 | end 12 | end 13 | 14 | describe "call/2" do 15 | test "adds info for batching to cache and reschedules the execution with the 'suspended' state" do 16 | batch = fn -> nil end 17 | batch_loader = %BatchLoader{item: 1, batch: batch} 18 | res = %{state: :unresolved, middleware: [], acc: %{}} 19 | 20 | result = Middleware.call(res, batch_loader) 21 | 22 | assert result.state == :suspended 23 | assert result.middleware == [{BatchLoader.Absinthe.Middleware, batch_loader}] 24 | cache = %Cache{items: [1], value_by_item: %{}, batch: batch} 25 | assert CacheStore.fetch_cache(result.acc, batch_loader) == cache 26 | end 27 | 28 | test "reads the batched value from the cache" do 29 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end} 30 | cache = %Cache{value_by_item: %{1 => 2}, batch: batch_loader.batch} 31 | res = %{state: :suspended, acc: CacheStore.upsert_cache(%{}, cache)} 32 | 33 | result = Middleware.call(res, batch_loader, DummyResolution) 34 | 35 | assert result.result == 2 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.1.0-beta.6" 5 | 6 | def project do 7 | [ 8 | app: :batch_loader, 9 | version: @version, 10 | elixir: "~> 1.9", 11 | start_permanent: Mix.env() == :prod, 12 | deps: deps(), 13 | description: description(), 14 | package: package(), 15 | docs: docs_config(), 16 | source_url: "https://github.com/exAspArk/batch_loader", 17 | test_coverage: [tool: ExCoveralls] 18 | ] 19 | end 20 | 21 | # Run "mix help compile.app" to learn about applications. 22 | def application do 23 | [ 24 | extra_applications: [:logger] 25 | ] 26 | end 27 | 28 | # Run "mix help deps" to learn about dependencies. 29 | defp deps do 30 | [ 31 | {:absinthe, "~> 1.4.0 or ~> 1.5.0-beta"}, 32 | {:ex_doc, "~> 0.21", only: :dev, runtime: false}, 33 | {:excoveralls, "~> 0.11", only: :test}, 34 | {:credo, "~> 1.1", only: [:dev, :test], runtime: false} 35 | ] 36 | end 37 | 38 | defp description() do 39 | "Powerful tool to avoid N+1 DB or HTTP queries" 40 | end 41 | 42 | defp package do 43 | [ 44 | maintainers: ["exAspArk"], 45 | licenses: ["MIT"], 46 | links: %{"Github" => "https://github.com/exAspArk/batch_loader"} 47 | ] 48 | end 49 | 50 | defp docs_config do 51 | [ 52 | extras: [ 53 | {"README.md", [title: "Overview"]}, 54 | {"CHANGELOG.md", [title: "Changelog"]} 55 | ], 56 | main: "readme" 57 | ] 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/batch_loader/cache_store.ex: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.CacheStore do 2 | @moduledoc """ 3 | A module which operates on the cache store (map). 4 | """ 5 | 6 | alias BatchLoader.Cache 7 | 8 | def fetch_cache(store, batch_loader) do 9 | cache_key = BatchLoader.cache_key(batch_loader) 10 | batch_loader_store(store)[cache_key] || Cache.new(batch_loader) 11 | end 12 | 13 | def clean(store) do 14 | batches = 15 | store 16 | |> batch_loader_store() 17 | |> Enum.map(fn {cache_key, cache} -> {cache_key, Cache.clean(cache)} end) 18 | |> Map.new() 19 | 20 | Map.put(store, BatchLoader, batches) 21 | end 22 | 23 | def replace_caches(store, caches) do 24 | batches = 25 | caches 26 | |> Enum.map(fn cache -> {BatchLoader.cache_key(cache), cache} end) 27 | |> Map.new() 28 | 29 | Map.put(store, BatchLoader, batches) 30 | end 31 | 32 | def unbatched_caches(store) do 33 | store 34 | |> batch_loader_store() 35 | |> Enum.filter(fn {_cache_key, cache} -> !Cache.batched?(cache) end) 36 | |> Enum.map(fn {_cache_key, cache} -> cache end) 37 | end 38 | 39 | def upsert_cache(store, cache) do 40 | cache_key = BatchLoader.cache_key(cache) 41 | batches = batch_loader_store(store) |> Map.put(cache_key, cache) 42 | Map.put(store, BatchLoader, batches) 43 | end 44 | 45 | def batched?(store) do 46 | store 47 | |> batch_loader_store() 48 | |> Enum.all?(fn {_cache_key, cache} -> Cache.batched?(cache) end) 49 | end 50 | 51 | defp batch_loader_store(store) do 52 | store[BatchLoader] || %{} 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /test/batch_loader/absinthe/plugin_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.Absinthe.PluginTest do 2 | use ExUnit.Case 3 | 4 | alias BatchLoader.Cache 5 | alias BatchLoader.CacheStore 6 | alias BatchLoader.Absinthe.Plugin 7 | 8 | describe "pipeline/2" do 9 | test "appends an extra step to the pipeline if not batched yet" do 10 | cache = %Cache{items: [1, 2], batch: fn -> nil end} 11 | res = %{acc: CacheStore.upsert_cache(%{}, cache)} 12 | 13 | result = Plugin.pipeline([:step2], res) 14 | 15 | assert result == [Absinthe.Phase.Document.Execution.Resolution, :step2] 16 | end 17 | end 18 | 19 | describe "after_resolution/1" do 20 | test "runs the batch function and stores the results in the cache" do 21 | batch = fn ids -> Enum.map(ids, &{&1, &1 + 1}) end 22 | batch_loader = %BatchLoader{item: 1, batch: batch} 23 | cache = %Cache{items: [1], batch: batch} 24 | res = %{acc: CacheStore.upsert_cache(%{}, cache)} 25 | 26 | result = Plugin.after_resolution(res) 27 | 28 | assert result.acc 29 | |> CacheStore.fetch_cache(batch_loader) 30 | |> Cache.value(batch_loader) == 2 31 | end 32 | end 33 | 34 | describe "before_resolution/1" do 35 | test "cleans the batched caches" do 36 | batch = fn ids -> Enum.map(ids, &{&1, &1 + 1}) end 37 | cache = %Cache{items: [1], batch: batch} 38 | res = %{acc: CacheStore.upsert_cache(%{}, cache)} 39 | 40 | result = Plugin.before_resolution(res) 41 | 42 | assert result.acc 43 | |> CacheStore.fetch_cache(cache) 44 | |> Map.get(:items) == [] 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | The following are lists of the notable changes included with each release. 4 | This is intended to help keep people informed about notable changes between 5 | versions, as well as provide a rough history. Each item is prefixed with 6 | one of the following labels: `Added`, `Changed`, `Deprecated`, 7 | `Removed`, `Fixed`, `Security`. We also use [Semantic Versioning](http://semver.org) 8 | to manage the versions of this gem so that you can set version constraints properly. 9 | 10 | #### [Unreleased](https://github.com/exAspArk/batch_loader/compare/v0.1.0-beta.6...HEAD) 11 | 12 | * WIP 13 | 14 | #### [v0.1.0-beta.6](https://github.com/exAspArk/batch_loader/compare/v0.1.0-beta.5...v0.1.0-beta.6) – 2019-10-22 15 | 16 | * `Added`: `BatchLoader.Absinthe.load_assoc/4`. 17 | 18 | #### [v0.1.0-beta.5](https://github.com/exAspArk/batch_loader/compare/v0.1.0-beta.4...v0.1.0-beta.5) – 2019-10-16 19 | 20 | * `Fixed`: preloading associations for a list with different Ecto Schemas. 21 | 22 | #### [v0.1.0-beta.4](https://github.com/exAspArk/batch_loader/compare/v0.1.0-beta.3...v0.1.0-beta.4) – 2019-10-08 23 | 24 | * `Fixed`: detecting unique batch functions with `BatchLoader.cache_key/1`. 25 | 26 | #### [v0.1.0-beta.3](https://github.com/exAspArk/batch_loader/compare/v0.1.0-beta.2...v0.1.0-beta.3) – 2019-10-07 27 | 28 | * `Added`: support for `absinthe` dependency version `~> 1.4.0`. 29 | 30 | #### [v0.1.0-beta.2](https://github.com/exAspArk/batch_loader/compare/v0.1.0-beta.1...v0.1.0-beta.2) – 2019-10-07 31 | 32 | * `Added`: `BatchLoader.Absinthe.resolve_assoc/2` and `BatchLoader.Absinthe.preload_assoc/3`. 33 | 34 | #### [v0.1.0-beta.1](https://github.com/exAspArk/batch_loader/compare/7a303cefa55bd5e8d22ae19e6e6c537808fd70a0...v0.1.0-beta.1) – 2019-10-06 35 | 36 | * `Added`: initial functional version. 37 | -------------------------------------------------------------------------------- /test/batch_loader/cache_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.CacheTest do 2 | use ExUnit.Case 3 | 4 | alias BatchLoader.Cache 5 | 6 | describe "new/1" do 7 | test "generates a cache struct based on the BatchLoader" do 8 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end} 9 | 10 | cache = Cache.new(batch_loader) 11 | 12 | assert cache.items == [] 13 | assert cache.value_by_item == %{} 14 | assert cache.batch == batch_loader.batch 15 | end 16 | end 17 | 18 | describe "batched?/1" do 19 | test "returns true if there are no items for batching" do 20 | cache = %Cache{items: [], batch: fn -> nil end} 21 | 22 | assert Cache.batched?(cache) 23 | end 24 | 25 | test "returns false if there are items for batching" do 26 | cache = %Cache{items: [1], batch: fn -> nil end} 27 | 28 | assert !Cache.batched?(cache) 29 | end 30 | end 31 | 32 | describe "add_item/2" do 33 | test "adds an item from a BatchLoader struct" do 34 | cache = %Cache{items: [1], batch: fn -> nil end} 35 | batch_loader = %BatchLoader{item: 2, batch: fn -> nil end} 36 | 37 | result = Cache.add_item(cache, batch_loader) 38 | 39 | assert result.items == [2, 1] 40 | end 41 | end 42 | 43 | describe "batch/1" do 44 | test "calls a batch function and stores value_by_item" do 45 | cache = %Cache{ 46 | items: [1], 47 | batch: fn items -> Enum.map(items, &{&1, &1 + 1}) end 48 | } 49 | 50 | result = Cache.batch(cache) 51 | 52 | assert result.value_by_item == %{1 => 2} 53 | end 54 | end 55 | 56 | describe "clean/1" do 57 | test "returns a new cache with empty items" do 58 | cache = %Cache{items: [1], batch: fn -> nil end} 59 | 60 | result = Cache.clean(cache) 61 | 62 | assert result.items == [] 63 | end 64 | end 65 | 66 | describe "value/2" do 67 | test "reads a batched value based on the BatchLoader struct" do 68 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end} 69 | cache = %Cache{value_by_item: %{1 => 2}, batch: batch_loader.batch} 70 | 71 | result = Cache.value(cache, batch_loader) 72 | 73 | assert result == 2 74 | end 75 | 76 | test "fallbacks to BatchLoader's default_value if batched value couldn't be found" do 77 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end, opts: [default_value: 2]} 78 | cache = %Cache{value_by_item: %{}, batch: batch_loader.batch} 79 | 80 | result = Cache.value(cache, batch_loader) 81 | 82 | assert result == 2 83 | end 84 | 85 | test "runs a callback if it exists by using the batched value" do 86 | callback = fn i -> i + i end 87 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end, opts: [callback: callback]} 88 | cache = %Cache{value_by_item: %{1 => 2}, batch: batch_loader.batch} 89 | 90 | result = Cache.value(cache, batch_loader) 91 | 92 | assert result == 4 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /test/batch_loader/cache_store_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.CacheStoreTest do 2 | use ExUnit.Case 3 | 4 | alias BatchLoader.Cache 5 | alias BatchLoader.CacheStore 6 | 7 | describe "fetch_cache/2" do 8 | test "fetches the existing cache value from the store based on the BatchLoader struct" do 9 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end} 10 | cache = Cache.new(batch_loader) 11 | store = %{BatchLoader => %{BatchLoader.cache_key(batch_loader) => cache}} 12 | 13 | result = CacheStore.fetch_cache(store, batch_loader) 14 | 15 | assert result == cache 16 | end 17 | 18 | test "returns a new cache if it doesn't exist in the store" do 19 | batch_loader = %BatchLoader{item: 1, batch: fn -> nil end} 20 | store = %{} 21 | 22 | result = CacheStore.fetch_cache(store, batch_loader) 23 | 24 | assert result == Cache.new(batch_loader) 25 | end 26 | end 27 | 28 | describe "clean/1" do 29 | test "removes items for batching from the store" do 30 | batch_loader1 = %BatchLoader{item: 1, batch: fn -> nil end} 31 | cache_key1 = BatchLoader.cache_key(batch_loader1) 32 | batch_loader2 = %BatchLoader{item: 2, batch: fn -> nil end} 33 | cache_key2 = BatchLoader.cache_key(batch_loader2) 34 | 35 | store = %{ 36 | BatchLoader => %{ 37 | cache_key1 => %Cache{items: [1], batch: batch_loader1.batch}, 38 | cache_key2 => %Cache{items: [2], batch: batch_loader2.batch} 39 | } 40 | } 41 | 42 | result = CacheStore.clean(store) 43 | 44 | assert result[BatchLoader][cache_key1].items == [] 45 | assert result[BatchLoader][cache_key2].items == [] 46 | end 47 | end 48 | 49 | describe "replace_caches/2" do 50 | test "replaces caches in the store" do 51 | cache1 = %Cache{batch: fn -> nil end} 52 | cache_key1 = BatchLoader.cache_key(cache1) 53 | cache2 = %Cache{batch: fn -> nil end} 54 | cache_key2 = BatchLoader.cache_key(cache2) 55 | 56 | result = CacheStore.replace_caches(%{}, [cache1, cache2]) 57 | 58 | assert result[BatchLoader] == %{cache_key1 => cache1, cache_key2 => cache2} 59 | end 60 | end 61 | 62 | describe "upsert_cache/2" do 63 | test "adds a new cache to the store" do 64 | cache = %Cache{items: [1], batch: fn -> nil end} 65 | cache_key = BatchLoader.cache_key(cache) 66 | 67 | result = CacheStore.upsert_cache(%{}, cache) 68 | 69 | assert result[BatchLoader] == %{cache_key => cache} 70 | end 71 | 72 | test "updates an exisitng cache in the store" do 73 | prev_cache = %Cache{items: [1], batch: fn -> nil end} 74 | cache = %Cache{items: [2], batch: fn -> nil end} 75 | cache_key = BatchLoader.cache_key(cache) 76 | store = %{BatchLoader => %{cache_key => prev_cache}} 77 | 78 | result = CacheStore.upsert_cache(store, cache) 79 | 80 | assert result[BatchLoader] == %{cache_key => cache} 81 | end 82 | end 83 | 84 | describe "batched?/1" do 85 | test "returns true if all caches were batched" do 86 | cache = %Cache{items: [], batch: fn -> nil end} 87 | cache_key = BatchLoader.cache_key(cache) 88 | store = %{BatchLoader => %{cache_key => cache}} 89 | 90 | assert CacheStore.batched?(store) 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "absinthe": {:hex, :absinthe, "1.5.0-beta.2", "960c7c4e7aec89ad951048ac374f527c34e3a8cad2053bad5e57bece13ab36b4", [:mix], [{:dataloader, "~> 1.0.0", [hex: :dataloader, repo: "hexpm", optional: true]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, 3 | "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, 4 | "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, 5 | "credo": {:hex, :credo, "1.1.4", "c2f3b73c895d81d859cec7fcee7ffdb972c595fd8e85ab6f8c2adbf01cf7c29c", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, 6 | "earmark": {:hex, :earmark, "1.4.1", "07bb382826ee8d08d575a1981f971ed41bd5d7e86b917fd012a93c51b5d28727", [:mix], [], "hexpm"}, 7 | "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, 8 | "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, 9 | "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, 10 | "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, 11 | "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, 12 | "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, 13 | "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, 14 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, 15 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, 16 | "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, 17 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, 18 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, 19 | "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"}, 20 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, 21 | } 22 | -------------------------------------------------------------------------------- /test/batch_loader/absinthe_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.AbsintheTest do 2 | use ExUnit.Case 3 | 4 | defmodule DummyRepo do 5 | @preloaded_assoc %{object: :bar} 6 | 7 | def preload([object], assoc, _opts \\ []) do 8 | [%{object | assoc => @preloaded_assoc}] 9 | end 10 | end 11 | 12 | describe "for/2" do 13 | test "creates a BatchLoader and returns the BatchLoader.Absinthe.Middleware" do 14 | batch = fn _ -> nil end 15 | batch_loader = %BatchLoader{item: 1, batch: batch, opts: [default_value: {:ok, nil}]} 16 | 17 | result = BatchLoader.Absinthe.for(1, batch) 18 | 19 | assert result == {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} 20 | end 21 | 22 | test "creates a BatchLoader with a callback" do 23 | batch = fn -> nil end 24 | callback = fn -> nil end 25 | 26 | batch_loader = %BatchLoader{ 27 | item: 1, 28 | batch: batch, 29 | opts: [default_value: {:ok, nil}, callback: callback] 30 | } 31 | 32 | result = BatchLoader.Absinthe.for(1, batch, callback: callback) 33 | 34 | assert result == {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} 35 | end 36 | end 37 | 38 | describe "resolve_assoc/2" do 39 | test "returns a resolve function which returns a middleware with BatchLoader" do 40 | object = %{object: :foo, assoc: nil} 41 | preloaded_assoc = %{object: :bar} 42 | 43 | resolve = BatchLoader.Absinthe.resolve_assoc(:assoc, repo: DummyRepo) 44 | 45 | middleware = resolve.(object, nil, nil) 46 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} = middleware 47 | assert batch_loader.item == object 48 | assert batch_loader.opts == [default_value: {:ok, nil}, preload_opts: [], repo: DummyRepo] 49 | assert batch_loader.batch 50 | assert batch_loader.batch.([object]) == [{object, {:ok, preloaded_assoc}}] 51 | end 52 | 53 | test "returns a resolve function with a middleware with a batch function which preload assocs" do 54 | object1 = %{__struct__: :foo1, object: :foo1, assoc: nil} 55 | object2 = %{__struct__: :foo2, object: :foo2, assoc: nil} 56 | preloaded_assoc = %{object: :bar} 57 | 58 | resolve = BatchLoader.Absinthe.resolve_assoc(:assoc, repo: DummyRepo) 59 | 60 | middleware = resolve.(object1, nil, nil) 61 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} = middleware 62 | 63 | assert batch_loader.batch.([object1, object2]) == [ 64 | {object1, {:ok, preloaded_assoc}}, 65 | {object2, {:ok, preloaded_assoc}} 66 | ] 67 | end 68 | end 69 | 70 | describe "preloaded_assoc/2" do 71 | test "returns a middleware with BatchLoader" do 72 | object = %{object: :foo, assoc: nil} 73 | preloaded_object = %{object: :foo, assoc: %{object: :bar}} 74 | callback = fn preloaded_obj -> preloaded_obj end 75 | 76 | result = BatchLoader.Absinthe.preload_assoc(object, :assoc, callback, repo: DummyRepo) 77 | 78 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} = result 79 | assert batch_loader.item == object 80 | assert batch_loader.batch 81 | assert batch_loader.batch.([object]) == [{object, preloaded_object}] 82 | 83 | assert batch_loader.opts == [ 84 | default_value: {:ok, nil}, 85 | preload_opts: [], 86 | repo: DummyRepo, 87 | callback: callback 88 | ] 89 | end 90 | end 91 | 92 | describe "loaded_assoc/2" do 93 | test "returns a middleware with BatchLoader" do 94 | object = %{object: :foo, assoc: nil} 95 | loaded_object = %{object: :bar} 96 | callback = fn loaded_obj -> loaded_obj end 97 | 98 | result = BatchLoader.Absinthe.load_assoc(object, :assoc, callback, repo: DummyRepo) 99 | 100 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} = result 101 | assert batch_loader.item == object 102 | assert batch_loader.batch 103 | assert batch_loader.batch.([object]) == [{object, loaded_object}] 104 | 105 | assert batch_loader.opts == [ 106 | default_value: {:ok, nil}, 107 | preload_opts: [], 108 | repo: DummyRepo, 109 | callback: callback 110 | ] 111 | end 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /lib/batch_loader/absinthe.ex: -------------------------------------------------------------------------------- 1 | defmodule BatchLoader.Absinthe do 2 | @moduledoc """ 3 | A module which integrates BatchLoader with Absinthe. 4 | """ 5 | 6 | @default_value {:ok, nil} 7 | @default_opts_for [default_value: @default_value] 8 | @default_opts_resolve_assoc [default_value: @default_value, repo: nil, preload_opts: []] 9 | 10 | @doc """ 11 | Creates a BatchLoader struct and calls the BatchLoader.Absinthe.Middleware, which will batch all collected items. 12 | 13 | ## Example 14 | 15 | field :user, :user_type do 16 | resolve(fn post, _, _ -> 17 | BatchLoader.Absinthe.for(post.user_id, fn user_ids -> 18 | Repo.all(from u in User, where: u.id in ^user_ids) 19 | |> Enum.map(fn user -> {user.id, {:ok, user}} end) 20 | end) 21 | end) 22 | end 23 | """ 24 | def for(item, batch, options \\ @default_opts_for) do 25 | opts = Keyword.merge(@default_opts_for, options) 26 | batch_loader = %BatchLoader{item: item, batch: batch, opts: opts} 27 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} 28 | end 29 | 30 | @doc """ 31 | Creates a resolve function, which creates a BatchLoader struct and calls the BatchLoader.Absinthe.Middleware, which return an Ecto association. 32 | 33 | ## Example 34 | 35 | field :user, :user_type, resolve: BatchLoader.Absinthe.resolve_assoc(:user) 36 | """ 37 | def resolve_assoc(assoc, options \\ @default_opts_resolve_assoc) do 38 | opts = Keyword.merge(@default_opts_resolve_assoc, options) 39 | 40 | batch = fn items -> 41 | preloaded_item_by_item = preloaded_item_by_item(items, assoc, opts) 42 | 43 | items 44 | |> Enum.map(fn item -> 45 | preloaded_item = preloaded_item_by_item[item] 46 | {item, {:ok, Map.get(preloaded_item, assoc)}} 47 | end) 48 | end 49 | 50 | fn item, _args, _resolution -> 51 | batch_loader = %BatchLoader{item: item, batch: batch, opts: opts} 52 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} 53 | end 54 | end 55 | 56 | @doc """ 57 | Creates a BatchLoader struct and calls the BatchLoader.Absinthe.Middleware, which will preload an Ecto association. 58 | 59 | ## Example 60 | 61 | field :title, :string do 62 | resolve(fn post, _, _ -> 63 | BatchLoader.Absinthe.preload_assoc(post, :user, fn post_with_user -> 64 | {:ok, "\#{post_with_user.title} - \#{post_with_user.user.name}"} 65 | end) 66 | end) 67 | end 68 | """ 69 | def preload_assoc(item, assoc, callback, options \\ @default_opts_resolve_assoc) do 70 | opts = 71 | @default_opts_resolve_assoc 72 | |> Keyword.merge(options) 73 | |> Keyword.merge(callback: callback) 74 | 75 | batch = fn items -> 76 | preloaded_item_by_item = preloaded_item_by_item(items, assoc, opts) 77 | 78 | items 79 | |> Enum.map(fn item -> 80 | preloaded_item = preloaded_item_by_item[item] 81 | {item, preloaded_item} 82 | end) 83 | end 84 | 85 | batch_loader = %BatchLoader{item: item, batch: batch, opts: opts} 86 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} 87 | end 88 | 89 | @doc """ 90 | Creates a BatchLoader struct and calls the BatchLoader.Absinthe.Middleware, which will load an Ecto association. 91 | 92 | ## Example 93 | 94 | field :author, :string do 95 | resolve(fn post, _, _ -> 96 | BatchLoader.Absinthe.load_assoc(post, :user, fn user -> 97 | {:ok, user.name} 98 | end) 99 | end) 100 | end 101 | """ 102 | def load_assoc(item, assoc, callback, options \\ @default_opts_resolve_assoc) do 103 | opts = 104 | @default_opts_resolve_assoc 105 | |> Keyword.merge(options) 106 | |> Keyword.merge(callback: callback) 107 | 108 | batch = fn items -> 109 | preloaded_item_by_item = preloaded_item_by_item(items, assoc, opts) 110 | 111 | items 112 | |> Enum.map(fn item -> 113 | preloaded_item = preloaded_item_by_item[item] 114 | {item, Map.get(preloaded_item, assoc)} 115 | end) 116 | end 117 | 118 | batch_loader = %BatchLoader{item: item, batch: batch, opts: opts} 119 | {:middleware, BatchLoader.Absinthe.Middleware, batch_loader} 120 | end 121 | 122 | defp preloaded_item_by_item(items, assoc, opts) do 123 | repo = opts[:repo] || default_repo() 124 | 125 | items 126 | |> Enum.group_by(&Map.get(&1, :__struct__)) 127 | |> Map.values() 128 | |> Enum.flat_map(fn homogeneous_items -> 129 | preloaded_items = repo.preload(homogeneous_items, assoc, opts[:preload_opts]) 130 | 131 | homogeneous_items 132 | |> Enum.with_index() 133 | |> Enum.map(fn {item, index} -> 134 | preloaded_item = Enum.at(preloaded_items, index) 135 | {item, preloaded_item} 136 | end) 137 | end) 138 | |> Map.new() 139 | end 140 | 141 | defp default_repo, do: Application.get_env(:batch_loader, :default_repo) 142 | end 143 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BatchLoader 2 | 3 | [![Build Status](https://img.shields.io/travis/exAspArk/batch_loader.svg)](https://travis-ci.org/exAspArk/batch_loader) 4 | [![Coverage Status](https://coveralls.io/repos/github/exAspArk/batch_loader/badge.svg)](https://coveralls.io/github/exAspArk/batch_loader) 5 | [![Latest Version](https://img.shields.io/hexpm/v/batch_loader.svg)](https://hex.pm/packages/batch_loader) 6 | 7 | This package provides a generic lazy batching mechanism to avoid N+1 DB queries, HTTP queries, etc. 8 | 9 | ## Contents 10 | 11 | * [Highlights](#highlights) 12 | * [Usage](#usage) 13 | * [Ecto Resolve Association](#ecto-resolve-association) 14 | * [Ecto Load Association](#ecto-load-association) 15 | * [Ecto Preload Association](#ecto-preload-association) 16 | * [DIY Batching](#diy-batching) 17 | * [Customization](#customization) 18 | * [Installation](#installation) 19 | * [Testing](#testing) 20 | 21 | ## Highlights 22 | 23 | * Generic utility to avoid N+1 DB queries, HTTP requests, etc. 24 | * Adapted Elixir implementation of the battle-tested tools like [Haskell Haxl](https://github.com/facebook/Haxl), [JS DataLoader](https://github.com/graphql/dataloader), [Ruby BatchLoader](https://github.com/exaspark/batch-loader). 25 | * Convenient and flexible integration with Ecto Schemas. 26 | * Allows inlining the code without defining extra named functions, unlike [Absinthe Batch](https://hexdocs.pm/absinthe/Absinthe.Middleware.Batch.html). 27 | * Allows using batching with any data sources, not just DB, unlike [Absinthe DataLoader](https://hexdocs.pm/dataloader/Dataloader.html). 28 | 29 | ## Usage 30 | 31 | Let's imagine that we have a `Post` GraphQL type defined with [Absinthe](https://github.com/absinthe-graphql/absinthe): 32 | 33 | ```elixir 34 | defmodule MyApp.PostType do 35 | use Absinthe.Schema.Notation 36 | alias MyApp.Repo 37 | 38 | object :post_type do 39 | field :title, :string 40 | 41 | field :user, :user_type do 42 | resolve(fn post, _, _ -> 43 | user = post |> Ecto.assoc(:user) |> Repo.one() # N+1 DB requests 44 | {:ok, user} 45 | end) 46 | end 47 | end 48 | end 49 | ``` 50 | 51 | This will produce N+1 DB requests if we send this GraphQL request: 52 | 53 | ```gql 54 | query { 55 | posts { 56 | title 57 | user { # N+1 request per each post 58 | name 59 | } 60 | } 61 | } 62 | ``` 63 | 64 | ### Ecto Resolve Association 65 | 66 | We can get rid of the N+1 DB requests by loading all `Users` for all `Posts` at once in. 67 | All we have to do is to use `resolve_assoc` function by passing the Ecto associations name: 68 | 69 | ```elixir 70 | import BatchLoader.Absinthe, only: [resolve_assoc: 1] 71 | 72 | field :user, :user_type, resolve: resolve_assoc(:user) 73 | ``` 74 | 75 | Set the default `repo` in your `config.exs` file: 76 | 77 | ```elixir 78 | config :batch_loader, :default_repo, MyApp.Repo 79 | ``` 80 | 81 | And finally, add `BatchLoader.Absinthe.Plugin` plugin to the GraphQL schema. 82 | This will allow to lazily collect information about all users which need to be loaded and then batch them all together: 83 | 84 | ```elixir 85 | defmodule MyApp.Schema do 86 | use Absinthe.Schema 87 | import_types MyApp.PostType 88 | 89 | def plugins do 90 | [BatchLoader.Absinthe.Plugin] ++ Absinthe.Plugin.defaults() 91 | end 92 | end 93 | ``` 94 | 95 | ### Ecto Load Association 96 | 97 | You can use `load_assoc` to load Ecto associations in the existing schema: 98 | 99 | ```elixir 100 | import BatchLoader.Absinthe, only: [load_assoc: 3] 101 | 102 | field :author, :string do 103 | resolve(fn post, _, _ -> 104 | load_assoc(post, :user, fn user -> 105 | {:ok, user.name} 106 | end) 107 | end) 108 | end 109 | ``` 110 | 111 | ### Ecto Preload Association 112 | 113 | You can use `preload_assoc` to preload Ecto associations in the existing schema: 114 | 115 | ```elixir 116 | import BatchLoader.Absinthe, only: [preload_assoc: 3] 117 | 118 | field :title, :string do 119 | resolve(fn post, _, _ -> 120 | preload_assoc(post, :user, fn post_with_user -> 121 | {:ok, "#{post_with_user.title} - #{post_with_user.user.name}"} 122 | end) 123 | end) 124 | end 125 | ``` 126 | 127 | ### DIY Batching 128 | 129 | You can also use `BatchLoader` to batch in the `resolve` function manually, for example, to fix N+1 HTTP requests: 130 | 131 | ```elixir 132 | field :user, :user_type do 133 | resolve(fn post, _, _ -> 134 | BatchLoader.Absinthe.for(post.user_id, &resolved_users_by_user_ids/1) 135 | end) 136 | end 137 | 138 | def resolved_users_by_user_ids(user_ids) do 139 | MyApp.HttpClient.users(user_ids) # load all users at once 140 | |> Enum.map(fn user -> {user.id, {:ok, user}} end) # return "{user.id, result}" tuples 141 | end 142 | ``` 143 | 144 | Alternatively, you can simply inline the batch function: 145 | 146 | ```elixir 147 | field :user, :user_type do 148 | resolve(fn post, _, _ -> 149 | BatchLoader.Absinthe.for(post.user_id, fn user_ids -> 150 | MyApp.HttpClient.users(user_ids) 151 | |> Enum.map(fn user -> {user.id, {:ok, user}} end) 152 | end) 153 | end) 154 | end 155 | ``` 156 | 157 | ### Customization 158 | 159 | * To specify default resolve Absinthe values: 160 | 161 | ```elixir 162 | BatchLoader.Absinthe.for(post.user_id, &resolved_users_by_user_ids/1, default_value: {:error, "NOT FOUND"}) 163 | ``` 164 | 165 | * To use custom callback function: 166 | 167 | ```elixir 168 | BatchLoader.Absinthe.for(post.user_id, &users_by_user_ids/1, callback: fn user -> 169 | {:ok, user.name} 170 | end) 171 | ``` 172 | 173 | * To use custom Ecto repos: 174 | 175 | ```elixir 176 | BatchLoader.Absinthe.resolve_assoc(:user, repo: AnotherRepo) 177 | BatchLoader.Absinthe.preload_assoc(post, :user, &callback/1, repo: AnotherRepo) 178 | ``` 179 | 180 | * To pass custom options to `Ecto.Repo.preload`: 181 | 182 | ```elixir 183 | BatchLoader.Absinthe.resolve_assoc(:user, preload_opts: [prefix: nil]) 184 | BatchLoader.Absinthe.preload_assoc(post, :user, &callback/1, preload_opts: [prefix: nil]) 185 | ``` 186 | 187 | ## Installation 188 | 189 | Add `batch_loader` to your list of dependencies in `mix.exs`: 190 | 191 | ```elixir 192 | def deps do 193 | [ 194 | {:batch_loader, "~> 0.1.0-beta.6"} 195 | ] 196 | end 197 | ``` 198 | 199 | ## Testing 200 | 201 | ```ex 202 | make install 203 | make test 204 | ``` 205 | --------------------------------------------------------------------------------