├── .circleci └── config.yml ├── .formatter.exs ├── .gitignore ├── LICENSE ├── README.md ├── bench ├── basic_operations.exs ├── full_bench.exs └── propagation.exs ├── lib ├── benchmark_helper.ex ├── delta_crdt.ex └── delta_crdt │ ├── aw_lww_map.ex │ ├── causal_crdt.ex │ └── storage.ex ├── mix.exs ├── mix.lock └── test ├── aw_lww_map_property_test.exs ├── aw_lww_map_test.exs ├── causal_crdt_test.exs ├── delta_subscriber_test.exs ├── support ├── awlww_map_property.ex └── memory_storage.ex └── test_helper.exs /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Elixir CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-elixir/ for more details 4 | version: 2 5 | jobs: 6 | build: 7 | docker: 8 | # specify the version here 9 | - image: circleci/elixir:1.7 10 | 11 | working_directory: ~/repo 12 | environment: 13 | MIX_ENV: test 14 | steps: 15 | - checkout 16 | - run: mix local.hex --force 17 | - run: mix local.rebar --force 18 | 19 | - run: mix deps.get 20 | - run: mix compile --warnings-as-errors 21 | - run: mix test 22 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | delta_crdt-*.tar 24 | 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Derek Kraan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeltaCrdt 2 | 3 | [![Hex pm](http://img.shields.io/hexpm/v/delta_crdt.svg?style=flat)](https://hex.pm/packages/delta_crdt) [![CircleCI badge](https://circleci.com/gh/derekkraan/delta_crdt_ex.svg?style=svg)](https://circleci.com/gh/derekkraan/delta_crdt_ex) 4 | 5 | DeltaCrdt implements a key/value store using concepts from Delta CRDTs, and relies on [`MerkleMap`](https://github.com/derekkraan/merkle_map) for efficient synchronization. 6 | 7 | There is a (slightly out of date) [introductory blog post](https://moosecode.nl/blog/how_deltacrdt_can_help_write_distributed_elixir_applications) and the (very much up to date) official documentation on [hexdocs.pm](https://hexdocs.pm/delta_crdt) is also very good. 8 | 9 | The following papers have been used to implement this library: 10 | - [`Delta State Replicated Data Types – Almeida et al. 2016`](https://arxiv.org/pdf/1603.01529.pdf) 11 | - [`Efficient Synchronization of State-based CRDTs – Enes et al. 2018`](https://arxiv.org/pdf/1803.02750.pdf) 12 | 13 | ## Usage 14 | 15 | Documentation can be found on [hexdocs.pm](https://hexdocs.pm/delta_crdt). 16 | 17 | Here's a short example to illustrate adding an entry to a map: 18 | 19 | ```elixir 20 | # start 2 Delta CRDTs 21 | {:ok, crdt1} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap) 22 | {:ok, crdt2} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap) 23 | 24 | # make them aware of each other 25 | DeltaCrdt.set_neighbours(crdt1, [crdt2]) 26 | 27 | # show the initial value 28 | DeltaCrdt.read(crdt1) 29 | %{} 30 | 31 | # add a key/value in crdt1 32 | DeltaCrdt.put(crdt1, "CRDT", "is magic!") 33 | DeltaCrdt.put(crdt1, "magic", "is awesome!") 34 | 35 | # read it after it has been replicated to crdt2 36 | DeltaCrdt.read(crdt2) 37 | %{"CRDT" => "is magic!", "magic" => "is awesome!"} 38 | 39 | # get only a subset of keys 40 | DeltaCrdt.take(crdt2, ["magic"]) 41 | %{"magic" => "is awesome!"} 42 | 43 | # get one value 44 | DeltaCrdt.get(crdt2, "magic") 45 | "is awesome!" 46 | 47 | # Other map-like functions are available, see the documentation for details. 48 | ``` 49 | 50 | ⚠️ **Use atoms carefully** : Any atom contained in a key or value will be replicated across all nodes, and will never be garbage collected by the BEAM. 51 | 52 | ## Telemetry metrics 53 | 54 | DeltaCrdt publishes the metric `[:delta_crdt, :sync, :done]`. 55 | 56 | ## Installation 57 | 58 | The package can be installed by adding `delta_crdt` to your list of dependencies in `mix.exs`: 59 | 60 | ```elixir 61 | def deps do 62 | [ 63 | {:delta_crdt, "~> 0.6.3"} 64 | ] 65 | end 66 | ``` 67 | -------------------------------------------------------------------------------- /bench/basic_operations.exs: -------------------------------------------------------------------------------- 1 | setup_crdt = fn number_of_items -> 2 | {:ok, crdt1} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap) 3 | 4 | 1..number_of_items |> Enum.each(fn x -> DeltaCrdt.mutate(crdt1, :add, [x, x]) end) 5 | 6 | crdt1 7 | end 8 | 9 | trace = fn -> 10 | crdt = setup_crdt.(1000) 11 | 12 | # TRACING: 13 | :fprof.trace(:start, procs: [crdt]) 14 | # # add_and_remove.(1000).() 15 | Enum.each(1..1000, fn x -> 16 | DeltaCrdt.mutate(crdt, :add, ["key#{x}", "value"]) 17 | end) 18 | 19 | :fprof.trace(:stop) 20 | # 21 | :fprof.profile() 22 | :fprof.analyse(dest: 'perf', cols: 120) 23 | end 24 | 25 | bench = fn -> 26 | Benchee.run( 27 | %{ 28 | "read" => fn crdt -> DeltaCrdt.read(crdt) end, 29 | "add" => fn crdt -> DeltaCrdt.mutate(crdt, :add, ["key4", "value"]) end, 30 | "update" => fn crdt -> DeltaCrdt.mutate(crdt, :add, [10, 12]) end, 31 | "remove" => fn crdt -> DeltaCrdt.mutate(crdt, :remove, [10]) end 32 | }, 33 | inputs: %{ 34 | "with 1000" => setup_crdt.(1000), 35 | "with 10_000" => setup_crdt.(10_000) 36 | }, 37 | before_each: fn crdt -> 38 | DeltaCrdt.mutate(crdt, :add, [10, 10]) 39 | DeltaCrdt.mutate(crdt, :remove, ["key4"]) 40 | crdt 41 | end 42 | ) 43 | end 44 | 45 | bench.() 46 | -------------------------------------------------------------------------------- /bench/full_bench.exs: -------------------------------------------------------------------------------- 1 | do_test = fn number -> 2 | bench_pid = self() 3 | {:ok, c1} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap, sync_interval: 20, max_sync_size: 500) 4 | 5 | {:ok, c2} = 6 | DeltaCrdt.start_link(DeltaCrdt.AWLWWMap, 7 | on_diffs: {Kernel, :send, [bench_pid]}, 8 | subscribe_updates: {:diffs, bench_pid}, 9 | sync_interval: 20, 10 | max_sync_size: 500 11 | ) 12 | 13 | DeltaCrdt.set_neighbours(c1, [c2]) 14 | DeltaCrdt.set_neighbours(c2, [c1]) 15 | 16 | Enum.each(1..number, fn x -> 17 | DeltaCrdt.mutate(c1, :add, [x, x], 60_000) 18 | end) 19 | 20 | wait_loop = fn next_loop -> 21 | receive do 22 | diffs when is_list(diffs) -> 23 | # Enum.each(diffs, fn diff -> IO.inspect(diff) end) 24 | 25 | Enum.any?(diffs, fn 26 | {:add, ^number, ^number} -> true 27 | {:remove, ^number} -> true 28 | _ -> false 29 | end) 30 | |> if do 31 | nil 32 | else 33 | next_loop.(next_loop) 34 | end 35 | after 36 | 60_000 -> raise "timed out" 37 | end 38 | end 39 | 40 | wait_loop.(wait_loop) 41 | 42 | Enum.each(1..number, fn x -> 43 | DeltaCrdt.mutate(c1, :remove, [x], 60_000) 44 | end) 45 | 46 | wait_loop.(wait_loop) 47 | 48 | Process.exit(c1, :normal) 49 | Process.exit(c2, :normal) 50 | end 51 | 52 | Benchee.run(%{"add and remove" => do_test}, 53 | # inputs: %{10 => 10, 100 => 100, 500 => 500, 1000 => 1000, 5000 => 5000, 10_000 => 10_000} 54 | inputs: %{ 55 | 10 => 10, 56 | 100 => 100, 57 | 1000 => 1000, 58 | 10_000 => 10_000, 59 | 20_000 => 20_000, 60 | 30_000 => 30_000 61 | } 62 | # formatters: [Benchee.Formatters.HTML, Benchee.Formatters.Console] 63 | ) 64 | -------------------------------------------------------------------------------- /bench/propagation.exs: -------------------------------------------------------------------------------- 1 | defmodule BenchRecorder do 2 | use GenServer 3 | 4 | def subscribe_to(msg) do 5 | GenServer.call(__MODULE__, {:set_pid_msg, self(), msg}) 6 | end 7 | 8 | def start_link() do 9 | GenServer.start_link(__MODULE__, nil, name: __MODULE__) 10 | end 11 | 12 | def init(nil), do: {:ok, {nil, nil}} 13 | 14 | def on_diffs(diffs) do 15 | send(__MODULE__, {:diffs, diffs}) 16 | end 17 | 18 | def handle_info({:diffs, _diffs}, {nil, nil}) do 19 | {:noreply, {nil, nil}} 20 | end 21 | 22 | def handle_info({:diffs, diffs}, {pid, msg}) do 23 | if Enum.member?(diffs, msg) do 24 | send(pid, msg) 25 | {:noreply, {nil, nil}} 26 | else 27 | {:noreply, {pid, msg}} 28 | end 29 | end 30 | 31 | def handle_call({:set_pid_msg, pid, msg}, _from, _) do 32 | {:reply, :ok, {pid, msg}} 33 | end 34 | end 35 | 36 | BenchRecorder.start_link() 37 | 38 | prepare = fn number -> 39 | BenchRecorder.subscribe_to({:add, number, number}) 40 | 41 | {:ok, c1} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap, sync_interval: 5) 42 | 43 | {:ok, c2} = 44 | DeltaCrdt.start_link(DeltaCrdt.AWLWWMap, 45 | on_diffs: {BenchRecorder, :on_diffs, []} 46 | ) 47 | 48 | DeltaCrdt.set_neighbours(c1, [c2]) 49 | DeltaCrdt.set_neighbours(c2, [c1]) 50 | 51 | Enum.each(1..number, fn x -> 52 | DeltaCrdt.mutate(c1, :add, [x, x], 60_000) 53 | end) 54 | 55 | receive do 56 | {:add, ^number, ^number} -> :ok 57 | after 58 | 60_000 -> raise "waited for 60s" 59 | end 60 | 61 | :ok = GenServer.call(c1, :hibernate, 15_000) 62 | :ok = GenServer.call(c2, :hibernate, 15_000) 63 | :ok = GenServer.call(c1, :ping, 15_000) 64 | :ok = GenServer.call(c2, :ping, 15_000) 65 | 66 | {c1, c2} 67 | end 68 | 69 | perform = fn {c1, c2}, op -> 70 | range = 71 | case op do 72 | :add -> 73 | 100_000..100_010 74 | 75 | :remove -> 76 | 1..10 77 | end 78 | 79 | Enum.each(range, fn x -> 80 | case op do 81 | :add -> 82 | BenchRecorder.subscribe_to({:add, 100_010, 100_010}) 83 | DeltaCrdt.mutate(c1, :add, [x, x], 60_000) 84 | 85 | :remove -> 86 | BenchRecorder.subscribe_to({:remove, 10}) 87 | DeltaCrdt.mutate(c1, :remove, [x], 60_000) 88 | end 89 | end) 90 | 91 | case op do 92 | :add -> 93 | receive do 94 | {:add, 100_010, 100_010} -> :ok 95 | after 96 | 60_000 -> raise "waited for 60s" 97 | end 98 | 99 | :remove -> 100 | receive do 101 | {:remove, 10} -> :ok 102 | after 103 | 60_000 -> raise "waited for 60s" 104 | end 105 | end 106 | 107 | Process.exit(c1, :normal) 108 | Process.exit(c2, :normal) 109 | end 110 | 111 | Benchee.run( 112 | %{ 113 | "add 10" => fn input -> perform.(input, :add) end, 114 | "remove 10" => fn input -> perform.(input, :remove) end 115 | }, 116 | before_each: fn input -> prepare.(input) end, 117 | inputs: %{ 118 | # 10 => 10, 119 | # 100 => 100, 120 | # 1000 => 1000, 121 | # 10_000 => 10_000, 122 | 20_000 => 20_000, 123 | 30_000 => 30_000 124 | } 125 | # formatters: [Benchee.Formatters.HTML, Benchee.Formatters.Console] 126 | ) 127 | -------------------------------------------------------------------------------- /lib/benchmark_helper.ex: -------------------------------------------------------------------------------- 1 | defmodule BenchmarkHelper do 2 | defmacro inject_in_dev() do 3 | quote do 4 | if Mix.env() == :dev do 5 | def handle_call(:hibernate, _from, state) do 6 | {:reply, :ok, state, :hibernate} 7 | end 8 | 9 | def handle_call(:ping, _from, state) do 10 | {:reply, :ok, state} 11 | end 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/delta_crdt.ex: -------------------------------------------------------------------------------- 1 | defmodule DeltaCrdt do 2 | @moduledoc """ 3 | Start and interact with the Delta CRDTs provided by this library. 4 | 5 | A CRDT is a conflict-free replicated data-type. That is to say, it is a distributed data structure that automatically resolves conflicts in a way that is consistent across all replicas of the data. In other words, your distributed data is guaranteed to eventually converge globally. 6 | 7 | Normal CRDTs (otherwise called "state CRDTs") require transmission of the entire CRDT state with every change. This clearly doesn't scale, but there has been exciting research in the last few years into "Delta CRDTs", CRDTs that only transmit their deltas. This has enabled a whole new scale of applications for CRDTs, and it's also what this library is based on. 8 | 9 | A Delta CRDT is made of two parts. First, the data structure itself, and second, an anti-entropy algorithm, which is responsible for ensuring convergence. `DeltaCrdt` implements Algorithm 2 from ["Delta State Replicated Data Types – Almeida et al. 2016"](https://arxiv.org/pdf/1603.01529.pdf) which is an anti-entropy algorithm for δ-CRDTs. `DeltaCrdt` also implements join decomposition to ensure that deltas aren't transmitted unnecessarily in the cluster. 10 | 11 | While it is certainly interesting to have a look at this paper and spend time grokking it, in theory I've done the hard work so that you don't have to, and this library is the result. 12 | 13 | With this library, you can build distributed applications that share some state. [`Horde.Supervisor`](https://hexdocs.pm/horde/Horde.Supervisor.html) and [`Horde.Registry`](https://hexdocs.pm/horde/Horde.Registry.html) are both built atop `DeltaCrdt`, but there are certainly many more possibilities. 14 | 15 | Here's a simple example for illustration: 16 | 17 | ``` 18 | iex> {:ok, crdt1} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap, sync_interval: 3) 19 | iex> {:ok, crdt2} = DeltaCrdt.start_link(DeltaCrdt.AWLWWMap, sync_interval: 3) 20 | iex> DeltaCrdt.set_neighbours(crdt1, [crdt2]) 21 | iex> DeltaCrdt.set_neighbours(crdt2, [crdt1]) 22 | iex> DeltaCrdt.to_map(crdt1) 23 | %{} 24 | iex> DeltaCrdt.put(crdt1, "CRDT", "is magic!") 25 | iex> Process.sleep(10) # need to wait for propagation for the doctest 26 | iex> DeltaCrdt.to_map(crdt2) 27 | %{"CRDT" => "is magic!"} 28 | ``` 29 | """ 30 | 31 | require Logger 32 | 33 | @warn_sync_interval Mix.env() != :test 34 | 35 | @default_sync_interval 200 36 | @default_max_sync_size 200 37 | @default_timeout 5_000 38 | 39 | @type t :: GenServer.server() 40 | @type key :: any() 41 | @type value :: any() 42 | @type diff :: {:add, key :: any(), value :: any()} | {:remove, key :: any()} 43 | @type crdt_option :: 44 | {:on_diffs, ([diff()] -> any()) | {module(), function(), [any()]}} 45 | | {:sync_interval, pos_integer()} 46 | | {:max_sync_size, pos_integer() | :infinite} 47 | | {:name, GenServer.name()} 48 | | {:storage_module, DeltaCrdt.Storage.t()} 49 | 50 | @type crdt_options :: [crdt_option()] 51 | 52 | @doc """ 53 | Start a DeltaCrdt and link it to the calling process. 54 | 55 | There are a number of options you can specify to tweak the behaviour of DeltaCrdt: 56 | - `:sync_interval` - the delta CRDT will attempt to sync its local changes with its neighbours at this interval (specified in milliseconds). Default is 200. 57 | - `:on_diffs` - function which will be invoked on every diff 58 | - `:max_sync_size` - maximum size of synchronization (specified in number of items to sync) 59 | - `:name` - name of the CRDT process 60 | - `:storage_module` - module which implements `DeltaCrdt.Storage` behaviour 61 | """ 62 | @spec start_link( 63 | crdt_module :: module(), 64 | opts :: crdt_options() 65 | ) :: GenServer.on_start() 66 | def start_link(crdt_module, opts \\ []) do 67 | init_arg = 68 | Keyword.put(opts, :crdt_module, crdt_module) 69 | |> Keyword.put_new(:sync_interval, @default_sync_interval) 70 | |> Keyword.put_new(:max_sync_size, @default_max_sync_size) 71 | 72 | warn_low_sync_interval(init_arg) 73 | 74 | GenServer.start_link(DeltaCrdt.CausalCrdt, init_arg, Keyword.take(opts, [:name])) 75 | end 76 | 77 | defp warn_low_sync_interval(init_arg) do 78 | sync_interval = Keyword.get(init_arg, :sync_interval) 79 | 80 | if @warn_sync_interval && sync_interval < 100 do 81 | Logger.warning( 82 | "sync_interval (#{sync_interval}) less than 100. This parameter governs the amount of time in milliseconds between syncs. A value of 50 will therefore cause DeltaCrdt to attempt syncing its data with neighbour nodes 20 times a second!" 83 | ) 84 | end 85 | end 86 | 87 | @doc """ 88 | Include DeltaCrdt in a supervision tree with `{DeltaCrdt, [crdt: DeltaCrdt.AWLWWMap, name: MyCRDTMap]}` 89 | """ 90 | def child_spec(opts \\ []) do 91 | name = Keyword.get(opts, :name, __MODULE__) 92 | crdt_module = Keyword.get(opts, :crdt, nil) 93 | shutdown = Keyword.get(opts, :shutdown, 5000) 94 | 95 | if is_nil(crdt_module) do 96 | raise "must specify :crdt in options, got: #{inspect(opts)}" 97 | end 98 | 99 | %{ 100 | id: name, 101 | start: {DeltaCrdt, :start_link, [crdt_module, opts]}, 102 | shutdown: shutdown 103 | } 104 | end 105 | 106 | @doc """ 107 | Notify a CRDT of its neighbours. 108 | 109 | This function allows CRDTs to communicate with each other and sync their states. 110 | 111 | **Note: this sets up a unidirectional sync, so if you want bidirectional syncing (which is normally desirable), then you must call this function twice (or thrice for 3 nodes, etc):** 112 | ``` 113 | DeltaCrdt.set_neighbours(c1, [c2, c3]) 114 | DeltaCrdt.set_neighbours(c2, [c1, c3]) 115 | DeltaCrdt.set_neighbours(c3, [c1, c2]) 116 | ``` 117 | """ 118 | @spec set_neighbours(crdt :: t(), neighbours :: list(t())) :: :ok 119 | def set_neighbours(crdt, neighbours) when is_list(neighbours) do 120 | send(crdt, {:set_neighbours, neighbours}) 121 | :ok 122 | end 123 | 124 | @spec put(t(), key(), value(), timeout()) :: t() 125 | def put(crdt, key, value, timeout \\ @default_timeout) do 126 | :ok = GenServer.call(crdt, {:operation, {:add, [key, value]}}, timeout) 127 | crdt 128 | end 129 | 130 | @spec merge(t(), map(), timeout()) :: t() 131 | def merge(crdt, map, timeout \\ @default_timeout) do 132 | :ok = 133 | GenServer.call( 134 | crdt, 135 | {:bulk_operation, Enum.map(map, fn {key, value} -> {:add, [key, value]} end)}, 136 | timeout 137 | ) 138 | 139 | crdt 140 | end 141 | 142 | @spec drop(t(), [key()], timeout()) :: t() 143 | def drop(crdt, keys, timeout \\ @default_timeout) do 144 | :ok = 145 | GenServer.call( 146 | crdt, 147 | {:bulk_operation, Enum.map(keys, fn key -> {:remove, [key]} end)}, 148 | timeout 149 | ) 150 | 151 | crdt 152 | end 153 | 154 | @spec delete(t(), key(), timeout()) :: t() 155 | def delete(crdt, key, timeout \\ @default_timeout) do 156 | :ok = GenServer.call(crdt, {:operation, {:remove, [key]}}, timeout) 157 | crdt 158 | end 159 | 160 | @spec get(t(), key(), timeout()) :: value() 161 | def get(crdt, key, timeout \\ @default_timeout) do 162 | case GenServer.call(crdt, {:read, [key]}, timeout) do 163 | %{^key => elem} -> elem 164 | _ -> nil 165 | end 166 | end 167 | 168 | @spec take(t(), [key()], timeout()) :: [{key(), value()}] 169 | def take(crdt, keys, timeout \\ @default_timeout) when is_list(keys) do 170 | GenServer.call(crdt, {:read, keys}, timeout) 171 | end 172 | 173 | @spec to_map(t(), timeout()) :: map() 174 | def to_map(crdt, timeout \\ @default_timeout) do 175 | GenServer.call(crdt, :read, timeout) 176 | end 177 | 178 | @spec mutate( 179 | crdt :: t(), 180 | function :: atom, 181 | arguments :: list(), 182 | timeout :: timeout() 183 | ) :: :ok 184 | @doc """ 185 | Mutate the CRDT synchronously. 186 | 187 | For the asynchronous version of this function, see `mutate_async/3`. 188 | 189 | To see which operations are available, see the documentation for the crdt module that was provided in `start_link/3`. 190 | 191 | For example, `DeltaCrdt.AWLWWMap` has a function `add` that takes 4 arguments. The last 2 arguments are supplied by DeltaCrdt internally, so you have to provide only the first two arguments: `key` and `val`. That would look like this: `DeltaCrdt.mutate(crdt, :add, ["CRDT", "is magic!"])`. This pattern is repeated for all mutation functions. Another example: to call `DeltaCrdt.AWLWWMap.clear`, use `DeltaCrdt.mutate(crdt, :clear, [])`. 192 | """ 193 | @deprecated "Use put/4 instead" 194 | def mutate(crdt, f, a, timeout \\ @default_timeout) 195 | when is_atom(f) and is_list(a) do 196 | GenServer.call(crdt, {:operation, {f, a}}, timeout) 197 | end 198 | 199 | @spec mutate_async(crdt :: t(), function :: atom, arguments :: list()) :: :ok 200 | @doc """ 201 | Mutate the CRDT asynchronously. 202 | """ 203 | @deprecated "Will be removed without replacement in a future version" 204 | def mutate_async(crdt, f, a) 205 | when is_atom(f) and is_list(a) do 206 | GenServer.cast(crdt, {:operation, {f, a}}) 207 | end 208 | 209 | @doc """ 210 | Read the state of the CRDT. 211 | 212 | Forwards arguments to the used crdt module, so `read(crdt, ["my-key"])` would call `crdt_module.read(state, ["my-key"])`. 213 | 214 | For example, `DeltaCrdt.AWLWWMap` accepts a list of keys to limit the returned values instead of returning everything. 215 | """ 216 | @spec read(crdt :: t()) :: crdt_state :: term() 217 | @spec read(crdt :: t(), timeout :: timeout()) :: crdt_state :: term() 218 | @spec read(crdt :: t(), keys :: list()) :: crdt_state :: term() 219 | @spec read(crdt :: t(), keys :: list(), timeout :: timeout()) :: 220 | crdt_state :: term() 221 | @deprecated "Use get/2 or take/3 or to_map/2" 222 | def read(crdt), do: read(crdt, @default_timeout) 223 | def read(crdt, keys) when is_list(keys), do: read(crdt, keys, @default_timeout) 224 | 225 | def read(crdt, timeout) do 226 | GenServer.call(crdt, :read, timeout) 227 | end 228 | 229 | def read(crdt, keys, timeout) when is_list(keys) do 230 | GenServer.call(crdt, {:read, keys}, timeout) 231 | end 232 | end 233 | -------------------------------------------------------------------------------- /lib/delta_crdt/aw_lww_map.ex: -------------------------------------------------------------------------------- 1 | defmodule DeltaCrdt.AWLWWMap do 2 | defstruct dots: MapSet.new(), 3 | value: %{} 4 | 5 | require Logger 6 | 7 | @doc false 8 | def new(), do: %__MODULE__{} 9 | 10 | defmodule Dots do 11 | @moduledoc false 12 | 13 | def compress(dots = %MapSet{}) do 14 | Enum.reduce(dots, %{}, fn {c, i}, dots_map -> 15 | Map.update(dots_map, c, i, fn 16 | x when x > i -> x 17 | _x -> i 18 | end) 19 | end) 20 | end 21 | 22 | def decompress(dots = %MapSet{}), do: dots 23 | 24 | def decompress(dots) do 25 | Enum.flat_map(dots, fn {i, x} -> 26 | Enum.map(1..x, fn y -> {i, y} end) 27 | end) 28 | end 29 | 30 | def next_dot(i, c = %MapSet{}) do 31 | Logger.warning("inefficient next_dot computation") 32 | next_dot(i, compress(c)) 33 | end 34 | 35 | def next_dot(i, c) do 36 | {i, Map.get(c, i, 0) + 1} 37 | end 38 | 39 | def union(dots1 = %MapSet{}, dots2 = %MapSet{}) do 40 | MapSet.union(dots1, dots2) 41 | end 42 | 43 | def union(dots1 = %MapSet{}, dots2), do: union(dots2, dots1) 44 | 45 | def union(dots1, dots2) do 46 | Enum.reduce(dots2, dots1, fn {c, i}, dots_map -> 47 | Map.update(dots_map, c, i, fn 48 | x when x > i -> x 49 | _x -> i 50 | end) 51 | end) 52 | end 53 | 54 | def difference(dots1 = %MapSet{}, dots2 = %MapSet{}) do 55 | MapSet.difference(dots1, dots2) 56 | end 57 | 58 | def difference(_dots1, _dots2 = %MapSet{}), do: raise("this should not happen") 59 | 60 | def difference(dots1, dots2) do 61 | Enum.reject(dots1, fn dot -> 62 | member?(dots2, dot) 63 | end) 64 | |> MapSet.new() 65 | end 66 | 67 | def member?(dots = %MapSet{}, dot = {_, _}) do 68 | MapSet.member?(dots, dot) 69 | end 70 | 71 | def member?(dots, {i, x}) do 72 | Map.get(dots, i, 0) >= x 73 | end 74 | 75 | def strict_expansion?(_dots = %MapSet{}, _delta_dots) do 76 | raise "we should not get here" 77 | end 78 | 79 | def strict_expansion?(dots, delta_dots) do 80 | Enum.all?(min_dots(delta_dots), fn {i, x} -> 81 | Map.get(dots, i, 0) + 1 >= x 82 | end) 83 | end 84 | 85 | def min_dots(dots = %MapSet{}) do 86 | Enum.reduce(dots, %{}, fn {i, x}, min -> 87 | Map.update(min, i, x, fn 88 | min when min < x -> min 89 | _min -> x 90 | end) 91 | end) 92 | end 93 | 94 | def min_dots(_dots) do 95 | %{} 96 | end 97 | end 98 | 99 | def add(key, value, i, state) do 100 | rem = remove(key, i, state) 101 | 102 | add = 103 | fn aw_set, context -> 104 | aw_set_add(i, {value, System.monotonic_time(:nanosecond)}, {aw_set, context}) 105 | end 106 | |> apply_op(key, state) 107 | 108 | case MapSet.size(rem.dots) do 109 | 0 -> add 110 | _ -> join(rem, add, [key]) 111 | end 112 | end 113 | 114 | @doc false 115 | def compress_dots(state) do 116 | %{state | dots: Dots.compress(state.dots)} 117 | end 118 | 119 | defp aw_set_add(i, el, {aw_set, c}) do 120 | d = Dots.next_dot(i, c) 121 | {%{el => MapSet.new([d])}, MapSet.put(Map.get(aw_set, el, MapSet.new()), d)} 122 | end 123 | 124 | defp apply_op(op, key, %{value: m, dots: c}) do 125 | {val, c_p} = op.(Map.get(m, key, %{}), c) 126 | 127 | %__MODULE__{ 128 | dots: MapSet.new(c_p), 129 | value: %{key => val} 130 | } 131 | end 132 | 133 | def remove(key, _i, state) do 134 | %{value: val} = state 135 | 136 | to_remove_dots = 137 | case Map.fetch(val, key) do 138 | {:ok, value} -> Enum.flat_map(value, fn {_val, to_remove_dots} -> to_remove_dots end) 139 | :error -> [] 140 | end 141 | 142 | %__MODULE__{ 143 | dots: MapSet.new(to_remove_dots), 144 | value: %{} 145 | } 146 | end 147 | 148 | def clear(_i, state) do 149 | Map.put(state, :value, %{}) 150 | end 151 | 152 | @doc false 153 | def join(delta1, delta2, keys) do 154 | new_dots = Dots.union(delta1.dots, delta2.dots) 155 | 156 | join_or_maps(delta1, delta2, [:join_or_maps, :join_dot_sets], keys) 157 | |> Map.put(:dots, new_dots) 158 | end 159 | 160 | @doc false 161 | def join_or_maps(delta1, delta2, nested_joins, keys) do 162 | resolved_conflicts = 163 | Enum.flat_map(keys, fn key -> 164 | sub_delta1 = Map.put(delta1, :value, Map.get(delta1.value, key, %{})) 165 | 166 | sub_delta2 = Map.put(delta2, :value, Map.get(delta2.value, key, %{})) 167 | 168 | keys = 169 | (Map.keys(sub_delta1.value) ++ Map.keys(sub_delta2.value)) 170 | |> Enum.uniq() 171 | 172 | [next_join | other_joins] = nested_joins 173 | 174 | %{value: new_sub} = 175 | apply(__MODULE__, next_join, [sub_delta1, sub_delta2, other_joins, keys]) 176 | 177 | if Enum.empty?(new_sub) do 178 | [] 179 | else 180 | [{key, new_sub}] 181 | end 182 | end) 183 | |> Map.new() 184 | 185 | new_val = 186 | Map.drop(delta1.value, keys) 187 | |> Map.merge(Map.drop(delta2.value, keys)) 188 | |> Map.merge(resolved_conflicts) 189 | 190 | %__MODULE__{ 191 | value: new_val 192 | } 193 | end 194 | 195 | @doc false 196 | def join_dot_sets(%{value: s1, dots: c1}, %{value: s2, dots: c2}, [], _keys) do 197 | s1 = MapSet.new(s1) 198 | s2 = MapSet.new(s2) 199 | 200 | new_s = 201 | [ 202 | MapSet.intersection(s1, s2), 203 | Dots.difference(s1, c2), 204 | Dots.difference(s2, c1) 205 | ] 206 | |> Enum.reduce(&MapSet.union/2) 207 | 208 | %__MODULE__{value: new_s} 209 | end 210 | 211 | def read(%{value: values}) do 212 | Map.new(values, fn {key, values} -> 213 | {{val, _ts}, _c} = Enum.max_by(values, fn {{_val, ts}, _c} -> ts end) 214 | {key, val} 215 | end) 216 | end 217 | 218 | def read(_crdt, []), do: %{} 219 | 220 | def read(%{value: values}, keys) when is_list(keys) do 221 | read(%{value: Map.take(values, keys)}) 222 | end 223 | 224 | def read(crdt, key) do 225 | read(crdt, [key]) 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /lib/delta_crdt/causal_crdt.ex: -------------------------------------------------------------------------------- 1 | defmodule DeltaCrdt.CausalCrdt do 2 | use GenServer 3 | 4 | require Logger 5 | 6 | require BenchmarkHelper 7 | 8 | BenchmarkHelper.inject_in_dev() 9 | 10 | @type delta :: {k :: integer(), delta :: any()} 11 | @type delta_interval :: {a :: integer(), b :: integer(), delta :: delta()} 12 | 13 | @moduledoc false 14 | 15 | defstruct node_id: nil, 16 | name: nil, 17 | on_diffs: nil, 18 | storage_module: nil, 19 | crdt_module: nil, 20 | crdt_state: nil, 21 | merkle_map: MerkleMap.new(), 22 | sequence_number: 0, 23 | neighbours: MapSet.new(), 24 | neighbour_monitors: %{}, 25 | outstanding_syncs: %{}, 26 | sync_interval: nil, 27 | max_sync_size: nil 28 | 29 | defmodule(Diff, do: defstruct(continuation: nil, dots: nil, originator: nil, from: nil, to: nil)) 30 | 31 | defmacrop strip_continue(tuple) do 32 | if System.otp_release() |> String.to_integer() > 20 do 33 | tuple 34 | else 35 | quote do 36 | case unquote(tuple) do 37 | {tup1, tup2, {:continue, _}} -> {tup1, tup2} 38 | end 39 | end 40 | end 41 | end 42 | 43 | ### GenServer callbacks 44 | 45 | def init(opts) do 46 | send(self(), :sync) 47 | 48 | Process.flag(:trap_exit, true) 49 | 50 | crdt_module = Keyword.get(opts, :crdt_module) 51 | 52 | max_sync_size = 53 | case Keyword.get(opts, :max_sync_size) do 54 | :infinite -> 55 | :infinite 56 | 57 | size when is_integer(size) and size > 0 -> 58 | size 59 | 60 | invalid_size -> 61 | raise ArgumentError, "#{inspect(invalid_size)} is not a valid max_sync_size" 62 | end 63 | 64 | initial_state = %__MODULE__{ 65 | node_id: :rand.uniform(1_000_000_000), 66 | name: Keyword.get(opts, :name), 67 | on_diffs: Keyword.get(opts, :on_diffs), 68 | storage_module: Keyword.get(opts, :storage_module), 69 | sync_interval: Keyword.get(opts, :sync_interval), 70 | max_sync_size: max_sync_size, 71 | crdt_module: crdt_module, 72 | crdt_state: crdt_module.new() |> crdt_module.compress_dots() 73 | } 74 | 75 | strip_continue({:ok, initial_state, {:continue, :read_storage}}) 76 | end 77 | 78 | def handle_continue(:read_storage, state) do 79 | {:noreply, read_from_storage(state)} 80 | end 81 | 82 | def handle_info({:ack_diff, to}, state) do 83 | {:noreply, %{state | outstanding_syncs: Map.delete(state.outstanding_syncs, to)}} 84 | end 85 | 86 | def handle_info({:diff, diff, keys}, state) do 87 | new_state = update_state_with_delta(state, diff, keys) 88 | {:noreply, new_state} 89 | end 90 | 91 | def handle_info({:diff, diff}, state) do 92 | diff = reverse_diff(diff) 93 | 94 | new_merkle_map = MerkleMap.update_hashes(state.merkle_map) 95 | 96 | case MerkleMap.continue_partial_diff(diff.continuation, new_merkle_map, 8) do 97 | {:continue, continuation} -> 98 | %Diff{diff | continuation: truncate(continuation, state.max_sync_size)} 99 | |> send_diff_continue() 100 | 101 | {:ok, []} -> 102 | ack_diff(diff) 103 | 104 | {:ok, keys} -> 105 | send_diff(diff, truncate(keys, state.max_sync_size), state) 106 | ack_diff(diff) 107 | end 108 | 109 | {:noreply, Map.put(state, :merkle_map, new_merkle_map)} 110 | end 111 | 112 | def handle_info({:get_diff, diff, keys}, state) do 113 | diff = reverse_diff(diff) 114 | 115 | send( 116 | diff.to, 117 | {:diff, 118 | %{state.crdt_state | dots: diff.dots, value: Map.take(state.crdt_state.value, keys)}, keys} 119 | ) 120 | 121 | ack_diff(diff) 122 | {:noreply, state} 123 | end 124 | 125 | def handle_info({:EXIT, _pid, :normal}, state), do: {:noreply, state} 126 | 127 | def handle_info({:DOWN, ref, :process, _object, _reason}, state) do 128 | {neighbour, _ref} = 129 | Enum.find(state.neighbour_monitors, fn 130 | {_neighbour, ^ref} -> true 131 | _ -> false 132 | end) 133 | 134 | new_neighbour_monitors = Map.delete(state.neighbour_monitors, neighbour) 135 | 136 | new_outstanding_syncs = Map.delete(state.outstanding_syncs, neighbour) 137 | 138 | new_state = %{ 139 | state 140 | | neighbour_monitors: new_neighbour_monitors, 141 | outstanding_syncs: new_outstanding_syncs 142 | } 143 | 144 | {:noreply, new_state} 145 | end 146 | 147 | def handle_info({:set_neighbours, neighbours}, state) do 148 | new_neighbours = MapSet.new(neighbours) 149 | ex_neighbours = MapSet.difference(state.neighbours, new_neighbours) 150 | 151 | for n <- ex_neighbours do 152 | case Map.get(state.neighbour_monitors, n) do 153 | ref when is_reference(ref) -> Process.demonitor(ref, [:flush]) 154 | _ -> nil 155 | end 156 | end 157 | 158 | new_outstanding_syncs = 159 | Enum.filter(state.outstanding_syncs, fn {neighbour, 1} -> 160 | MapSet.member?(new_neighbours, neighbour) 161 | end) 162 | |> Map.new() 163 | 164 | new_neighbour_monitors = 165 | Enum.filter(state.neighbour_monitors, fn {neighbour, _ref} -> 166 | MapSet.member?(new_neighbours, neighbour) 167 | end) 168 | |> Map.new() 169 | 170 | state = %{ 171 | state 172 | | neighbours: new_neighbours, 173 | outstanding_syncs: new_outstanding_syncs, 174 | neighbour_monitors: new_neighbour_monitors 175 | } 176 | 177 | {:noreply, sync_interval_or_state_to_all(state)} 178 | end 179 | 180 | def handle_info(:sync, state) do 181 | state = sync_interval_or_state_to_all(state) 182 | 183 | Process.send_after(self(), :sync, state.sync_interval) 184 | 185 | {:noreply, state} 186 | end 187 | 188 | def handle_call(:read, _from, state) do 189 | {:reply, state.crdt_module.read(state.crdt_state), state} 190 | end 191 | 192 | def handle_call({:read, keys}, _from, state) do 193 | {:reply, state.crdt_module.read(state.crdt_state, keys), state} 194 | end 195 | 196 | def handle_call({:bulk_operation, operations}, _from, state) do 197 | {:reply, :ok, 198 | Enum.reduce(operations, state, fn operation, state -> 199 | handle_operation(operation, state) 200 | end)} 201 | end 202 | 203 | def handle_call({:operation, operation}, _from, state) do 204 | {:reply, :ok, handle_operation(operation, state)} 205 | end 206 | 207 | def handle_cast({:operation, operation}, state) do 208 | {:noreply, handle_operation(operation, state)} 209 | end 210 | 211 | # TODO this won't sync everything anymore, since syncing is now a 2-step process. 212 | # Figure out how to do this properly. Maybe with a `receive` block. 213 | def terminate(_reason, state) do 214 | sync_interval_or_state_to_all(state) 215 | end 216 | 217 | defp truncate(list, :infinite), do: list 218 | 219 | defp truncate(list, size) when is_list(list) and is_integer(size) do 220 | Enum.take(list, size) 221 | end 222 | 223 | defp truncate(diff, size) when is_integer(size) do 224 | MerkleMap.truncate_diff(diff, size) 225 | end 226 | 227 | defp read_from_storage(%{storage_module: nil} = state) do 228 | state 229 | end 230 | 231 | defp read_from_storage(state) do 232 | case state.storage_module.read(state.name) do 233 | nil -> 234 | state 235 | 236 | {node_id, sequence_number, crdt_state, merkle_map} -> 237 | Map.put(state, :sequence_number, sequence_number) 238 | |> Map.put(:crdt_state, crdt_state) 239 | |> Map.put(:merkle_map, merkle_map) 240 | |> Map.put(:node_id, node_id) 241 | |> remove_crdt_state_keys() 242 | end 243 | end 244 | 245 | defp remove_crdt_state_keys(state) do 246 | %{state | crdt_state: Map.put(state.crdt_state, :keys, MapSet.new())} 247 | end 248 | 249 | defp write_to_storage(%{storage_module: nil} = state) do 250 | state 251 | end 252 | 253 | defp write_to_storage(state) do 254 | :ok = 255 | state.storage_module.write( 256 | state.name, 257 | {state.node_id, state.sequence_number, state.crdt_state, state.merkle_map} 258 | ) 259 | 260 | state 261 | end 262 | 263 | defp sync_interval_or_state_to_all(state) do 264 | state = monitor_neighbours(state) 265 | new_merkle_map = MerkleMap.update_hashes(state.merkle_map) 266 | {:continue, continuation} = MerkleMap.prepare_partial_diff(new_merkle_map, 8) 267 | 268 | diff = %Diff{ 269 | continuation: continuation, 270 | dots: state.crdt_state.dots, 271 | from: self(), 272 | originator: self() 273 | } 274 | 275 | new_outstanding_syncs = 276 | Enum.map(state.neighbour_monitors, fn {neighbour, _monitor} -> neighbour end) 277 | |> Enum.reject(fn pid -> self() == pid end) 278 | |> Enum.reduce(state.outstanding_syncs, fn neighbour, outstanding_syncs -> 279 | Map.put_new_lazy(outstanding_syncs, neighbour, fn -> 280 | try do 281 | send(neighbour, {:diff, %Diff{diff | to: neighbour}}) 282 | 1 283 | rescue 284 | _ in ArgumentError -> 285 | # This happens when we attempt to sync with a neighbour that is dead. 286 | # This can happen, and is not a big deal, since syncing is idempotent. 287 | Logger.debug( 288 | "tried to sync with a dead neighbour: #{inspect(neighbour)}, ignore the error and move on" 289 | ) 290 | 291 | 0 292 | end 293 | end) 294 | end) 295 | |> Enum.filter(&match?({_, 0}, &1)) 296 | |> Map.new() 297 | 298 | Map.put(state, :outstanding_syncs, new_outstanding_syncs) 299 | |> Map.put(:merkle_map, new_merkle_map) 300 | end 301 | 302 | defp monitor_neighbours(state) do 303 | new_neighbour_monitors = 304 | Enum.reduce(state.neighbours, state.neighbour_monitors, fn neighbour, monitors -> 305 | Map.put_new_lazy(monitors, neighbour, fn -> 306 | try do 307 | Process.monitor(neighbour) 308 | rescue 309 | _ in ArgumentError -> 310 | # This happens can happen when we attempt to monitor a that is dead. 311 | # This can happen, and is not a big deal, since we'll re-add the monitor later 312 | # to get notified when the neighbour comes back online. 313 | Logger.debug( 314 | "tried to monitor a dead neighbour: #{inspect(neighbour)}, ignore the error and move on" 315 | ) 316 | 317 | :error 318 | end 319 | end) 320 | end) 321 | |> Enum.reject(&match?({_, :error}, &1)) 322 | |> Map.new() 323 | 324 | Map.put(state, :neighbour_monitors, new_neighbour_monitors) 325 | end 326 | 327 | defp reverse_diff(diff) do 328 | %Diff{diff | from: diff.to, to: diff.from} 329 | end 330 | 331 | defp send_diff_continue(diff) do 332 | send(diff.to, {:diff, diff}) 333 | end 334 | 335 | defp send_diff(diff, keys, state) do 336 | if diff.originator == diff.to do 337 | send(diff.to, {:get_diff, diff, keys}) 338 | else 339 | send( 340 | diff.to, 341 | {:diff, 342 | %{state.crdt_state | dots: diff.dots, value: Map.take(state.crdt_state.value, keys)}, 343 | keys} 344 | ) 345 | end 346 | end 347 | 348 | defp handle_operation({function, [key | rest_args]}, state) do 349 | delta = 350 | apply(state.crdt_module, function, [key | rest_args] ++ [state.node_id, state.crdt_state]) 351 | 352 | update_state_with_delta(state, delta, [key]) 353 | end 354 | 355 | defp diff(old_state, new_state, keys) do 356 | Enum.flat_map(keys, fn key -> 357 | case {Map.get(old_state.crdt_state.value, key), Map.get(new_state.crdt_state.value, key)} do 358 | {old, old} -> [] 359 | {_old, nil} -> [{:remove, key}] 360 | {_old, new} -> [{:add, key, new}] 361 | end 362 | end) 363 | end 364 | 365 | defp diffs_keys(diffs) do 366 | Enum.map(diffs, fn 367 | {:add, key, _val} -> key 368 | {:remove, key} -> key 369 | end) 370 | end 371 | 372 | defp diffs_to_callback(_old_state, _new_state, []), do: nil 373 | 374 | defp diffs_to_callback(old_state, new_state, keys) do 375 | old = new_state.crdt_module.read(old_state.crdt_state, keys) 376 | new = new_state.crdt_module.read(new_state.crdt_state, keys) 377 | 378 | diffs = 379 | Enum.flat_map(keys, fn key -> 380 | case {Map.get(old, key), Map.get(new, key)} do 381 | {old, old} -> [] 382 | {_old, nil} -> [{:remove, key}] 383 | {_old, new} -> [{:add, key, new}] 384 | end 385 | end) 386 | 387 | case new_state.on_diffs do 388 | function when is_function(function) -> function.(diffs) 389 | {module, function, args} -> apply(module, function, args ++ [diffs]) 390 | nil -> nil 391 | end 392 | end 393 | 394 | defp update_state_with_delta(state, delta, keys) do 395 | new_crdt_state = state.crdt_module.join(state.crdt_state, delta, keys) 396 | 397 | new_state = Map.put(state, :crdt_state, new_crdt_state) 398 | 399 | diffs = diff(state, new_state, keys) 400 | 401 | {new_merkle_map, count} = 402 | Enum.reduce(diffs, {state.merkle_map, 0}, fn 403 | {:add, key, value}, {mm, count} -> {MerkleMap.put(mm, key, value), count + 1} 404 | {:remove, key}, {mm, count} -> {MerkleMap.delete(mm, key), count + 1} 405 | end) 406 | 407 | :telemetry.execute([:delta_crdt, :sync, :done], %{keys_updated_count: count}, %{ 408 | name: state.name 409 | }) 410 | 411 | diffs_to_callback(state, new_state, diffs_keys(diffs)) 412 | 413 | Map.put(new_state, :merkle_map, new_merkle_map) 414 | |> write_to_storage() 415 | end 416 | 417 | defp ack_diff(%{originator: originator, from: originator, to: to}) do 418 | send(originator, {:ack_diff, to}) 419 | end 420 | 421 | defp ack_diff(%{originator: originator, from: from, to: originator}) do 422 | send(originator, {:ack_diff, from}) 423 | end 424 | end 425 | -------------------------------------------------------------------------------- /lib/delta_crdt/storage.ex: -------------------------------------------------------------------------------- 1 | defmodule DeltaCrdt.Storage do 2 | @moduledoc """ 3 | This behaviour can be used to enable persistence of the CRDT. 4 | 5 | This can be helpful in the event of crashes. 6 | 7 | To use, implement this behaviour in a module, and pass it to your CRDT with the `storage_module` option. 8 | """ 9 | 10 | @type t :: module() 11 | 12 | @opaque storage_format :: 13 | {node_id :: term(), sequence_number :: integer(), crdt_state :: term()} 14 | 15 | @callback write(name :: term(), storage_format()) :: :ok 16 | @callback read(name :: term()) :: storage_format() | nil 17 | end 18 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule DeltaCrdt.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :delta_crdt, 7 | version: "0.6.5", 8 | elixir: "~> 1.11", 9 | elixirc_paths: elixirc_paths(Mix.env()), 10 | start_permanent: Mix.env() == :prod, 11 | package: package(), 12 | source_url: "https://github.com/derekkraan/delta_crdt_ex", 13 | deps: deps() 14 | ] 15 | end 16 | 17 | # Run "mix help compile.app" to learn about applications. 18 | def application do 19 | [ 20 | extra_applications: [:logger] 21 | ] 22 | end 23 | 24 | # Run "mix help deps" to learn about dependencies. 25 | defp deps do 26 | [ 27 | {:telemetry, "~> 0.4 or ~> 1.0"}, 28 | {:benchee, ">= 0.0.0", only: :dev, runtime: false}, 29 | {:benchee_html, ">= 0.0.0", only: :dev, runtime: false}, 30 | {:exprof, "~> 0.2.0", only: :dev, runtime: false}, 31 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 32 | {:merkle_map, "~> 0.2.0"}, 33 | {:stream_data, "~> 0.4", only: :test} 34 | ] 35 | end 36 | 37 | defp package do 38 | [ 39 | name: "delta_crdt", 40 | description: "Implementations of δ-CRDTs", 41 | licenses: ["MIT"], 42 | maintainers: ["Derek Kraan"], 43 | links: %{GitHub: "https://github.com/derekkraan/delta_crdt_ex"} 44 | ] 45 | end 46 | 47 | defp elixirc_paths(:test), do: ["lib", "test/support"] 48 | defp elixirc_paths(_), do: ["lib"] 49 | end 50 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"}, 3 | "benchee_html": {:hex, :benchee_html, "1.0.0", "5b4d24effebd060f466fb460ec06576e7b34a00fc26b234fe4f12c4f05c95947", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "5280af9aac432ff5ca4216d03e8a93f32209510e925b60e7f27c33796f69e699"}, 4 | "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"}, 5 | "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, 6 | "earmark": {:hex, :earmark, "1.2.6", "b6da42b3831458d3ecc57314dff3051b080b9b2be88c2e5aa41cd642a5b044ed", [:mix], [], "hexpm", "b42a23e9bd92d65d16db2f75553982e58519054095356a418bb8320bbacb58b1"}, 7 | "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, 8 | "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"}, 9 | "exprintf": {:hex, :exprintf, "0.2.1", "b7e895dfb00520cfb7fc1671303b63b37dc3897c59be7cbf1ae62f766a8a0314", [:mix], [], "hexpm", "20a0e8c880be90e56a77fcc82533c5d60c643915c7ce0cc8aa1e06ed6001da28"}, 10 | "exprof": {:hex, :exprof, "0.2.4", "13ddc0575a6d24b52e7c6809d2a46e9ad63a4dd179628698cdbb6c1f6e497c98", [:mix], [{:exprintf, "~> 0.2", [hex: :exprintf, repo: "hexpm", optional: false]}], "hexpm", "0884bcb66afc421c75d749156acbb99034cc7db6d3b116c32e36f32551106957"}, 11 | "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, 12 | "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, 13 | "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, 14 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, 15 | "merkle_map": {:hex, :merkle_map, "0.2.1", "01a88c87a6b9fb594c67c17ebaf047ee55ffa34e74297aa583ed87148006c4c8", [:mix], [], "hexpm", "fed4d143a5c8166eee4fa2b49564f3c4eace9cb252f0a82c1613bba905b2d04d"}, 16 | "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, 17 | "stream_data": {:hex, :stream_data, "0.5.0", "b27641e58941685c75b353577dc602c9d2c12292dd84babf506c2033cd97893e", [:mix], [], "hexpm", "012bd2eec069ada4db3411f9115ccafa38540a3c78c4c0349f151fc761b9e271"}, 18 | "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"}, 19 | "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"}, 20 | "xxhash": {:hex, :xxhash, "0.2.1", "ab0893a8124f3c11116c57e500485dc5f67817d1d4c44f0fff41f3fd3c590607", [:mix], [], "hexpm"}, 21 | } 22 | -------------------------------------------------------------------------------- /test/aw_lww_map_property_test.exs: -------------------------------------------------------------------------------- 1 | defmodule AWLWWMapPropertyTest do 2 | alias DeltaCrdt.AWLWWMap 3 | use ExUnit.Case, async: true 4 | use ExUnitProperties 5 | 6 | setup do 7 | operation_gen = 8 | ExUnitProperties.gen all op <- StreamData.member_of([:add, :remove]), 9 | node_id <- term(), 10 | key <- term(), 11 | value <- term() do 12 | {op, key, value, node_id} 13 | end 14 | 15 | [operation_gen: operation_gen] 16 | end 17 | 18 | describe ".add/4" do 19 | property "can add an element" do 20 | check all key <- term(), 21 | val <- term(), 22 | node_id <- term() do 23 | assert %{key => val} == 24 | AWLWWMap.join( 25 | AWLWWMap.compress_dots(AWLWWMap.new()), 26 | AWLWWMap.add(key, val, node_id, AWLWWMap.compress_dots(AWLWWMap.new())), 27 | [key] 28 | ) 29 | |> AWLWWMap.read() 30 | end 31 | end 32 | end 33 | 34 | property "arbitrary add and remove sequence results in correct map", context do 35 | check all operations <- list_of(context.operation_gen) do 36 | actual_result = 37 | operations 38 | |> Enum.reduce(AWLWWMap.compress_dots(AWLWWMap.new()), fn 39 | {:add, key, val, node_id}, map -> 40 | AWLWWMap.join(map, AWLWWMap.add(key, val, node_id, map), [key]) 41 | 42 | {:remove, key, _val, node_id}, map -> 43 | AWLWWMap.join(map, AWLWWMap.remove(key, node_id, map), [key]) 44 | end) 45 | |> AWLWWMap.read() 46 | 47 | correct_result = 48 | operations 49 | |> Enum.reduce(%{}, fn 50 | {:add, key, value, _node_id}, map -> 51 | Map.put(map, key, value) 52 | 53 | {:remove, key, _value, _node_id}, map -> 54 | Map.delete(map, key) 55 | end) 56 | 57 | assert actual_result == correct_result 58 | end 59 | end 60 | 61 | describe ".remove/3" do 62 | property "can remove an element" do 63 | check all key <- term(), 64 | val <- term(), 65 | node_id <- term() do 66 | crdt = AWLWWMap.compress_dots(AWLWWMap.new()) 67 | crdt = AWLWWMap.join(crdt, AWLWWMap.add(key, val, node_id, crdt), [key]) 68 | 69 | crdt = 70 | AWLWWMap.join(crdt, AWLWWMap.remove(key, node_id, crdt), [key]) 71 | |> AWLWWMap.read() 72 | 73 | assert %{} == crdt 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /test/aw_lww_map_test.exs: -------------------------------------------------------------------------------- 1 | defmodule NewAWLWWMapTest do 2 | use ExUnit.Case 3 | use ExUnitProperties 4 | 5 | alias DeltaCrdt.AWLWWMap 6 | 7 | test "can add and read a value" do 8 | assert %{1 => 2} = 9 | AWLWWMap.add(1, 2, :foo_node, AWLWWMap.new()) 10 | |> AWLWWMap.read() 11 | end 12 | 13 | test "can join two adds" do 14 | add1 = AWLWWMap.add(1, 2, :foo_node, AWLWWMap.new()) 15 | add2 = AWLWWMap.add(2, 2, :foo_node, add1) 16 | 17 | assert %{1 => 2, 2 => 2} = 18 | AWLWWMap.join(add1, add2, [1, 2]) 19 | |> AWLWWMap.read() 20 | end 21 | 22 | test "can add multiple values and only read one" do 23 | new = AWLWWMap.new() 24 | add1 = AWLWWMap.add(1, 2, :foo_node, new) 25 | add2 = AWLWWMap.add(2, 3, :foo_node, add1) 26 | state = AWLWWMap.join(add1, add2, [1, 2]) 27 | 28 | assert %{1 => 2} = AWLWWMap.read(state, 1) 29 | end 30 | 31 | test "can remove elements" do 32 | add1 = AWLWWMap.add(1, 2, :foo_node, AWLWWMap.new()) 33 | remove1 = AWLWWMap.remove(1, :foo_node, add1) 34 | 35 | assert %{} = 36 | AWLWWMap.join(add1, remove1, [1]) 37 | |> AWLWWMap.read() 38 | end 39 | 40 | test "can resolve conflicts" do 41 | add1 = AWLWWMap.add(1, 2, :foo_node, AWLWWMap.new()) 42 | add2 = AWLWWMap.add(1, 3, :foo_node, add1) 43 | 44 | # TODO assert that the state doesn't include anything about value 2 45 | 46 | assert %{1 => 3} = 47 | AWLWWMap.join(add1, add2, [1]) 48 | |> AWLWWMap.read() 49 | end 50 | 51 | test "can compute actual dots present" do 52 | add1 = AWLWWMap.add(1, 2, :foo_node, AWLWWMap.new()) 53 | change1 = AWLWWMap.add(1, 3, :foo_node, add1) 54 | 55 | final = AWLWWMap.join(add1, change1, [1]) 56 | 57 | assert 1 = map_size(AWLWWMap.compress_dots(final) |> Map.get(:dots)) 58 | end 59 | 60 | property "arbitrary add and remove sequence results in correct map" do 61 | operation_gen = 62 | ExUnitProperties.gen all op <- StreamData.member_of([:add, :remove]), 63 | node_id <- term(), 64 | key <- term(), 65 | value <- term() do 66 | {op, key, value, node_id} 67 | end 68 | 69 | check all operations <- list_of(operation_gen) do 70 | actual_result = 71 | operations 72 | |> Enum.reduce(AWLWWMap.new(), fn 73 | {:add, key, val, node_id}, map -> 74 | AWLWWMap.add(key, val, node_id, map) 75 | |> AWLWWMap.join(map, [key]) 76 | 77 | {:remove, key, _val, node_id}, map -> 78 | AWLWWMap.remove(key, node_id, map) 79 | |> AWLWWMap.join(map, [key]) 80 | end) 81 | |> AWLWWMap.read() 82 | 83 | correct_result = 84 | operations 85 | |> Enum.reduce(%{}, fn 86 | {:add, key, value, _node_id}, map -> 87 | Map.put(map, key, value) 88 | 89 | {:remove, key, _value, _node_id}, map -> 90 | Map.delete(map, key) 91 | end) 92 | 93 | assert actual_result == correct_result 94 | end 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /test/causal_crdt_test.exs: -------------------------------------------------------------------------------- 1 | defmodule CausalCrdtTest do 2 | use ExUnit.Case, async: true 3 | doctest DeltaCrdt 4 | 5 | alias DeltaCrdt.AWLWWMap 6 | 7 | describe "with context" do 8 | setup do 9 | {:ok, c1} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 10 | 11 | {:ok, c2} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 12 | 13 | {:ok, c3} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 14 | 15 | DeltaCrdt.set_neighbours(c1, [c1, c2, c3]) 16 | DeltaCrdt.set_neighbours(c2, [c1, c2, c3]) 17 | DeltaCrdt.set_neighbours(c3, [c1, c2, c3]) 18 | [c1: c1, c2: c2, c3: c3] 19 | end 20 | 21 | test "basic test case", context do 22 | DeltaCrdt.put(context.c1, "Derek", "Kraan") 23 | DeltaCrdt.put(context.c1, :Tonci, "Galic") 24 | 25 | assert %{"Derek" => "Kraan", Tonci: "Galic"} == DeltaCrdt.to_map(context.c1) 26 | end 27 | 28 | test "merge/3", context do 29 | DeltaCrdt.merge(context.c1, %{"Derek" => "Kraan", "Moose" => "Code"}) 30 | Process.sleep(100) 31 | assert %{"Derek" => "Kraan", "Moose" => "Code"} == DeltaCrdt.to_map(context.c2) 32 | end 33 | 34 | test "drop/3", context do 35 | DeltaCrdt.merge(context.c1, %{ 36 | "Netherlands" => "Amsterdam", 37 | "Belgium" => "Brussel", 38 | "Germany" => "Berlin" 39 | }) 40 | 41 | Process.sleep(100) 42 | DeltaCrdt.drop(context.c2, ["Belgium", "Germany"]) 43 | Process.sleep(100) 44 | assert %{"Netherlands" => "Amsterdam"} == DeltaCrdt.to_map(context.c1) 45 | end 46 | 47 | test "conflicting updates resolve", context do 48 | DeltaCrdt.put(context.c1, "Derek", "one_wins") 49 | DeltaCrdt.put(context.c1, "Derek", "two_wins") 50 | DeltaCrdt.put(context.c1, "Derek", "three_wins") 51 | Process.sleep(100) 52 | assert %{"Derek" => "three_wins"} == DeltaCrdt.to_map(context.c1) 53 | assert %{"Derek" => "three_wins"} == DeltaCrdt.to_map(context.c2) 54 | assert %{"Derek" => "three_wins"} == DeltaCrdt.to_map(context.c3) 55 | end 56 | 57 | test "add wins", context do 58 | DeltaCrdt.put(context.c1, "Derek", "add_wins") 59 | DeltaCrdt.delete(context.c2, "Derek") 60 | Process.sleep(100) 61 | assert %{"Derek" => "add_wins"} == DeltaCrdt.to_map(context.c1) 62 | assert %{"Derek" => "add_wins"} == DeltaCrdt.to_map(context.c2) 63 | end 64 | 65 | test "can remove", context do 66 | DeltaCrdt.put(context.c1, "Derek", "add_wins") 67 | Process.sleep(100) 68 | assert %{"Derek" => "add_wins"} == DeltaCrdt.to_map(context.c2) 69 | DeltaCrdt.delete(context.c1, "Derek") 70 | Process.sleep(100) 71 | assert %{} == DeltaCrdt.to_map(context.c1) 72 | assert %{} == DeltaCrdt.to_map(context.c2) 73 | end 74 | end 75 | 76 | test "synchronization is directional, diffs are sent TO neighbours" do 77 | {:ok, c1} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 78 | {:ok, c2} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 79 | DeltaCrdt.set_neighbours(c1, [c2]) 80 | DeltaCrdt.put(c1, "Derek", "Kraan") 81 | DeltaCrdt.put(c2, "Tonci", "Galic") 82 | Process.sleep(100) 83 | assert %{"Derek" => "Kraan"} == DeltaCrdt.to_map(c1) 84 | assert %{"Derek" => "Kraan", "Tonci" => "Galic"} == DeltaCrdt.to_map(c2) 85 | end 86 | 87 | test "can sync to neighbours specified by name" do 88 | {:ok, c1} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50, name: :neighbour_name_1) 89 | {:ok, c2} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50, name: :neighbour_name_2) 90 | DeltaCrdt.set_neighbours(c1, [:neighbour_name_2]) 91 | DeltaCrdt.set_neighbours(c2, [{:neighbour_name_1, node()}]) 92 | DeltaCrdt.put(c1, "Derek", "Kraan") 93 | DeltaCrdt.put(c2, "Tonci", "Galic") 94 | Process.sleep(100) 95 | assert %{"Derek" => "Kraan", "Tonci" => "Galic"} = DeltaCrdt.to_map(c1) 96 | assert %{"Derek" => "Kraan", "Tonci" => "Galic"} = DeltaCrdt.to_map(c2) 97 | end 98 | 99 | test "storage backend can store and retrieve state" do 100 | DeltaCrdt.start_link(AWLWWMap, storage_module: MemoryStorage, name: :storage_test) 101 | 102 | DeltaCrdt.put(:storage_test, "Derek", "Kraan") 103 | assert %{"Derek" => "Kraan"} = DeltaCrdt.to_map(:storage_test) 104 | end 105 | 106 | test "storage backend is used to rehydrate state after a crash" do 107 | task = 108 | Task.async(fn -> 109 | DeltaCrdt.start_link(AWLWWMap, storage_module: MemoryStorage, name: :storage_test) 110 | DeltaCrdt.put(:storage_test, "Derek", "Kraan") 111 | end) 112 | 113 | Task.await(task) 114 | 115 | # time for the previous process to deregister itself 116 | Process.sleep(10) 117 | 118 | {:ok, _} = DeltaCrdt.start_link(AWLWWMap, storage_module: MemoryStorage, name: :storage_test) 119 | 120 | assert %{"Derek" => "Kraan"} = DeltaCrdt.to_map(:storage_test) 121 | end 122 | 123 | test "syncs after adding neighbour" do 124 | {:ok, c1} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 125 | {:ok, c2} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 126 | DeltaCrdt.put(c1, "CRDT1", "represent") 127 | DeltaCrdt.put(c2, "CRDT2", "also here") 128 | DeltaCrdt.set_neighbours(c1, [c2]) 129 | Process.sleep(100) 130 | assert %{} = DeltaCrdt.to_map(c1) 131 | end 132 | 133 | test "can sync after network partition" do 134 | {:ok, c1} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 135 | 136 | {:ok, c2} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 50) 137 | 138 | DeltaCrdt.set_neighbours(c1, [c2]) 139 | DeltaCrdt.set_neighbours(c2, [c1]) 140 | 141 | DeltaCrdt.put(c1, "CRDT1", "represent") 142 | 143 | DeltaCrdt.put(c2, "CRDT2", "also here") 144 | 145 | Process.sleep(200) 146 | assert %{"CRDT1" => "represent", "CRDT2" => "also here"} = DeltaCrdt.to_map(c1) 147 | 148 | # uncouple them 149 | DeltaCrdt.set_neighbours(c1, []) 150 | DeltaCrdt.set_neighbours(c2, []) 151 | 152 | DeltaCrdt.put(c1, "CRDTa", "only present in 1") 153 | DeltaCrdt.put(c1, "CRDTb", "only present in 1") 154 | DeltaCrdt.delete(c1, "CRDT1") 155 | 156 | Process.sleep(200) 157 | 158 | assert Map.has_key?(DeltaCrdt.to_map(c1), "CRDTa") 159 | refute Map.has_key?(DeltaCrdt.to_map(c2), "CRDTa") 160 | 161 | # make them neighbours again 162 | DeltaCrdt.set_neighbours(c1, [c2]) 163 | DeltaCrdt.set_neighbours(c2, [c1]) 164 | 165 | Process.sleep(200) 166 | 167 | assert Map.has_key?(DeltaCrdt.to_map(c1), "CRDTa") 168 | refute Map.has_key?(DeltaCrdt.to_map(c1), "CRDT1") 169 | assert Map.has_key?(DeltaCrdt.to_map(c2), "CRDTa") 170 | refute Map.has_key?(DeltaCrdt.to_map(c2), "CRDT1") 171 | end 172 | 173 | test "syncing when values happen to be the same" do 174 | {:ok, c1} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 20) 175 | {:ok, c2} = DeltaCrdt.start_link(AWLWWMap, sync_interval: 20) 176 | DeltaCrdt.set_neighbours(c1, [c2]) 177 | DeltaCrdt.set_neighbours(c2, [c1]) 178 | 179 | DeltaCrdt.put(c1, "key", "value") 180 | DeltaCrdt.put(c2, "key", "value") 181 | 182 | Process.sleep(50) 183 | 184 | DeltaCrdt.delete(c1, "key") 185 | 186 | Process.sleep(50) 187 | 188 | refute Map.has_key?(DeltaCrdt.to_map(c1), "key") 189 | refute Map.has_key?(DeltaCrdt.to_map(c2), "key") 190 | end 191 | 192 | test "can read a single key" do 193 | {:ok, c} = DeltaCrdt.start_link(AWLWWMap) 194 | 195 | DeltaCrdt.put(c, "key1", "value1") 196 | DeltaCrdt.put(c, "key2", "value2") 197 | 198 | assert %{"key1" => "value1"} == DeltaCrdt.read(c, ~w[key1]) 199 | assert "value1" == DeltaCrdt.get(c, "key1") 200 | end 201 | 202 | test "can read multiple keys" do 203 | {:ok, c} = DeltaCrdt.start_link(AWLWWMap) 204 | 205 | DeltaCrdt.put(c, "key1", "value1") 206 | DeltaCrdt.put(c, "key2", "value2") 207 | DeltaCrdt.put(c, "key3", "value3") 208 | 209 | assert %{"key1" => "value1", "key3" => "value3"} == DeltaCrdt.read(c, ~w[key1 key3]) 210 | assert %{"key1" => "value1", "key3" => "value3"} == DeltaCrdt.take(c, ~w[key1 key3]) 211 | end 212 | end 213 | -------------------------------------------------------------------------------- /test/delta_subscriber_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DeltaSubscriberTest do 2 | use ExUnit.Case, async: true 3 | use ExUnitProperties 4 | 5 | alias DeltaCrdt.AWLWWMap 6 | 7 | def on_diffs(test_pid, diffs) do 8 | send(test_pid, {:diff, diffs}) 9 | end 10 | 11 | test "receives deltas updates with MFA" do 12 | test_pid = self() 13 | 14 | {:ok, c1} = 15 | DeltaCrdt.start_link(AWLWWMap, 16 | sync_interval: 50, 17 | on_diffs: {DeltaSubscriberTest, :on_diffs, [test_pid]} 18 | ) 19 | 20 | ^c1 = DeltaCrdt.put(c1, "Derek", "Kraan") 21 | assert_received({:diff, [{:add, "Derek", "Kraan"}]}) 22 | 23 | ^c1 = DeltaCrdt.put(c1, "Derek", "Kraan") 24 | refute_received({:diff, [{:add, "Derek", "Kraan"}]}) 25 | 26 | ^c1 = DeltaCrdt.put(c1, "Derek", nil) 27 | assert_received({:diff, [{:remove, "Derek"}]}) 28 | end 29 | 30 | test "receives deltas updates with function" do 31 | test_pid = self() 32 | 33 | {:ok, c1} = 34 | DeltaCrdt.start_link(AWLWWMap, 35 | sync_interval: 50, 36 | on_diffs: fn diffs -> send(test_pid, {:diff, diffs}) end 37 | ) 38 | 39 | ^c1 = DeltaCrdt.put(c1, "Derek", "Kraan") 40 | assert_received({:diff, [{:add, "Derek", "Kraan"}]}) 41 | 42 | ^c1 = DeltaCrdt.put(c1, "Derek", "Kraan") 43 | refute_received({:diff, [{:add, "Derek", "Kraan"}]}) 44 | 45 | ^c1 = DeltaCrdt.put(c1, "Derek", nil) 46 | assert_received({:diff, [{:remove, "Derek"}]}) 47 | end 48 | 49 | test "updates are bundled" do 50 | {:ok, c1} = 51 | DeltaCrdt.start_link(AWLWWMap, 52 | sync_interval: 50 53 | ) 54 | 55 | test_pid = self() 56 | 57 | {:ok, c2} = 58 | DeltaCrdt.start_link(AWLWWMap, 59 | sync_interval: 50, 60 | on_diffs: {DeltaSubscriberTest, :on_diffs, [test_pid]} 61 | ) 62 | 63 | ^c1 = DeltaCrdt.put(c1, "Derek", "Kraan") 64 | ^c1 = DeltaCrdt.put(c1, "Andrew", "Kraan") 65 | ^c1 = DeltaCrdt.put(c1, "Nathan", "Kraan") 66 | 67 | DeltaCrdt.set_neighbours(c1, [c2]) 68 | DeltaCrdt.set_neighbours(c2, [c1]) 69 | 70 | assert_receive({:diff, diff}, 100) 71 | 72 | assert Map.new(diff, fn {:add, k, v} -> {k, v} end) == %{ 73 | "Derek" => "Kraan", 74 | "Andrew" => "Kraan", 75 | "Nathan" => "Kraan" 76 | } 77 | end 78 | 79 | property "add and remove operations result in correct map" do 80 | op = 81 | ExUnitProperties.gen all( 82 | op <- StreamData.member_of([:add, :remove]), 83 | key <- term(), 84 | value <- term() 85 | ) do 86 | case op do 87 | :add -> {:add, key, value} 88 | :remove -> {:remove, key} 89 | end 90 | end 91 | 92 | check all(ops <- list_of(op)) do 93 | test_pid = self() 94 | 95 | {:ok, c1} = 96 | DeltaCrdt.start_link(AWLWWMap, 97 | sync_interval: 50, 98 | on_diffs: {DeltaSubscriberTest, :on_diffs, [test_pid]} 99 | ) 100 | 101 | Enum.each(ops, fn 102 | {:add, k, v} -> 103 | DeltaCrdt.put(c1, k, v) 104 | 105 | {:remove, k} -> 106 | DeltaCrdt.delete(c1, k) 107 | end) 108 | 109 | out = 110 | Enum.reduce(ops, %{}, fn 111 | {:add, k, v}, map -> Map.put(map, k, v) 112 | {:remove, k}, map -> Map.delete(map, k) 113 | end) 114 | 115 | assert out == construct_map() 116 | end 117 | end 118 | 119 | defp construct_map(map \\ %{}) do 120 | receive do 121 | {:diff, diffs} -> 122 | Enum.reduce(diffs, map, fn 123 | {:add, k, v}, map -> 124 | Map.put(map, k, v) 125 | 126 | {:remove, k}, map -> 127 | Map.delete(map, k) 128 | end) 129 | |> construct_map() 130 | after 131 | 50 -> map 132 | end 133 | end 134 | end 135 | -------------------------------------------------------------------------------- /test/support/awlww_map_property.ex: -------------------------------------------------------------------------------- 1 | defmodule AWLWWMapProperty do 2 | use ExUnitProperties 3 | alias DeltaCrdt.{AWLWWMap} 4 | 5 | def add_operation do 6 | ExUnitProperties.gen all key <- binary(), 7 | val <- binary(), 8 | node_id <- integer() do 9 | fn state -> AWLWWMap.add(key, val, node_id, state) end 10 | end 11 | end 12 | 13 | def remove_operation do 14 | ExUnitProperties.gen all key <- binary(), 15 | node_id <- integer() do 16 | fn state -> AWLWWMap.remove(key, node_id, state) end 17 | end 18 | end 19 | 20 | def random_operation do 21 | ExUnitProperties.gen all operation <- one_of([:add, :remove]), 22 | key <- binary(), 23 | val <- binary(), 24 | node_id <- integer() do 25 | case operation do 26 | :add -> 27 | fn 28 | state -> AWLWWMap.add(key, val, node_id, state) 29 | end 30 | 31 | :remove -> 32 | fn state -> AWLWWMap.remove(key, node_id, state) end 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/support/memory_storage.ex: -------------------------------------------------------------------------------- 1 | defmodule MemoryStorage do 2 | @behaviour DeltaCrdt.Storage 3 | def start_link() do 4 | GenServer.start_link(__MODULE__, [], name: __MODULE__) 5 | end 6 | 7 | def write(name, state) do 8 | GenServer.call(__MODULE__, {:write, name, state}) 9 | end 10 | 11 | def read(name) do 12 | GenServer.call(__MODULE__, {:read, name}) 13 | end 14 | 15 | def init(_) do 16 | {:ok, %{}} 17 | end 18 | 19 | def handle_call({:write, name, state}, _from, map) do 20 | {:reply, :ok, Map.put(map, name, state)} 21 | end 22 | 23 | def handle_call({:read, name}, _from, map) do 24 | {:reply, Map.get(map, name), map} 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | MemoryStorage.start_link() 2 | ExUnit.start() 3 | --------------------------------------------------------------------------------