├── test ├── test_helper.exs └── distributed_test_test.exs ├── sys.config ├── mix.lock ├── .gitignore ├── lib ├── server_supervisor.ex ├── node_monitor.ex ├── distributed_test.ex └── server.ex ├── mix.exs ├── config └── config.exs └── README.md /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /sys.config: -------------------------------------------------------------------------------- 1 | [{kernel, 2 | [ 3 | {sync_nodes_optional, ['n1@127.0.0.1', 'n2@127.0.0.1', 'n3@127.0.0.1']}, 4 | {sync_nodes_timeout, 10000} 5 | ]} 6 | ]. 7 | -------------------------------------------------------------------------------- /test/distributed_test_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DistributedTestTest do 2 | use ExUnit.Case 3 | doctest DistributedTest 4 | 5 | test "the truth" do 6 | assert 1 + 1 == 2 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"bunt": {:hex, :bunt, "0.1.6", "5d95a6882f73f3b9969fdfd1953798046664e6f77ec4e486e6fafc7caad97c6f", [:mix], []}, 2 | "credo": {:hex, :credo, "0.4.13", "0e9d0479c7e0591e36b093bf17fbdde012e2a69b5571e2df8d0b75b4fc8adc74", [:mix], [{:bunt, "~> 0.1.6", [hex: :bunt, optional: false]}]}} 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc 12 | 13 | # If the VM crashes, it generates a dump, let's ignore it too. 14 | erl_crash.dump 15 | 16 | # Also ignore archive artifacts (built via "mix archive.build"). 17 | *.ez 18 | -------------------------------------------------------------------------------- /lib/server_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule DistributedTest.ServerSupervisor do 2 | @moduledoc """ 3 | Main worker supervisor. It starts the `DistributedTest.Server` worker. 4 | """ 5 | 6 | use Supervisor 7 | 8 | alias DistributedTest.Server 9 | 10 | def start_link, do: Supervisor.start_link(__MODULE__, :ok, name: __MODULE__) 11 | 12 | def init(:ok) do 13 | children = [worker(Server, [], restart: :transient)] 14 | supervise(children, strategy: :simple_one_for_one) 15 | end 16 | 17 | def start_worker, do: Supervisor.start_child(__MODULE__, []) 18 | end 19 | -------------------------------------------------------------------------------- /lib/node_monitor.ex: -------------------------------------------------------------------------------- 1 | defmodule DistributedTest.NodeMonitor do 2 | @moduledoc """ 3 | Monitors the nodes and notifies about any node joining or leaving the cluster. 4 | If a node leaves, it calls the `ServerSupervisor.start_worker` function. 5 | """ 6 | 7 | alias DistributedTest.ServerSupervisor 8 | 9 | require Logger 10 | 11 | def start_link do 12 | {:ok, spawn_link fn -> 13 | :global_group.monitor_nodes true 14 | 15 | monitor() 16 | end} 17 | end 18 | 19 | defp monitor do 20 | receive do 21 | {:nodeup, node} -> 22 | Logger.info "---- Node #{node} joined" 23 | 24 | {:nodedown, node} -> 25 | Logger.info "---- Node #{node} left" 26 | 27 | ServerSupervisor.start_worker 28 | end 29 | 30 | monitor() 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule DistributedTest.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :distributed_test, 6 | version: "0.1.0", 7 | elixir: "~> 1.3", 8 | build_embedded: Mix.env == :prod, 9 | start_permanent: Mix.env == :prod, 10 | deps: deps()] 11 | end 12 | 13 | # Configuration for the OTP application 14 | # 15 | # Type "mix help compile.app" for more information 16 | def application do 17 | [applications: [:logger], 18 | mod: {DistributedTest, []}] 19 | end 20 | 21 | # Dependencies can be Hex packages: 22 | # 23 | # {:mydep, "~> 0.3.0"} 24 | # 25 | # Or git/path repositories: 26 | # 27 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1.0"} 28 | # 29 | # Type "mix help deps" for more examples and options 30 | defp deps do 31 | [{:credo, "~> 0.4", only: [:dev, :test]}] 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/distributed_test.ex: -------------------------------------------------------------------------------- 1 | defmodule DistributedTest do 2 | use Application 3 | 4 | alias DistributedTest.{NodeMonitor, Server, ServerSupervisor} 5 | 6 | # See http://elixir-lang.org/docs/stable/elixir/Application.html 7 | # for more information on OTP Applications 8 | def start(_type, _args) do 9 | import Supervisor.Spec, warn: false 10 | 11 | # Define workers and child supervisors to be supervised 12 | children = [ 13 | # Starts a worker by calling: DistributedTest.Worker.start_link(arg1, arg2, arg3) 14 | # worker(DistributedTest.Worker, [arg1, arg2, arg3]), 15 | supervisor(ServerSupervisor, []), 16 | worker(NodeMonitor, []), 17 | worker(Server, []) 18 | ] 19 | 20 | # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html 21 | # for other strategies and supported options 22 | opts = [strategy: :one_for_one, name: DistributedTest.Supervisor] 23 | Supervisor.start_link(children, opts) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/server.ex: -------------------------------------------------------------------------------- 1 | defmodule DistributedTest.Server do 2 | @moduledoc """ 3 | Main worker which performs a task every @time. It's registerd 4 | globally accross all the connected nodes. 5 | """ 6 | 7 | use GenServer 8 | 9 | require Logger 10 | 11 | @time 2_000 12 | 13 | def start_link() do 14 | case GenServer.start_link(__MODULE__, %{}, name: {:global, __MODULE__}) do 15 | {:ok, pid} -> 16 | Logger.info "---- #{__MODULE__} worker started" 17 | {:ok, pid} 18 | {:error, {:already_started, pid}} -> 19 | Logger.info "---- #{__MODULE__} worker already running" 20 | {:ok, pid} 21 | end 22 | end 23 | 24 | def init(state) do 25 | schedule_work() 26 | 27 | {:ok, state} 28 | end 29 | 30 | def handle_info(:work, state) do 31 | Logger.info "---- Working..." 32 | 33 | schedule_work() 34 | 35 | {:noreply, state} 36 | end 37 | 38 | defp schedule_work() do 39 | Process.send_after(self(), :work, @time) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # 3rd-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure for your application as: 12 | # 13 | # config :distributed_test, key: :value 14 | # 15 | # And access this configuration in your application as: 16 | # 17 | # Application.get_env(:distributed_test, :key) 18 | # 19 | # Or configure a 3rd-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | # import_config "#{Mix.env}.exs" 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Distributed supervisors test 2 | 3 | ## Background 4 | We need a worker which performs a task periodically. 5 | 6 | ## What we want 7 | If we distribute the application in multiple nodes, we want the worker process to be running just once. 8 | If the process dies, we want the current node to start it again. 9 | If the node running the process dies, we want a different node to take care of restarting the worker again. 10 | 11 | ## Alternative solutions 12 | - https://github.com/bigardone/elixir-distributed-supervisors-test/tree/princemaple-version by [princemaple](https://github.com/princemaple) 13 | 14 | ## Original solution 15 | We have 3 different components: 16 | 17 | - The `DistributedTest.Server` worker, which is registered globally. 18 | - The `DistributedTest.ServerSupervisor` which tries to start the previous worker. 19 | - The `DistributedTest.NodeMonitor` which checks for nodes dying. 20 | 21 | 22 | The application starts the three of them. 23 | When the `DistributedTest.Server` gets started, it will return the new pid or the existing one, if it was started on a different node, and it will begin doing its periodical task. 24 | The `DistributedTest.NodeMonitor`, monitors the existing nodes, and in case one dies, it tries to start a new `DistributedTest.Server` process using the `DistributedTest.ServerSupervisor`. 25 | 26 | ## Test 27 | Open three different terminal windows and start one different node in each of them: 28 | 29 | ``` 30 | $ iex --name n1@127.0.0.1 --erl "-config sys.config" -S mix 31 | $ iex --name n2@127.0.0.1 --erl "-config sys.config" -S mix 32 | $ iex --name n3@127.0.0.1 --erl "-config sys.config" -S mix 33 | ``` 34 | 35 | Try to kill the `DistributedTest.Server` process and watch it restart again. Finish the node session where it is currently running, and watch how any of the other nodes starts the process again. 36 | 37 | Feedback and suggestions are welcomed :) 38 | 39 | 40 | ![http://g.recordit.co/rbxRoxoHXc.gif](http://g.recordit.co/rbxRoxoHXc.gif) 41 | --------------------------------------------------------------------------------