├── VERSION ├── test ├── test_helper.exs ├── graceful_stop │ ├── application_test.exs │ └── handler_test.exs └── graceful_stop_test.exs ├── .formatter.exs ├── .travis.yml ├── lib ├── graceful_stop │ ├── application.ex │ └── handler.ex └── graceful_stop.ex ├── config └── config.exs ├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── mix.exs ├── mix.lock └── README.md /VERSION: -------------------------------------------------------------------------------- 1 | 0.2.2 -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /test/graceful_stop/application_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GracefulStop.ApplicationTest do 2 | use ExUnit.Case 3 | end 4 | -------------------------------------------------------------------------------- /test/graceful_stop_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GracefulStopTest do 2 | use ExUnit.Case 3 | doctest GracefulStop 4 | end 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | elixir: 3 | - 1.6.6 4 | otp_release: 5 | - 20.0 6 | - 21.0 7 | script: MIX_ENV=test mix compile --warnings-as-errors && MIX_ENV=test mix test 8 | -------------------------------------------------------------------------------- /lib/graceful_stop/application.ex: -------------------------------------------------------------------------------- 1 | defmodule GracefulStop.Application do 2 | use Application 3 | 4 | alias GracefulStop.Handler 5 | 6 | def start(_type, _args) do 7 | children = [Handler] 8 | opts = [strategy: :one_for_one, name: GracefulStop.Supervisor] 9 | Supervisor.start_link(children, opts) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/graceful_stop.ex: -------------------------------------------------------------------------------- 1 | defmodule GracefulStop do 2 | @moduledoc """ 3 | Documentation for GracefulStop. 4 | """ 5 | 6 | def stop do 7 | GracefulStop.Handler.system_stop() 8 | end 9 | 10 | @spec get_status() :: :running | :stopping 11 | def get_status do 12 | GracefulStop.Handler.get_status() 13 | end 14 | 15 | end 16 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | import Config 4 | 5 | # How long hook functions can run before they are :brutal_kill'ed 6 | config :graceful_stop, :hook_timeout, 15_000 7 | 8 | # List of MFAs to run on graceful stop, for instance: 9 | # config :graceful_stop, :hooks, [ [IO, :puts, ["Stopping the system"] ] ] 10 | config :graceful_stop, :hooks, [] 11 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: push 2 | 3 | name: test 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | name: OTP ${{matrix.otp}} / Elixir ${{matrix.elixir}} 9 | env: 10 | MIX_ENV: test 11 | strategy: 12 | matrix: 13 | otp: ["24.0"] 14 | elixir: ["1.14"] 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: erlef/setup-elixir@v1 18 | with: 19 | otp-version: ${{matrix.otp}} 20 | elixir-version: ${{matrix.elixir}} 21 | - run: mix deps.get 22 | - run: mix compile --warnings-as-errors 23 | - run: mix test 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | graceful_stop-*.tar 24 | 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Arjan Scherpenisse 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule GracefulStop.MixProject do 2 | use Mix.Project 3 | 4 | @source_url "https://github.com/botsquad/graceful_stop" 5 | @version File.read!("VERSION") 6 | 7 | def project do 8 | [ 9 | app: :graceful_stop, 10 | version: @version, 11 | elixir: "~> 1.11", 12 | description: description(), 13 | package: package(), 14 | source_url: @source_url, 15 | homepage_url: @source_url, 16 | start_permanent: Mix.env() == :prod, 17 | deps: deps() 18 | ] 19 | end 20 | 21 | defp description do 22 | "Gracefully stop the system after running shutdown hooks. Also catches SIGTERM." 23 | end 24 | 25 | defp package do 26 | %{ 27 | files: ["lib", "mix.exs", "*.md", "LICENSE", "VERSION"], 28 | maintainers: ["Arjan Scherpenisse"], 29 | licenses: ["MIT"], 30 | links: %{"GitHub" => "https://github.com/botsquad/graceful_stop"} 31 | } 32 | end 33 | 34 | # Run "mix help compile.app" to learn about applications. 35 | def application do 36 | [ 37 | mod: {GracefulStop.Application, []}, 38 | extra_applications: [:logger] 39 | ] 40 | end 41 | 42 | # Run "mix help deps" to learn about dependencies. 43 | defp deps do 44 | [ 45 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false} 46 | ] 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "earmark_parser": {:hex, :earmark_parser, "1.4.32", "fa739a0ecfa34493de19426681b23f6814573faee95dfd4b4aafe15a7b5b32c6", [:mix], [], "hexpm", "b8b0dd77d60373e77a3d7e8afa598f325e49e8663a51bcc2b88ef41838cca755"}, 3 | "ex_doc": {:hex, :ex_doc, "0.29.4", "6257ecbb20c7396b1fe5accd55b7b0d23f44b6aa18017b415cb4c2b91d997729", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "2c6699a737ae46cb61e4ed012af931b57b699643b24dabe2400a8168414bc4f5"}, 4 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 5 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 6 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, 7 | "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, 8 | } 9 | -------------------------------------------------------------------------------- /test/graceful_stop/handler_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GracefulStop.HandlerTest do 2 | use ExUnit.Case 3 | 4 | alias GracefulStop.Handler 5 | 6 | setup do 7 | Process.register(self(), GracefulStop.Test) 8 | :ok 9 | end 10 | 11 | test "handler installed" do 12 | assert Process.whereis(:erl_signal_server) 13 | assert Handler in :gen_event.which_handlers(:erl_signal_server) 14 | end 15 | 16 | test "handler responds to get_status command" do 17 | assert :running = Handler.get_status() 18 | end 19 | 20 | test "system_stop calls :init.stop()" do 21 | Handler.system_stop() 22 | assert_receive :init_stop 23 | end 24 | 25 | test "system_stop calls the hooks" do 26 | Application.put_env(:graceful_stop, :hooks, [ 27 | [__MODULE__, :hook, [:hook1]], 28 | [__MODULE__, :hook, [:hook2]] 29 | ]) 30 | 31 | Handler.system_stop() 32 | assert_receive :hook1 33 | assert_receive :hook2 34 | assert_receive :init_stop 35 | end 36 | 37 | test "handler status = :stopping while system is stopping" do 38 | Application.put_env(:graceful_stop, :hooks, [[__MODULE__, :hook, [:hook1, 50]]]) 39 | Handler.system_stop() 40 | Process.sleep(10) 41 | assert :stopping = Handler.get_status() 42 | assert_receive :hook1 43 | assert_receive :init_stop 44 | end 45 | 46 | test "system_stop kills hooks that are taking too long" do 47 | Application.put_env(:graceful_stop, :hook_timeout, 100) 48 | 49 | Application.put_env(:graceful_stop, :hooks, [ 50 | [__MODULE__, :hook, [:slow, 500]], 51 | [__MODULE__, :hook, [:slower, 600]] 52 | ]) 53 | 54 | Handler.system_stop() 55 | refute_receive :slow, 1000 56 | refute_receive :slower, 1000 57 | assert_receive :init_stop 58 | Application.put_env(:graceful_stop, :hooks, []) 59 | end 60 | 61 | test "sigterm initiates system stop" do 62 | :gen_event.notify(:erl_signal_server, :sigterm) 63 | assert_receive :init_stop 64 | end 65 | 66 | def hook(arg, wait \\ 0) do 67 | Process.sleep(wait) 68 | send(GracefulStop.Test, arg) 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/graceful_stop/handler.ex: -------------------------------------------------------------------------------- 1 | defmodule GracefulStop.Handler do 2 | require Logger 3 | 4 | @name :erl_signal_server 5 | @table Module.concat(__MODULE__, Table) 6 | 7 | def child_spec(opts) do 8 | %{ 9 | id: __MODULE__, 10 | start: {__MODULE__, :start_link, [opts]}, 11 | type: :worker, 12 | restart: :permanent, 13 | shutdown: 500 14 | } 15 | end 16 | 17 | def start_link(_) do 18 | :ok = 19 | :gen_event.swap_sup_handler( 20 | @name, 21 | {:erl_signal_handler, []}, 22 | {__MODULE__, []} 23 | ) 24 | 25 | :ignore 26 | end 27 | 28 | def get_status() do 29 | [{:status, status}] = :ets.lookup(@table, :status) 30 | status 31 | end 32 | 33 | def system_stop() do 34 | :gen_event.notify(@name, :sigterm) 35 | end 36 | 37 | ## gen_event 38 | 39 | defmodule State do 40 | defstruct timeout: nil, 41 | hooks: nil 42 | end 43 | 44 | def init({_, :ok}) do 45 | :ets.new(@table, [:named_table, :public, read_concurrency: true]) 46 | {:ok, %State{} |> set_status(:running)} 47 | end 48 | 49 | def handle_event(:sigterm, state) do 50 | spawn_link(&perform_stop/0) 51 | {:ok, state |> set_status(:stopping)} 52 | end 53 | 54 | def handle_info(:resume, state) do 55 | {:ok, state |> set_status(:running)} 56 | end 57 | def handle_info(_message, state) do 58 | # Logger.info "handle_info: #{inspect message}" 59 | {:ok, state} 60 | end 61 | 62 | defp perform_stop() do 63 | Logger.debug("Initiating graceful stop…") 64 | 65 | hooks = Application.get_env(:graceful_stop, :hooks, []) 66 | timeout = Application.get_env(:graceful_stop, :hook_timeout, 15_000) 67 | 68 | hooks 69 | |> log_hooks() 70 | |> Enum.map(&Kernel.apply(Task, :async, &1)) 71 | |> Task.yield_many(timeout) 72 | |> Enum.zip(hooks) 73 | |> kill_slow_hooks() 74 | 75 | Logger.debug("Calling :init.stop()") 76 | init_stop() 77 | end 78 | 79 | defp log_hooks([]), do: [] 80 | 81 | defp log_hooks(hooks) do 82 | Logger.debug("Calling #{Enum.count(hooks)} shutdown hooks") 83 | hooks 84 | end 85 | 86 | defp kill_slow_hooks(tasks_with_results) do 87 | Enum.map(tasks_with_results, fn {{task, res}, hook} -> 88 | if res == nil do 89 | Logger.warning("Killing slow shutdown task #{inspect(hook)}") 90 | Task.shutdown(task, :brutal_kill) 91 | end 92 | end) 93 | end 94 | 95 | defp set_status(state, status) do 96 | :ets.delete_all_objects(@table) 97 | :ets.insert(@table, {:status, status}) 98 | state 99 | end 100 | 101 | if Mix.env() == :test do 102 | def init_stop() do 103 | send(GracefulStop.Test, :init_stop) 104 | send(@name, :resume) 105 | end 106 | else 107 | def init_stop() do 108 | :init.stop() 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GracefulStop 2 | 3 | [![Build Status](https://github.com/botsquad/graceful_stop/workflows/test/badge.svg)](https://github.com/botsquad/graceful_stop) 4 | [![Module Version](https://img.shields.io/hexpm/v/graceful_stop.svg)](https://hex.pm/packages/graceful_stop) 5 | [![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/graceful_stop/) 6 | [![Total Download](https://img.shields.io/hexpm/dt/graceful_stop.svg)](https://hex.pm/packages/graceful_stop) 7 | [![License](https://img.shields.io/hexpm/l/graceful_stop.svg)](https://github.com/botsquad/graceful_stop/blob/main/LICENSE) 8 | [![Last Updated](https://img.shields.io/github/last-commit/botsquad/graceful_stop.svg)](https://github.com/botsquad/graceful_stop/commits/main) 9 | 10 | Gracefully calls `:init.stop()` after running user-configured shutdown 11 | hooks. 12 | 13 | Also catches `SIGTERM` signal to gracefully stop the system and run 14 | the shutdown hooks. 15 | 16 | When running in a Kubernetes-managed cluster, nodes in the cluster 17 | come and go as kubernetes decides. It sends the SIGTERM signal, which 18 | by default triggers a `:init.stop()`. However, you might want to give 19 | the system some time to shut down, running cleanup processes, wait for 20 | running requests to finish, et cetera. 21 | 22 | ## Usage 23 | 24 | After adding `:graceful_stop` to your deps, you can configure it to 25 | call hooks when the application will stop: 26 | 27 | ``` 28 | config :graceful_stop, :hooks, [ 29 | [IO, :puts, ["Stopping the system"]] 30 | ] 31 | ``` 32 | 33 | Then: `kill $(pidof beam.smp)` sends a `SIGTERM` signal to your 34 | running BEAM process, and you will notice that you see "Stopping the 35 | system" printed on the console, before it shuts down. 36 | 37 | Note that these hooks run _before_ any of your OTP applications are 38 | being stopped, so you can do all kinds of things there, without 39 | worrying that parts of your system are already shut down (which would 40 | be the case if you try to trap the `{:EXIT, pid, :shutdown}` message). 41 | 42 | There is a `:hook_timeout` setting, defaulting to 15 seconds, which is 43 | the maximum time that a hook can run. Hooks run in parallel, using 44 | `Task.async` / `Task.yield_many`. 45 | 46 | ## Inspiration 47 | 48 | This project was inspired by the 49 | [k8s_traffic_plug](https://github.com/Financial-Times/k8s_traffic_plug) 50 | package and the corresponding [blog 51 | post](https://medium.com/@ellispritchard/graceful-shutdown-on-kubernetes-with-signals-erlang-otp-20-a22325e8ae98). 52 | However, it does not include a Plug. Creating a plug is simple, as you 53 | can call `GracefulStop.get_status()` which returns either `:running` 54 | or `:stopping`, and you can create a plug that serves a HTTP 503 55 | request based on this code. 56 | 57 | ### Phoenix Plug implementation example 58 | 59 | Mount this plug inside your [Phoenix Endpoint](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html) before the router plug is mounted. 60 | **DO NOT** mount this plug inside your router file. 61 | 62 | ```elixir 63 | defmodule MyAppWeb.Endpoint do 64 | use Phoenix.Endpoint, otp_app: :myapp 65 | 66 | .... 67 | 68 | plug MyAppWeb.Plug.TrafficDrain 69 | plug MyAppWeb.Router 70 | ``` 71 | 72 | Reference implementation of `TrafficDrain` plug. 73 | 74 | ```elixir 75 | defmodule MyAppWeb.Plug.TrafficDrain do 76 | @moduledoc """ 77 | Plug for handling Kubernetes readinessProbe. 78 | 79 | Plug starts responding with 503 - Service Unavailable from `/__traffic`, when traffic is being drained. 80 | Otherwise we respond with 200 - OK. 81 | """ 82 | 83 | import Plug.Conn 84 | 85 | @behaviour Plug 86 | 87 | @impl true 88 | def init(opts), do: opts 89 | 90 | @impl true 91 | def call(%Plug.Conn{path_info: ["__traffic"]} = conn, _opts) do 92 | case GracefulStop.get_status() do 93 | :stopping -> 94 | conn 95 | |> put_resp_content_type("text/plain") 96 | |> send_resp(:service_unavailable, "Draining") 97 | |> halt() 98 | 99 | :running -> 100 | conn 101 | |> put_resp_content_type("text/plain") 102 | |> send_resp(:ok, "Serving") 103 | |> halt() 104 | end 105 | end 106 | 107 | @impl true 108 | def call(conn, _opts) do 109 | conn 110 | end 111 | end 112 | 113 | ``` 114 | 115 | ## Installation 116 | 117 | If [available in Hex](https://hex.pm/docs/publish), the package can be installed 118 | by adding `graceful_stop` to your list of dependencies in `mix.exs`: 119 | 120 | ```elixir 121 | def deps do 122 | [ 123 | {:graceful_stop, "~> 0.1.0"} 124 | ] 125 | end 126 | ``` 127 | --------------------------------------------------------------------------------