├── .formatter.exs ├── .gitignore ├── README.md ├── lib └── off_broadway_pgmq.ex ├── mix.exs ├── mix.lock └── test ├── off_broadway_pgmq_test.exs └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | off_broadway_pgmq-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OffBroadwayPgmq 2 | 3 | **TODO: Add description** 4 | 5 | ## Installation 6 | 7 | If [available in Hex](https://hex.pm/docs/publish), the package can be installed 8 | by adding `off_broadway_pgmq` to your list of dependencies in `mix.exs`: 9 | 10 | ```elixir 11 | def deps do 12 | [ 13 | {:off_broadway_pgmq, "~> 0.1.0"} 14 | ] 15 | end 16 | ``` 17 | 18 | Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) 19 | and published on [HexDocs](https://hexdocs.pm). Once published, the docs can 20 | be found at . 21 | 22 | -------------------------------------------------------------------------------- /lib/off_broadway_pgmq.ex: -------------------------------------------------------------------------------- 1 | defmodule OffBroadwayPgmq do 2 | @moduledoc """ 3 | Pgmq producer for broadway, adapted from BroadwaySQS 4 | 5 | The producer receives 4 options: 6 | - `:repo`: the ecto repo to be used, mandatory. 7 | - `:dynamic_repo`: dynamic repo to be used, optional. 8 | - `:queue`: the queue name to be used, mandatory. 9 | - `:visibility_timeout`: the time the messages will be unavailable in the queue 10 | after being read. Required 11 | - `:max_poll_seconds`: how long the maximum poll request takes, optional, defaults 12 | to 5 seconds. 13 | - `:attempt_interval_ms`: interval in ms to wait before doing poll requests in 14 | case there is demand but no messages are found. Optional, defaults to 500 15 | - `:pgmq_poll_interval_ms`: option on pgmq side that dictates poll interval 16 | postgres-side. Optional, defaults to 250. 17 | 18 | If you're using many queues, this can be a bit heavy in your connection pool, 19 | so its important to configure properly. You might want to adjust `max_poll_seconds` 20 | and `:attempt_interval_ms` to trade off connection usage for more latency. You can 21 | also use a `:max_poll_seconds` of 0 to perform no polling at all. 22 | 23 | `:pgmq_poll_interval_ms` is the database side poll interval. By adjusting it, 24 | you can increase or decrease the amount of work performed database side at 25 | the risk of getting more latency. 26 | """ 27 | 28 | use GenStage 29 | 30 | @default_max_poll_seconds 5 31 | @default_pgmq_poll_interval_ms 250 32 | @default_attempt_interval_ms 500 33 | 34 | alias Broadway.Producer 35 | alias Broadway.Acknowledger 36 | 37 | @behaviour Producer 38 | @behaviour Acknowledger 39 | 40 | @impl GenStage 41 | def init(opts) do 42 | repo = Keyword.fetch!(opts, :repo) 43 | queue = Keyword.fetch!(opts, :queue) 44 | visibility_timeout = Keyword.fetch!(opts, :visibility_timeout) 45 | max_poll_seconds = Keyword.get(opts, :max_poll_seconds, @default_max_poll_seconds) 46 | poll_interval_ms = Keyword.get(opts, :db_poll_interval_ms, @default_pgmq_poll_interval_ms) 47 | attempt_interval_ms = Keyword.get(opts, :attempt_interval_ms, @default_attempt_interval_ms) 48 | maximum_failures = Keyword.get(opts, :maximum_failures, 10) 49 | dynamic_repo = Keyword.get(opts, :dynamic_repo) 50 | 51 | if dynamic_repo do 52 | repo.put_dynamic_repo(dynamic_repo) 53 | end 54 | 55 | {:producer, 56 | %{ 57 | demand: 0, 58 | receive_timer: nil, 59 | receive_interval: attempt_interval_ms, 60 | visibility_timeout: visibility_timeout, 61 | repo: repo, 62 | dynamic_repo: dynamic_repo, 63 | queue: queue, 64 | max_poll_seconds: max_poll_seconds, 65 | poll_interval_ms: poll_interval_ms, 66 | maximum_failures: maximum_failures 67 | }} 68 | end 69 | 70 | @impl GenStage 71 | def handle_demand(incoming_demand, %{demand: demand} = state) do 72 | handle_receive_messages(%{state | demand: demand + incoming_demand}) 73 | end 74 | 75 | @impl GenStage 76 | def handle_info(:receive_messages, %{receive_timer: nil} = state) do 77 | {:noreply, [], state} 78 | end 79 | 80 | @impl GenStage 81 | def handle_info(:receive_messages, state) do 82 | handle_receive_messages(%{state | receive_timer: nil}) 83 | end 84 | 85 | @impl GenStage 86 | def handle_info(_, state) do 87 | {:noreply, [], state} 88 | end 89 | 90 | @impl Producer 91 | def prepare_for_draining(%{receive_timer: receive_timer} = state) do 92 | receive_timer && Process.cancel_timer(receive_timer) 93 | {:noreply, [], %{state | receive_timer: nil}} 94 | end 95 | 96 | @impl Acknowledger 97 | def ack({queue_name, repo, dyn_repo, max_fails}, successful, failed) do 98 | if dyn_repo do 99 | repo.put_dynamic_repo(dyn_repo) 100 | end 101 | 102 | :ok = Pgmq.delete_messages(repo, queue_name, Enum.map(successful, fn m -> m.data end)) 103 | 104 | messages_to_archive = 105 | Enum.flat_map(failed, fn m -> 106 | if m.data.read_count >= max_fails do 107 | [m.data.id] 108 | else 109 | [] 110 | end 111 | end) 112 | 113 | Pgmq.archive_messages(repo, queue_name, messages_to_archive) 114 | end 115 | 116 | defp handle_receive_messages(%{receive_timer: nil, demand: demand} = state) when demand > 0 do 117 | messages = receive_messages(state, demand) 118 | new_demand = demand - length(messages) 119 | 120 | receive_timer = 121 | case {messages, new_demand} do 122 | {[], _} -> schedule_receive_messages(state.receive_interval) 123 | {_, 0} -> nil 124 | _ -> schedule_receive_messages(0) 125 | end 126 | 127 | {:noreply, messages, %{state | demand: new_demand, receive_timer: receive_timer}} 128 | end 129 | 130 | defp handle_receive_messages(state) do 131 | {:noreply, [], state} 132 | end 133 | 134 | defp receive_messages(s, total_demand) do 135 | :telemetry.span( 136 | [:off_broadway_pgmq, :receive_messages], 137 | %{}, 138 | fn -> 139 | messages = 140 | s.repo 141 | |> Pgmq.read_messages_with_poll( 142 | s.queue, 143 | s.visibility_timeout, 144 | total_demand, 145 | max_poll_second: s.max_poll_seconds, 146 | poll_interval_ms: s.poll_interval_ms 147 | ) 148 | |> Enum.map(fn message -> 149 | %Broadway.Message{ 150 | data: message, 151 | acknowledger: 152 | {__MODULE__, {s.queue, s.repo, s.dynamic_repo, s.maximum_failures}, []} 153 | } 154 | end) 155 | 156 | {messages, %{messages: messages}} 157 | end 158 | ) 159 | end 160 | 161 | defp schedule_receive_messages(interval) do 162 | Process.send_after(self(), :receive_messages, interval) 163 | end 164 | end 165 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule OffBroadwayPgmq.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.2.2" 5 | 6 | def project do 7 | [ 8 | app: :off_broadway_pgmq, 9 | version: @version, 10 | elixir: "~> 1.14", 11 | start_permanent: Mix.env() == :prod, 12 | deps: deps(), 13 | package: package(), 14 | docs: docs(), 15 | description: description(), 16 | name: "OffBroadwayPgmq", 17 | source_url: "https://github.com/v0idpwn/off_broadway_pgmq" 18 | ] 19 | end 20 | 21 | # Run "mix help compile.app" to learn about applications. 22 | def application do 23 | [ 24 | extra_applications: [:logger] 25 | ] 26 | end 27 | 28 | # Run "mix help deps" to learn about dependencies. 29 | defp deps do 30 | [ 31 | {:pgmq, "~> 0.2"}, 32 | {:broadway, "~> 1.0"}, 33 | {:ex_doc, ">= 0.0.0", runtime: false, only: :dev} 34 | ] 35 | end 36 | 37 | defp package do 38 | [ 39 | name: "off_broadway_pgmq", 40 | files: ~w(lib .formatter.exs mix.exs README.md), 41 | licenses: ["Apache-2.0"], 42 | links: %{"GitHub" => "https://github.com/v0idpwn/off_broadway_pgmq"} 43 | ] 44 | end 45 | 46 | defp docs do 47 | [ 48 | main: "OffBroadwayPgmq", 49 | source_ref: "v#{@version}" 50 | ] 51 | end 52 | 53 | defp description, do: "Broadway producer for PGMQ" 54 | end 55 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "broadway": {:hex, :broadway, "1.0.7", "7808f9e3eb6f53ca6d060f0f9d61012dd8feb0d7a82e62d087dd517b9b66fa53", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e76cfb0a7d64176c387b8b1ddbfb023e2ee8a63e92f43664d78e6d5d0b1177c6"}, 3 | "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, 4 | "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, 5 | "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, 6 | "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"}, 7 | "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"}, 8 | "ex_doc": {:hex, :ex_doc, "0.30.9", "d691453495c47434c0f2052b08dd91cc32bc4e1a218f86884563448ee2502dd2", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "d7aaaf21e95dc5cddabf89063327e96867d00013963eadf2c6ad135506a8bc10"}, 9 | "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, 10 | "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, 11 | "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"}, 12 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 13 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.3", "d684f4bac8690e70b06eb52dad65d26de2eefa44cd19d64a8095e1417df7c8fd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "b78dc853d2e670ff6390b605d807263bf606da3c82be37f9d7f68635bd886fc9"}, 14 | "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, 15 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, 16 | "pgmq": {:hex, :pgmq, "0.2.0", "4a6aff6b35beabfecaa816e5a8cf7687a06830ba92c49928b4da65bc87b71834", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "a58fdc8ea239c8512b7f4c75b3f2e85099f364b425d44a0e8b63254b7763510e"}, 17 | "postgrex": {:hex, :postgrex, "0.17.2", "a3ec9e3239d9b33f1e5841565c4eb200055c52cc0757a22b63ca2d529bbe764c", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "80a918a9e9531d39f7bd70621422f3ebc93c01618c645f2d91306f50041ed90c"}, 18 | "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, 19 | } 20 | -------------------------------------------------------------------------------- /test/off_broadway_pgmq_test.exs: -------------------------------------------------------------------------------- 1 | defmodule OffBroadwayPgmqTest do 2 | use ExUnit.Case 3 | end 4 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------