├── .formatter.exs ├── .gitignore ├── README.md ├── config ├── config.exs ├── dev.exs ├── prod.exs └── test.exs ├── lib ├── pgmq.ex └── pgmq │ └── message.ex ├── mix.exs ├── mix.lock └── test ├── pgmq_test.exs ├── support ├── helpers.ex └── repo.ex └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | pgmq-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pgmq 2 | Thin elixir client for the pgmq postgres extension. 3 | 4 | ## Installation 5 | For instructions on installing the pgmq extension, or getting a docker image 6 | with the extension installed, check the [official pgmq repo](https://github.com/tembo-io/pgmq). 7 | 8 | The package can be installed by adding `pgmq` to your list of dependencies in 9 | `mix.exs`: 10 | 11 | ```elixir 12 | def deps do 13 | [ 14 | {:pgmq, "~> 0.1.0"} 15 | ] 16 | end 17 | ``` 18 | If needed, you can create a migration to create the extension in your database: 19 | ```elixir 20 | defmodule MyApp.Repo.Migrations.CreatePgmqExtension do 21 | use Ecto.Migration 22 | 23 | def change do 24 | execute("CREATE EXTENSION pgmq CASCADE") 25 | end 26 | end 27 | ``` 28 | And to create queues: 29 | ```elixir 30 | defmodule MyApp.Repo.Migrations.CreateSomeQueues do 31 | use Ecto.Migration 32 | 33 | def up do 34 | Pgmq.create_queue(repo(), "queue_a") 35 | Pgmq.create_queue(repo(), "queue_b") 36 | Pgmq.create_queue(repo(), "queue_c") 37 | end 38 | 39 | def down do 40 | Pgmq.drop_queue(repo(), "queue_a") 41 | Pgmq.drop_queue(repo(), "queue_b") 42 | Pgmq.drop_queue(repo(), "queue_c") 43 | end 44 | end 45 | ``` 46 | 47 | ## Documentation 48 | Check [our documentation in Hexdocs](https://hexdocs.pm/pgmq). 49 | 50 | ## Usage with Broadway 51 | The [OffBroadwayPgmq](https://github.com/v0idpwn/off_broadway_pgmq) package 52 | provides a configurable Broadway adapter that manages reading, acking and 53 | archiving failing messages. 54 | 55 | ## Stability warning 56 | This package (and pgmq) are both pre-1.0 and might have breaking changes. 57 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | import_config "#{config_env()}.exs" 3 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v0idpwn/pgmq-elixir/a08a09ad933b089fe5327c245e7da830613ed0ca/config/dev.exs -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/v0idpwn/pgmq-elixir/a08a09ad933b089fe5327c245e7da830613ed0ca/config/prod.exs -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :pgmq, Pgmq.TestRepo, 4 | database: "postgres", 5 | username: "postgres", 6 | password: "postgres", 7 | port: 5432 8 | 9 | config :logger, level: :info 10 | -------------------------------------------------------------------------------- /lib/pgmq.ex: -------------------------------------------------------------------------------- 1 | defmodule Pgmq do 2 | @moduledoc """ 3 | Thin wrapper over the pgmq extension 4 | 5 | Provides APIs for sending, reading, archiving and deleting messages. 6 | 7 | ### Use-macros 8 | You can `use Pgmq` for the convenience of having a standardized repo and less 9 | convoluted function calls. By defining: 10 | ``` 11 | # lib/my_app/pgmq.ex 12 | defmodule MyApp.Pgmq do 13 | use Pgmq, repo: MyApp.Repo 14 | end 15 | ``` 16 | 17 | You can then call `MyApp.Pgmq.send_message("myqueue", "hello")`, without passing 18 | in the `MyApp.Repo` 19 | """ 20 | 21 | alias Pgmq.Message 22 | 23 | @typedoc "Queue name" 24 | @type queue :: String.t() 25 | 26 | @typedoc "An Ecto repository" 27 | @type repo :: Ecto.Repo.t() 28 | 29 | @default_max_poll_seconds 5 30 | @default_poll_interval_ms 250 31 | 32 | defmacro __using__(opts) do 33 | repo = Keyword.fetch!(opts, :repo) 34 | 35 | quote do 36 | @spec create_queue(Pgmq.queue(), Keyword.t()) :: :ok 37 | def create_queue(queue, opts \\ []), do: Pgmq.create_queue(unquote(repo), queue, opts) 38 | 39 | @spec drop_queue(Pgmq.queue()) :: :ok 40 | def drop_queue(queue), do: Pgmq.drop_queue(unquote(repo), queue) 41 | 42 | @spec send_message(Pgmq.queue(), binary()) :: {:ok, integer()} | {:error, term()} 43 | def send_message(queue, encoded_message) do 44 | Pgmq.send_message(unquote(repo), queue, encoded_message) 45 | end 46 | 47 | @spec send_messages(Pgmq.queue(), [binary()]) :: {:ok, [integer()]} | {:error, term()} 48 | def send_messages(queue, encoded_messages) do 49 | Pgmq.send_messages(unquote(repo), queue, encoded_messages) 50 | end 51 | 52 | @spec read_message(Pgmq.queue(), integer()) :: Pgmq.Message.t() | nil 53 | def read_message(queue, visibility_timeout_seconds) do 54 | Pgmq.read_message(unquote(repo), queue, visibility_timeout_seconds) 55 | end 56 | 57 | @spec read_messages( 58 | Pgmq.queue(), 59 | visibility_timeout_seconds :: integer(), 60 | count :: integer() 61 | ) :: [Pgmq.Message.t()] 62 | def read_messages(queue, visibility_timeout_seconds, count) do 63 | Pgmq.read_messages(unquote(repo), queue, visibility_timeout_seconds, count) 64 | end 65 | 66 | @spec read_messages_with_poll( 67 | Pgmq.queue(), 68 | visibility_timeout_seconds :: integer(), 69 | count :: integer(), 70 | opts :: Keyword.t() 71 | ) :: [Pgmq.Message.t()] 72 | def read_messages_with_poll( 73 | queue, 74 | count, 75 | visibility_timeout_seconds, 76 | opts \\ [] 77 | ) do 78 | Pgmq.read_messages_with_poll( 79 | unquote(repo), 80 | queue, 81 | visibility_timeout_seconds, 82 | opts 83 | ) 84 | end 85 | 86 | @spec archive_messages(Pgmq.queue(), messages :: [Pgmq.Message.t()] | [integer()]) :: :ok 87 | def archive_messages(queue, messages) do 88 | Pgmq.archive_messages(unquote(repo), queue, messages) 89 | end 90 | 91 | @spec delete_messages(Pgmq.queue(), messages :: [Pgmq.Message.t()] | [integer()]) :: :ok 92 | def delete_messages(queue, messages) do 93 | Pgmq.delete_messages(unquote(repo), queue, messages) 94 | end 95 | 96 | @doc """ 97 | Returns a list of queue names 98 | """ 99 | @spec list_queues() :: [ 100 | %{ 101 | queue_name: String.t(), 102 | is_partitioned: boolean(), 103 | is_unlogged: boolean(), 104 | created_at: DateTime.t() 105 | } 106 | ] 107 | def list_queues() do 108 | Pgmq.list_queues(unquote(repo)) 109 | end 110 | 111 | @doc """ 112 | Sets the visibility timeout of a message for X seconds from now 113 | 114 | Accepts either a message or a message id. 115 | """ 116 | @spec set_message_vt(Pgmq.queue(), Pgmq.Message.t() | integer(), integer()) :: :ok 117 | def set_message_vt(queue, message, vt) do 118 | Pgmq.set_message_vt(unquote(repo), queue, message, vt) 119 | end 120 | 121 | @doc """ 122 | Reads a message and instantly deletes it from the queue 123 | 124 | If there are no messages in the queue, returns `nil`. 125 | """ 126 | @spec pop_message(Pgmq.queue()) :: Pgmq.Message.t() | nil 127 | def pop_message(queue) do 128 | Pgmq.pop_message(unquote(repo), queue) 129 | end 130 | end 131 | end 132 | 133 | @doc """ 134 | Creates a queue in the database 135 | 136 | Notice that the queue name must: 137 | - have less than 48 characters 138 | - start with a letter 139 | - have only letters, numbers, and `_` 140 | 141 | Accepts the following options: 142 | - `:unlogged`: Boolean indicating if the queue should be unlogged. Unlogged 143 | queues are faster to write to, but data may be lost in database crashes or 144 | unclean exits. Can't be used together with `:partitioned`. 145 | - `:partitioned`: indicates if the queue is partitioned. Defaults to `false`. Requires 146 | `pg_partman` extension. 147 | - `:partition_interval:` interval to partition the queue, required if `:partitioned` 148 | is true. 149 | - `:retention_interval:` interval for partition retention, required if `:partitioned` 150 | is true. 151 | """ 152 | @spec create_queue(repo, queue, opts :: Keyword.t()) :: :ok | {:error, atom} 153 | def create_queue(repo, queue, opts \\ []) do 154 | if Keyword.get(opts, :partitioned, false) do 155 | if Keyword.get(opts, :unlogged), do: raise("Partitioned queues can't be unlogged") 156 | partition_interval = Keyword.fetch!(opts, :partition_interval) 157 | retention_interval = Keyword.fetch!(opts, :retention_interval) 158 | 159 | repo.query!("SELECT FROM pgmq.create_partitioned($1, $2, $3)", [ 160 | queue, 161 | partition_interval, 162 | retention_interval 163 | ]) 164 | else 165 | if Keyword.get(opts, :unlogged) do 166 | %Postgrex.Result{num_rows: 1} = 167 | repo.query!("SELECT FROM pgmq.create_unlogged($1)", [queue]) 168 | else 169 | %Postgrex.Result{num_rows: 1} = repo.query!("SELECT FROM pgmq.create($1)", [queue]) 170 | end 171 | end 172 | 173 | :ok 174 | end 175 | 176 | @doc """ 177 | Deletes a queue from the database 178 | """ 179 | @spec drop_queue(repo, queue) :: :ok | {:error, atom} 180 | def drop_queue(repo, queue) do 181 | %Postgrex.Result{num_rows: 1} = repo.query!("SELECT FROM pgmq.drop_queue($1)", [queue]) 182 | :ok 183 | end 184 | 185 | @doc """ 186 | Sends one message to a queue 187 | """ 188 | @spec send_message(repo, queue, encoded_message :: binary) :: 189 | {:ok, Message.t()} | {:error, term} 190 | def send_message(repo, queue, encoded_message) do 191 | case repo.query!("SELECT * FROM pgmq.send($1, $2)", [queue, encoded_message]) do 192 | %Postgrex.Result{rows: [[message_id]]} -> {:ok, message_id} 193 | result -> {:error, {:sending_error, result}} 194 | end 195 | end 196 | 197 | @doc """ 198 | Sends a message batch to a queue 199 | """ 200 | @spec send_messages(repo, queue, encoded_messages :: [binary]) :: 201 | {:ok, Message.t()} | {:error, term} 202 | def send_messages(repo, queue, encoded_messages) do 203 | case repo.query!("SELECT * FROM pgmq.send_batch($1, $2)", [queue, encoded_messages]) do 204 | %Postgrex.Result{rows: message_ids} -> {:ok, List.flatten(message_ids)} 205 | result -> {:error, {:sending_error, result}} 206 | end 207 | end 208 | 209 | @doc """ 210 | Reads one message from a queue 211 | 212 | Returns immediately. If there are no messages in the queue, returns `nil`. 213 | 214 | Messages read through this function are guaranteed not to be read by 215 | other calls for `visibility_timeout_seconds`. 216 | """ 217 | @spec read_message(repo, queue, visibility_timeout_seconds :: integer) :: Message.t() | nil 218 | def read_message(repo, queue, visibility_timeout_seconds) do 219 | %Postgrex.Result{rows: rows} = 220 | repo.query!("SELECT * FROM pgmq.read($1, $2, 1)", [queue, visibility_timeout_seconds]) 221 | 222 | case rows do 223 | [] -> nil 224 | [row] -> Message.from_row(row) 225 | end 226 | end 227 | 228 | @doc """ 229 | Reads a batch of messages from a queue 230 | 231 | Messages read through this function are guaranteed not to be read by 232 | other calls for `visibility_timeout_seconds`. 233 | """ 234 | @spec read_messages(repo, queue, visibility_timeout_seconds :: integer, count :: integer) :: [ 235 | Message.t() 236 | ] 237 | def read_messages(repo, queue, visibility_timeout_seconds, count) do 238 | %Postgrex.Result{rows: rows} = 239 | repo.query!("SELECT * FROM pgmq.read($1, $2, $3)", [ 240 | queue, 241 | visibility_timeout_seconds, 242 | count 243 | ]) 244 | 245 | Enum.map(rows, &Message.from_row/1) 246 | end 247 | 248 | @doc """ 249 | Reads a batch of messages from a queue, but waits if no messages are available 250 | 251 | Accepts two options: 252 | - `:max_poll_seconds`: the maximum duration of the poll. Defaults to 5. 253 | - `:poll_interval_ms`: dictates how often the poll is made database 254 | side. Defaults to 250. Can be tuned for lower latency or less database load, 255 | depending on your needs. 256 | 257 | When there are messages available in the queue, returns immediately. 258 | Otherwise, blocks until at least one message is available, or `max_poll_seconds` 259 | is reached. 260 | 261 | Notice that this function may put significant burden on the connection pool, 262 | as it may hold the connection for several seconds if there's no activity in 263 | the queue. 264 | 265 | Messages read through this function are guaranteed not to be read by 266 | other calls for `visibility_timeout_seconds`. 267 | """ 268 | @spec read_messages_with_poll( 269 | repo, 270 | queue, 271 | visibility_timeout_seconds :: integer, 272 | count :: integer, 273 | opts :: Keyword.t() 274 | ) :: [Message.t()] 275 | def read_messages_with_poll( 276 | repo, 277 | queue, 278 | visibility_timeout_seconds, 279 | count, 280 | opts \\ [] 281 | ) do 282 | max_poll_seconds = Keyword.get(opts, :max_poll_seconds, @default_max_poll_seconds) 283 | poll_interval_ms = Keyword.get(opts, :poll_interval_ms, @default_poll_interval_ms) 284 | 285 | %Postgrex.Result{rows: rows} = 286 | repo.query!("SELECT * FROM pgmq.read_with_poll($1, $2, $3, $4, $5)", [ 287 | queue, 288 | visibility_timeout_seconds, 289 | count, 290 | max_poll_seconds, 291 | poll_interval_ms 292 | ]) 293 | 294 | Enum.map(rows, &Message.from_row/1) 295 | end 296 | 297 | @doc """ 298 | Archives list of messages, removing them from the queue and putting 299 | them into the archive 300 | 301 | This function can receive a list of either `Message.t()` or message ids. Mixed 302 | lists aren't allowed. 303 | """ 304 | @spec archive_messages(repo, queue, [message_id :: integer] | [message :: Message.t()]) :: :ok 305 | def archive_messages(repo, queue, [%Message{} | _] = messages) do 306 | message_ids = Enum.map(messages, fn m -> m.id end) 307 | archive_messages(repo, queue, message_ids) 308 | end 309 | 310 | def archive_messages(repo, queue, [message_id]) do 311 | %Postgrex.Result{rows: [[true]]} = 312 | repo.query!("SELECT * FROM pgmq.archive($1, $2::bigint)", [queue, message_id]) 313 | 314 | :ok 315 | end 316 | 317 | def archive_messages(repo, queue, message_ids) do 318 | %Postgrex.Result{} = 319 | repo.query!("SELECT * FROM pgmq.archive($1, $2::bigint[])", [queue, message_ids]) 320 | 321 | :ok 322 | end 323 | 324 | @doc """ 325 | Deletes a batch of messages, removing them from the queue 326 | 327 | This function can receive a list of either `Message.t()` or message ids. Mixed 328 | lists aren't allowed. 329 | """ 330 | @spec delete_messages(repo, queue, [message_id :: integer] | [Message.t()]) :: :ok 331 | def delete_messages(repo, queue, [%Message{} | _] = messages) do 332 | message_ids = Enum.map(messages, fn m -> m.id end) 333 | delete_messages(repo, queue, message_ids) 334 | end 335 | 336 | def delete_messages(repo, queue, [message_id]) do 337 | %Postgrex.Result{rows: [[true]]} = 338 | repo.query!("SELECT * FROM pgmq.delete($1::text, $2::bigint)", [queue, message_id]) 339 | 340 | :ok 341 | end 342 | 343 | def delete_messages(repo, queue, message_ids) do 344 | %Postgrex.Result{} = 345 | repo.query!("SELECT * FROM pgmq.delete($1::text, $2::bigint[])", [queue, message_ids]) 346 | 347 | :ok 348 | end 349 | 350 | @doc """ 351 | Returns a list of queues 352 | """ 353 | @spec list_queues(repo) :: [ 354 | %{ 355 | queue_name: String.t(), 356 | is_partitioned: boolean(), 357 | is_unlogged: boolean(), 358 | created_at: DateTime.t() 359 | } 360 | ] 361 | def list_queues(repo) do 362 | %Postgrex.Result{ 363 | columns: ["queue_name", "is_partitioned", "is_unlogged", "created_at"], 364 | rows: queues 365 | } = repo.query!("SELECT * FROM pgmq.list_queues()", []) 366 | 367 | Enum.map(queues, fn [queue_name, is_partitioned, is_unlogged, created_at] -> 368 | %{ 369 | queue_name: queue_name, 370 | is_partitioned: is_partitioned, 371 | is_unlogged: is_unlogged, 372 | created_at: created_at 373 | } 374 | end) 375 | end 376 | 377 | @doc """ 378 | Returns a list of queues with stats 379 | """ 380 | @spec get_metrics_all(repo) :: [ 381 | %{ 382 | queue_name: String.t(), 383 | queue_length: pos_integer(), 384 | newest_msg_age_sec: pos_integer() | nil, 385 | oldest_msg_age_sec: pos_integer() | nil, 386 | total_messages: pos_integer(), 387 | scrape_time: DateTime.t() 388 | } 389 | ] 390 | def get_metrics_all(repo) do 391 | %Postgrex.Result{rows: queues} = repo.query!("SELECT * FROM pgmq.metrics_all()", []) 392 | 393 | Enum.map(queues, fn [ 394 | queue_name, 395 | queue_length, 396 | newest_msg_age_sec, 397 | oldest_msg_age_sec, 398 | total_messages, 399 | scrape_time 400 | ] -> 401 | %{ 402 | queue_name: queue_name, 403 | queue_length: queue_length, 404 | newest_msg_age_sec: newest_msg_age_sec, 405 | oldest_msg_age_sec: oldest_msg_age_sec, 406 | total_messages: total_messages, 407 | scrape_time: scrape_time 408 | } 409 | end) 410 | end 411 | 412 | @doc """ 413 | Returns metrics for a single queue 414 | """ 415 | @spec get_metrics(repo, queue) :: [ 416 | %{ 417 | queue_name: String.t(), 418 | queue_length: pos_integer(), 419 | newest_msg_age_sec: pos_integer() | nil, 420 | oldest_msg_age_sec: pos_integer() | nil, 421 | total_messages: pos_integer(), 422 | scrape_time: DateTime.t() 423 | } 424 | ] 425 | def get_metrics(repo, queue) do 426 | %Postgrex.Result{rows: [result]} = repo.query!("SELECT * FROM pgmq.metrics($1)", [queue]) 427 | 428 | [ 429 | queue_name, 430 | queue_length, 431 | newest_msg_age_sec, 432 | oldest_msg_age_sec, 433 | total_messages, 434 | scrape_time 435 | ] = result 436 | 437 | %{ 438 | queue_name: queue_name, 439 | queue_length: queue_length, 440 | newest_msg_age_sec: newest_msg_age_sec, 441 | oldest_msg_age_sec: oldest_msg_age_sec, 442 | total_messages: total_messages, 443 | scrape_time: scrape_time 444 | } 445 | end 446 | 447 | @doc """ 448 | Sets the visibility timeout of a message for X seconds from now 449 | """ 450 | @spec set_message_vt(repo, queue, Message.t() | integer(), visibility_timeout :: integer()) :: 451 | :ok 452 | def set_message_vt(repo, queue, %Message{id: message_id}, visibility_timeout) do 453 | set_message_vt(repo, queue, message_id, visibility_timeout) 454 | end 455 | 456 | def set_message_vt(repo, queue, message_id, visibility_timeout) do 457 | %Postgrex.Result{rows: [_]} = 458 | repo.query!("SELECT * FROM pgmq.set_vt($1, $2, $3)", [queue, message_id, visibility_timeout]) 459 | 460 | :ok 461 | end 462 | 463 | @doc """ 464 | Reads a message and instantly deletes it from the queue 465 | 466 | If there are no messages in the queue, returns `nil`. 467 | """ 468 | @spec pop_message(repo, queue) :: Message.t() | nil 469 | def pop_message(repo, queue) do 470 | case repo.query!("SELECT * FROM pgmq.pop($1)", [queue]) do 471 | %Postgrex.Result{rows: [columns]} -> 472 | Message.from_row(columns) 473 | 474 | %Postgrex.Result{rows: []} -> 475 | nil 476 | end 477 | end 478 | end 479 | -------------------------------------------------------------------------------- /lib/pgmq/message.ex: -------------------------------------------------------------------------------- 1 | defmodule Pgmq.Message do 2 | @moduledoc """ 3 | A message read from pgmq 4 | """ 5 | @enforce_keys [ 6 | :id, 7 | :read_count, 8 | :enqueued_at, 9 | :visibility_timeout, 10 | :body 11 | ] 12 | 13 | defstruct [ 14 | :id, 15 | :read_count, 16 | :enqueued_at, 17 | :visibility_timeout, 18 | :body 19 | ] 20 | 21 | @typedoc """ 22 | A message read from pgmq 23 | """ 24 | @type t :: %__MODULE__{ 25 | id: integer, 26 | body: binary, 27 | read_count: integer, 28 | enqueued_at: Date.t(), 29 | visibility_timeout: Date.t() 30 | } 31 | 32 | def from_row([ 33 | id, 34 | read_count, 35 | enqueued_at, 36 | visibility_timeout, 37 | body 38 | ]) do 39 | %__MODULE__{ 40 | id: id, 41 | read_count: read_count, 42 | enqueued_at: enqueued_at, 43 | visibility_timeout: visibility_timeout, 44 | body: body 45 | } 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Pgmq.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.3.2" 5 | 6 | def project do 7 | [ 8 | app: :pgmq, 9 | version: @version, 10 | elixir: "~> 1.14", 11 | start_permanent: Mix.env() == :prod, 12 | deps: deps(), 13 | elixirc_paths: elixirc_paths(Mix.env()), 14 | description: description(), 15 | package: package(), 16 | name: "Pgmq", 17 | docs: docs(), 18 | source_url: "https://github.com/v0idpwn/pgmq-elixir" 19 | ] 20 | end 21 | 22 | # Run "mix help compile.app" to learn about applications. 23 | def application do 24 | [ 25 | extra_applications: [:logger] 26 | ] 27 | end 28 | 29 | # Run "mix help deps" to learn about dependencies. 30 | defp deps do 31 | [ 32 | {:jason, "~> 1.0"}, 33 | {:ecto_sql, "~> 3.10"}, 34 | {:postgrex, ">= 0.0.0"}, 35 | {:dialyxir, "~> 1.0", only: :dev}, 36 | {:ex_doc, "~> 0.25", only: :dev} 37 | ] 38 | end 39 | 40 | defp docs do 41 | [ 42 | main: "Pgmq", 43 | source_ref: "v#{@version}" 44 | ] 45 | end 46 | 47 | defp package do 48 | [ 49 | name: "pgmq", 50 | files: ~w(lib .formatter.exs mix.exs README.md), 51 | licenses: ["Apache-2.0"], 52 | links: %{"GitHub" => "https://github.com/v0idpwn/pgmq-elixir"} 53 | ] 54 | end 55 | 56 | defp description, do: "Wrapper for the pgmq extension" 57 | 58 | defp elixirc_paths(:test), do: ["lib", "test/support"] 59 | defp elixirc_paths(_), do: ["lib"] 60 | end 61 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, 3 | "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, 4 | "dialyxir": {:hex, :dialyxir, "1.3.0", "fd1672f0922b7648ff9ce7b1b26fcf0ef56dda964a459892ad15f6b4410b5284", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "00b2a4bcd6aa8db9dcb0b38c1225b7277dca9bc370b6438715667071a304696f"}, 5 | "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, 6 | "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"}, 7 | "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"}, 8 | "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, 9 | "ex_doc": {:hex, :ex_doc, "0.30.6", "5f8b54854b240a2b55c9734c4b1d0dd7bdd41f71a095d42a70445c03cf05a281", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bd48f2ddacf4e482c727f9293d9498e0881597eae6ddc3d9562bd7923375109f"}, 10 | "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, 11 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 12 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 13 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, 14 | "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, 15 | "postgrex": {:hex, :postgrex, "0.17.2", "a3ec9e3239d9b33f1e5841565c4eb200055c52cc0757a22b63ca2d529bbe764c", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "80a918a9e9531d39f7bd70621422f3ebc93c01618c645f2d91306f50041ed90c"}, 16 | "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, 17 | } 18 | -------------------------------------------------------------------------------- /test/pgmq_test.exs: -------------------------------------------------------------------------------- 1 | defmodule PgmqTest do 2 | use ExUnit.Case 3 | doctest Pgmq 4 | 5 | alias Pgmq.TestRepo 6 | alias Pgmq.Message 7 | 8 | test "use-macro" do 9 | Code.compile_string(""" 10 | defmodule PgmqMacroUsage do 11 | use Pgmq, repo: SomeModule 12 | end 13 | """) 14 | end 15 | 16 | test "regular flow" do 17 | queue_name = "regular_flow_queue" 18 | assert :ok = Pgmq.create_queue(TestRepo, queue_name) 19 | assert {:ok, message_id} = Pgmq.send_message(TestRepo, queue_name, "1") 20 | assert is_integer(message_id) 21 | 22 | assert %Message{id: ^message_id, body: "1"} = 23 | message = Pgmq.read_message(TestRepo, queue_name, 2) 24 | 25 | assert is_nil(Pgmq.read_message(TestRepo, queue_name, 2)) 26 | 27 | assert :ok = Pgmq.delete_messages(TestRepo, "regular_flow_queue", [message]) 28 | 29 | assert {:ok, message_ids} = Pgmq.send_messages(TestRepo, queue_name, ["1", "2", "3"]) 30 | 31 | assert [m1, m2, m3] = message_ids 32 | assert is_integer(m1) 33 | assert is_integer(m2) 34 | assert is_integer(m3) 35 | end 36 | 37 | test "list queues" do 38 | # This is a trick to reset the tables before running this test, as it 39 | # can fail due to data from others (we aren't using sandbox) 40 | TestRepo.query!("DROP EXTENSION pgmq") 41 | TestRepo.query!("CREATE EXTENSION pgmq") 42 | 43 | # Actual test 44 | q1 = "list_queues_q1" 45 | q2 = "list_queues_q2" 46 | 47 | assert [] = Pgmq.list_queues(TestRepo) 48 | Pgmq.create_queue(TestRepo, q1) 49 | assert [%{queue_name: ^q1}] = Pgmq.list_queues(TestRepo) 50 | Pgmq.create_queue(TestRepo, q2) 51 | assert [%{queue_name: ^q1}, %{queue_name: ^q2}] = Pgmq.list_queues(TestRepo) 52 | end 53 | 54 | test "batches" do 55 | queue_name = "batches_queue" 56 | assert :ok = Pgmq.create_queue(TestRepo, queue_name) 57 | assert {:ok, _m1} = Pgmq.send_message(TestRepo, queue_name, "1") 58 | assert {:ok, _m2} = Pgmq.send_message(TestRepo, queue_name, "2") 59 | assert {:ok, m3} = Pgmq.send_message(TestRepo, queue_name, "3") 60 | assert {:ok, m4} = Pgmq.send_message(TestRepo, queue_name, "4") 61 | 62 | assert [_m1, _m2] = Pgmq.read_messages(TestRepo, queue_name, 0, 2) 63 | 64 | assert [%Message{}, %Message{}, full_m1, full_m2] = 65 | Pgmq.read_messages(TestRepo, queue_name, 0, 4) 66 | 67 | assert :ok = Pgmq.delete_messages(TestRepo, queue_name, [full_m1, full_m2]) 68 | assert [_, _] = Pgmq.read_messages(TestRepo, queue_name, 0, 4) 69 | assert :ok = Pgmq.delete_messages(TestRepo, queue_name, [m3, m4]) 70 | assert [] = Pgmq.read_messages(TestRepo, queue_name, 0, 4) 71 | end 72 | 73 | test "archive" do 74 | queue_name = "archive_queue" 75 | assert :ok = Pgmq.create_queue(TestRepo, queue_name) 76 | assert {:ok, m1} = Pgmq.send_message(TestRepo, queue_name, "1") 77 | assert {:ok, _m2} = Pgmq.send_message(TestRepo, queue_name, "2") 78 | 79 | assert :ok = Pgmq.archive_messages(TestRepo, queue_name, [m1]) 80 | assert [%Message{} = m2_full] = Pgmq.read_messages(TestRepo, queue_name, 0, 2) 81 | 82 | assert :ok = Pgmq.archive_messages(TestRepo, queue_name, [m2_full]) 83 | assert [] = Pgmq.read_messages(TestRepo, queue_name, 0, 2) 84 | 85 | assert Pgmq.Helpers.queue_size(TestRepo, queue_name) == 0 86 | assert Pgmq.Helpers.archive_size(TestRepo, queue_name) == 2 87 | end 88 | 89 | test "polling" do 90 | queue_name = "polling_queue" 91 | assert :ok = Pgmq.create_queue(TestRepo, queue_name) 92 | test_pid = self() 93 | 94 | inserter_pid = 95 | spawn(fn -> 96 | receive do 97 | :insert -> :ok 98 | end 99 | 100 | Pgmq.send_message(TestRepo, queue_name, "hello") 101 | end) 102 | 103 | poller_pid = 104 | spawn(fn -> 105 | receive do 106 | :start_polling -> :ok 107 | end 108 | 109 | assert [%Message{} = m] = Pgmq.read_messages_with_poll(TestRepo, queue_name, 5, 1) 110 | send(test_pid, {:got_result, m}) 111 | end) 112 | 113 | send(poller_pid, :start_polling) 114 | refute_receive {:got_result, _}, 2000 115 | send(inserter_pid, :insert) 116 | assert_receive {:got_result, %Message{}}, 1000 117 | end 118 | 119 | test "pop" do 120 | queue_name = "pop_queue" 121 | assert :ok = Pgmq.create_queue(TestRepo, queue_name) 122 | assert {:ok, _} = Pgmq.send_message(TestRepo, queue_name, "1") 123 | 124 | assert %{body: "1"} = Pgmq.pop_message(TestRepo, queue_name) 125 | assert is_nil(Pgmq.pop_message(TestRepo, queue_name)) 126 | end 127 | 128 | test "set message vt" do 129 | queue_name = "set_vt_queue" 130 | assert :ok = Pgmq.create_queue(TestRepo, queue_name) 131 | assert {:ok, m1} = Pgmq.send_message(TestRepo, queue_name, "1") 132 | assert Pgmq.set_message_vt(TestRepo, queue_name, m1, 50) 133 | assert is_nil(Pgmq.read_message(TestRepo, queue_name, 10)) 134 | assert Pgmq.set_message_vt(TestRepo, queue_name, m1, 0) 135 | assert %Message{} = Pgmq.read_message(TestRepo, queue_name, 10) 136 | end 137 | end 138 | -------------------------------------------------------------------------------- /test/support/helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule Pgmq.Helpers do 2 | def queue_size(repo, queue_name) do 3 | %Postgrex.Result{rows: [[c]]} = 4 | repo.query!("SELECT COUNT(1) FROM pgmq.q_#{queue_name}") 5 | 6 | c 7 | end 8 | 9 | def archive_size(repo, queue_name) do 10 | %Postgrex.Result{rows: [[c]]} = 11 | repo.query!("SELECT COUNT(1) FROM pgmq.a_#{queue_name}") 12 | 13 | c 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /test/support/repo.ex: -------------------------------------------------------------------------------- 1 | defmodule Pgmq.TestRepo do 2 | use Ecto.Repo, otp_app: :pgmq, adapter: Ecto.Adapters.Postgres 3 | end 4 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | _ = Pgmq.TestRepo.__adapter__().storage_down(Pgmq.TestRepo.config()) 2 | _ = Pgmq.TestRepo.__adapter__().storage_up(Pgmq.TestRepo.config()) 3 | Supervisor.start_link([Pgmq.TestRepo], strategy: :one_for_one) 4 | Pgmq.TestRepo.query!("DROP EXTENSION IF EXISTS pgmq") 5 | Pgmq.TestRepo.query!("CREATE EXTENSION pgmq CASCADE") 6 | ExUnit.start() 7 | --------------------------------------------------------------------------------