├── test ├── test_helper.exs └── gen_queue │ ├── job_test.ex │ ├── adapters │ └── simple_test.exs │ └── test_test.exs ├── lib ├── gen_queue │ ├── error.ex │ ├── adapters │ │ ├── mock_job.ex │ │ └── simple.ex │ ├── job_adapter.ex │ ├── job.ex │ ├── adapter.ex │ └── test.ex └── gen_queue.ex ├── .travis.yml ├── .formatter.exs ├── mix.lock ├── .gitignore ├── LICENSE ├── mix.exs └── README.md /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /lib/gen_queue/error.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Error do 2 | defexception [:message] 3 | end 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | 3 | elixir: 4 | - 1.6.0 5 | 6 | script: 7 | - mix test 8 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "earmark": {:hex, :earmark, "1.2.5", "4d21980d5d2862a2e13ec3c49ad9ad783ffc7ca5769cf6ff891a4553fbaae761", [:mix], [], "hexpm"}, 3 | "ex_doc": {:hex, :ex_doc, "0.18.3", "f4b0e4a2ec6f333dccf761838a4b253d75e11f714b85ae271c9ae361367897b7", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"}, 4 | } 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # Elixir language server files. 5 | /.elixir_ls/ 6 | 7 | # If you run "mix test --cover", coverage assets end up here. 8 | /cover/ 9 | 10 | # The directory Mix downloads your dependencies sources to. 11 | /deps/ 12 | 13 | # Where 3rd-party dependencies like ExDoc output generated docs. 14 | /doc/ 15 | 16 | # Ignore .fetch files in case you like to edit your project deps locally. 17 | /.fetch 18 | 19 | # If the VM crashes, it generates a dump, let's ignore it too. 20 | erl_crash.dump 21 | 22 | # Also ignore archive artifacts (built via "mix archive.build"). 23 | *.ez 24 | -------------------------------------------------------------------------------- /test/gen_queue/job_test.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.JobTest do 2 | use ExUnit.Case 3 | 4 | alias GenQueue.Job 5 | 6 | describe "new/2" do 7 | test "will create a job struct" do 8 | assert %Job{module: Test, args: []} = Job.new(Test) 9 | assert %Job{module: Test, args: []} = Job.new({Test}) 10 | assert %Job{module: Test, args: ["foo"]} = Job.new({Test, "foo"}) 11 | assert %Job{module: Test, queue: "foo"} = Job.new(Test, queue: "foo") 12 | assert %Job{module: Test, delay: 100} = Job.new(Test, delay: 100) 13 | assert %Job{module: Test, config: [:foo]} = Job.new(Test, config: [:foo]) 14 | end 15 | end 16 | end 17 | 18 | -------------------------------------------------------------------------------- /lib/gen_queue/adapters/mock_job.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Adapters.MockJob do 2 | @moduledoc """ 3 | A simple mock job queue implementation. 4 | """ 5 | 6 | use GenQueue.JobAdapter 7 | 8 | def start_link(_gen_queue, _opts) do 9 | :ignore 10 | end 11 | 12 | @doc """ 13 | Push a job that will be returned to the current (or globally set) processes 14 | mailbox. 15 | 16 | Please see `GenQueue.Test` for further details. 17 | """ 18 | @spec handle_job(gen_queue :: GenQueue.t(), job :: GenQueue.Job.t()) :: 19 | {:ok, GenQueue.Job.t()} | {:error, any} 20 | def handle_job(gen_queue, job) do 21 | GenQueue.Test.send_item(gen_queue, job) 22 | {:ok, job} 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/gen_queue/job_adapter.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.JobAdapter do 2 | @moduledoc """ 3 | A behaviour module for implementing job queue adapters. 4 | """ 5 | 6 | @doc """ 7 | Pushes a job to a queue 8 | """ 9 | @callback handle_job(gen_queue :: GenQueue.t(), job :: GenQueue.Job.t()) :: 10 | {:ok, GenQueue.Job.t()} | {:error, any} 11 | 12 | defmacro __using__(_) do 13 | quote location: :keep do 14 | @behaviour GenQueue.JobAdapter 15 | 16 | use GenQueue.Adapter 17 | 18 | @doc """ 19 | Callback implementation for `GenQueue.Adapter.push/2` 20 | """ 21 | def handle_push(gen_queue, item, opts) do 22 | handle_job(gen_queue, GenQueue.Job.new(item, opts)) 23 | end 24 | 25 | @doc false 26 | def handle_job(_gen_queue, _job) do 27 | {:error, :not_implemented} 28 | end 29 | 30 | defoverridable GenQueue.JobAdapter 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Nicholas Sweeting 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Mixfile do 2 | use Mix.Project 3 | 4 | @version "0.1.8" 5 | 6 | def project do 7 | [ 8 | app: :gen_queue, 9 | version: @version, 10 | elixir: "~> 1.6", 11 | start_permanent: Mix.env() == :prod, 12 | deps: deps(), 13 | description: description(), 14 | package: package(), 15 | docs: docs() 16 | ] 17 | end 18 | 19 | # Run "mix help compile.app" to learn about applications. 20 | def application do 21 | [ 22 | extra_applications: [:logger] 23 | ] 24 | end 25 | 26 | defp description do 27 | """ 28 | Queue specification with adapter support for Elixir 29 | """ 30 | end 31 | 32 | defp package do 33 | [ 34 | files: ["lib", "mix.exs", "README*"], 35 | maintainers: ["Nicholas Sweeting"], 36 | licenses: ["MIT"], 37 | links: %{"GitHub" => "https://github.com/nsweeting/gen_queue"} 38 | ] 39 | end 40 | 41 | defp docs do 42 | [ 43 | extras: ["README.md"], 44 | main: "readme", 45 | source_url: "https://github.com/nsweeting/gen_queue" 46 | ] 47 | end 48 | 49 | # Run "mix help deps" to learn about dependencies. 50 | defp deps do 51 | [ 52 | {:ex_doc, ">= 0.0.0", only: :dev} 53 | ] 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/gen_queue/job.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Job do 2 | defstruct [ 3 | :module, 4 | :args, 5 | :queue, 6 | :delay, 7 | :config 8 | ] 9 | 10 | @typedoc "Details on how and what to enqueue a job with" 11 | @type job :: module | {module} | {module, list} | {module, any} 12 | 13 | @typedoc "The name of a queue to place the job under" 14 | @type queue :: binary | atom | nil 15 | 16 | @typedoc "A delay to schedule the job with" 17 | @type delay :: integer | DateTime.t() | nil 18 | 19 | @typedoc "Any additional configuration that is adapter-specific" 20 | @type config :: list | nil 21 | 22 | @typedoc "Options for enqueuing jobs" 23 | @type options :: [{:delay, delay}, {:queue, queue}, {:config, config}] 24 | 25 | @type t :: %GenQueue.Job{ 26 | module: module, 27 | args: list, 28 | queue: queue, 29 | delay: delay, 30 | config: config 31 | } 32 | 33 | @spec new(job, options) :: GenQueue.Job.t() 34 | def new(module, opts \\ []) 35 | 36 | def new(module, opts) when is_atom(module) do 37 | new(module, [], opts) 38 | end 39 | 40 | def new({module}, opts) when is_atom(module) do 41 | new(module, [], opts) 42 | end 43 | 44 | def new({module, args}, opts) when is_atom(module) and is_list(args) do 45 | new(module, args, opts) 46 | end 47 | 48 | def new({module, arg}, opts) when is_atom(module) do 49 | new(module, [arg], opts) 50 | end 51 | 52 | @spec new(module, list, options) :: GenQueue.Job.t() 53 | def new(module, args, opts) when is_list(args) do 54 | job = Keyword.merge(opts, [module: module, args: args]) 55 | struct(__MODULE__, job) 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/gen_queue/adapter.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Adapter do 2 | @moduledoc """ 3 | A behaviour module for implementing queue adapters. 4 | """ 5 | 6 | @callback start_link(gen_queue :: GenQueue.t(), opts :: Keyword.t()) :: GenServer.on_start() 7 | 8 | @doc """ 9 | Pushes an item to a queue 10 | """ 11 | @callback handle_push(gen_queue :: GenQueue.t(), item :: any, opts :: Keyword.t()) :: 12 | {:ok, any} | {:error, any} 13 | 14 | @doc """ 15 | Pops an item from a queue 16 | """ 17 | @callback handle_pop(gen_queue :: GenQueue.t(), opts :: Keyword.t()) :: 18 | {:ok, any} | {:error, any} 19 | 20 | @doc """ 21 | Removes all items from a queue 22 | """ 23 | @callback handle_flush(gen_queue :: GenQueue.t(), opts :: Keyword.t()) :: 24 | {:ok, integer} | {:error, any} 25 | 26 | @doc """ 27 | Gets the number of items in a queue 28 | """ 29 | @callback handle_length(gen_queue :: GenQueue.t(), opts :: Keyword.t()) :: 30 | {:ok, integer} | {:error, any} 31 | 32 | @type t :: module 33 | 34 | defmacro __using__(_) do 35 | quote location: :keep do 36 | @behaviour GenQueue.Adapter 37 | 38 | @doc false 39 | def start_link(_gen_queue, _opts) do 40 | :ignore 41 | end 42 | 43 | @doc false 44 | def handle_push(_gen_queue, _item, _opts) do 45 | {:error, :not_implemented} 46 | end 47 | 48 | @doc false 49 | def handle_pop(_gen_queue, _opts) do 50 | {:error, :not_implemented} 51 | end 52 | 53 | @doc false 54 | def handle_flush(_gen_queue, _opts) do 55 | {:error, :not_implemented} 56 | end 57 | 58 | @doc false 59 | def handle_length(_gen_queue, _opts) do 60 | {:error, :not_implemented} 61 | end 62 | 63 | defoverridable GenQueue.Adapter 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GenQueue 2 | [![Build Status](https://travis-ci.org/nsweeting/gen_queue.svg?branch=master)](https://travis-ci.org/nsweeting/gen_queue) 3 | [![GenQueue Version](https://img.shields.io/hexpm/v/gen_queue.svg)](https://hex.pm/packages/gen_queue) 4 | 5 | GenQueue is a specification for queues. 6 | 7 | This project currently provides the following functionality: 8 | 9 | * `GenQueue` ([docs](https://hexdocs.pm/gen_queue/GenQueue.html)) - a behaviour for queues 10 | 11 | * `GenQueue.Adapter` ([docs](https://hexdocs.pm/gen_queue/GenQueue.Adapter.html)) - a behaviour for implementing adapters for a `GenQueue` 12 | 13 | * `GenQueue.JobAdapter` ([docs](https://hexdocs.pm/gen_queue/GenQueue.JobAdapter.html)) - a behaviour for implementing job-based adapters for a `GenQueue` 14 | 15 | * `GenQueue.Job` ([docs](https://hexdocs.pm/gen_queue/GenQueue.Job.html)) - a struct for containing job-enqueuing instructions 16 | 17 | 18 | ## Installation 19 | 20 | The package can be installed by adding `gen_queue` to your list of dependencies in `mix.exs`: 21 | 22 | ```elixir 23 | def deps do 24 | [ 25 | {:gen_queue, "~> 0.1.8"} 26 | ] 27 | end 28 | ``` 29 | 30 | ## Documentation 31 | 32 | See [HexDocs](https://hexdocs.pm/gen_queue) for additional documentation. 33 | 34 | ## Adapters 35 | 36 | The true functionality of `GenQueue` comes with use of its adapters. Currently, the following 37 | adapters are supported. 38 | 39 | * [GenQueue Exq](https://github.com/nsweeting/gen_queue_exq) - Redis-backed job queue. 40 | * [GenQueue TaskBunny](https://github.com/nsweeting/gen_queue_task_bunny) - RabbitMQ-backed job queue. 41 | * `GenQueue Que` - Mnesia-backed job queue. Currently has an Elixir 1.6 bug. Not available until this is fixed. 42 | * [GenQueue Toniq](https://github.com/nsweeting/gen_queue_toniq) - Redis-backed job queue. 43 | * [GenQueue Verk](https://github.com/nsweeting/gen_queue_verk) - Redis-backed job queue. 44 | * [GenQueue OPQ](https://github.com/nsweeting/gen_queue_opq) - GenStage-backed job queue. 45 | 46 | More adapters are always welcome! 47 | 48 | ## Contributors 49 | 50 | * Nick Sweeting - [@nsweeting](https://github.com/nsweeting) 51 | * Austin Ziegler - [@halostatue](https://github.com/halostatue) -------------------------------------------------------------------------------- /test/gen_queue/adapters/simple_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Adapters.SimpleTest do 2 | use ExUnit.Case, async: true 3 | 4 | defmodule Queue do 5 | use GenQueue 6 | end 7 | 8 | setup do 9 | Queue.start_link() 10 | :ok 11 | end 12 | 13 | describe "push/3" do 14 | test "responds with the item" do 15 | assert {:ok, "foo"} = Queue.push("foo") 16 | end 17 | 18 | test "stores an item under a default queue" do 19 | Queue.push("foo") 20 | assert {:ok, "foo"} = Queue.pop() 21 | end 22 | 23 | test "stores an item under a given queue" do 24 | Queue.push("foo", queue: :bar) 25 | assert {:ok, "foo"} = Queue.pop(queue: :bar) 26 | end 27 | end 28 | 29 | describe "pop/1" do 30 | test "returns items in the order they were pushed for default queues" do 31 | Queue.push("foo") 32 | Queue.push("bar") 33 | assert {:ok, "foo"} = Queue.pop() 34 | assert {:ok, "bar"} = Queue.pop() 35 | end 36 | 37 | test "returns items in the order they were pushed for provided queues" do 38 | Queue.push("foo", queue: :baz) 39 | Queue.push("bar", queue: :baz) 40 | assert {:ok, "foo"} = Queue.pop(queue: :baz) 41 | assert {:ok, "bar"} = Queue.pop(queue: :baz) 42 | end 43 | end 44 | 45 | describe "flush/1" do 46 | test "removes all jobs from the default queue" do 47 | Queue.push("foo") 48 | Queue.push("bar") 49 | assert {:ok, _} = Queue.flush() 50 | assert {:ok, nil} = Queue.pop() 51 | end 52 | 53 | test "removes all jobs from the provided queue" do 54 | Queue.push("foo", queue: :baz) 55 | Queue.push("bar", queue: :baz) 56 | assert {:ok, _} = Queue.flush(queue: :baz) 57 | assert {:ok, nil} = Queue.pop(queue: :baz) 58 | end 59 | 60 | test "returns the number of jobs removed from the default queue" do 61 | Queue.push("foo") 62 | Queue.push("bar") 63 | assert {:ok, 2} = Queue.flush() 64 | assert {:ok, 0} = Queue.flush() 65 | end 66 | 67 | test "returns the number of jobs removed from a provided queue" do 68 | Queue.push("foo", queue: :baz) 69 | Queue.push("bar", queue: :baz) 70 | assert {:ok, 2} = Queue.flush(queue: :baz) 71 | assert {:ok, 0} = Queue.flush(queue: :baz) 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /lib/gen_queue/adapters/simple.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Adapters.Simple do 2 | @moduledoc false 3 | 4 | use GenQueue.Adapter 5 | 6 | def start_link(gen_queue, _opts) do 7 | GenServer.start_link(__MODULE__, %{}, name: gen_queue) 8 | end 9 | 10 | def handle_push(gen_queue, item, opts) do 11 | GenServer.call(gen_queue, {:push, queue_from_opts(opts), item}) 12 | end 13 | 14 | def handle_pop(gen_queue, opts) do 15 | GenServer.call(gen_queue, {:pop, queue_from_opts(opts)}) 16 | end 17 | 18 | def handle_flush(gen_queue, opts) do 19 | GenServer.call(gen_queue, {:flush, queue_from_opts(opts)}) 20 | end 21 | 22 | def handle_length(gen_queue, opts) do 23 | GenServer.call(gen_queue, {:length, queue_from_opts(opts)}) 24 | end 25 | 26 | @doc false 27 | def init(queues) do 28 | {:ok, queues} 29 | end 30 | 31 | @doc false 32 | def handle_call({:push, queue, item}, _from, queues) do 33 | {_, queues} = 34 | Map.get_and_update(queues, queue, fn 35 | nil -> {nil, :queue.in(item, :queue.new())} 36 | current_queue -> {nil, :queue.in(item, current_queue)} 37 | end) 38 | 39 | {:reply, {:ok, item}, queues} 40 | end 41 | 42 | @doc false 43 | def handle_call({:flush, queue}, _from, queues) do 44 | queue_size = 45 | case Map.get(queues, queue) do 46 | nil -> 0 47 | current_queue -> :queue.len(current_queue) 48 | end 49 | 50 | queues = Map.put(queues, queue, :queue.new()) 51 | {:reply, {:ok, queue_size}, queues} 52 | end 53 | 54 | @doc false 55 | def handle_call({:pop, queue}, _from, queues) do 56 | {item, queues} = 57 | Map.get_and_update(queues, queue, fn 58 | nil -> 59 | {nil, :queue.new()} 60 | 61 | current_queue -> 62 | case :queue.out(current_queue) do 63 | {{:value, item}, new_queue} -> {item, new_queue} 64 | {:empty, new_queue} -> {nil, new_queue} 65 | end 66 | end) 67 | 68 | {:reply, {:ok, item}, queues} 69 | end 70 | 71 | @doc false 72 | def handle_call({:length, queue}, _from, queues) do 73 | queue_size = 74 | case Map.get(queues, queue) do 75 | nil -> 0 76 | current_queue -> :queue.len(current_queue) 77 | end 78 | 79 | {:reply, {:ok, queue_size}, queues} 80 | end 81 | 82 | @doc false 83 | defp queue_from_opts(opts) do 84 | Keyword.get(opts, :queue, "default") 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /lib/gen_queue/test.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.Test do 2 | @moduledoc """ 3 | Conveniences for testing queues. 4 | 5 | This module allows us to create or use existing adapter "mock" libraries. 6 | A mock adapter is an adapter that mirrors the functionality of an exisiting 7 | adapter, but instead sends the item to the mailbox of a specified process. 8 | 9 | defmodule Adapter do 10 | use GenQueue.Adapter 11 | 12 | def handle_push(gen_queue, item, _opts) do 13 | GenQueue.Test.send_item(gen_queue, item) 14 | end 15 | end 16 | 17 | We can then test that our items are being pushed correctly. 18 | 19 | use ExUnit.Case, async: true 20 | 21 | import GenQueue.Test 22 | 23 | # This test assumes we have a GenQueue named Queue 24 | 25 | setup do 26 | setup_test_queue(Queue) 27 | end 28 | 29 | test "that our queue works" do 30 | Queue.start_link() 31 | Queue.push(:foo) 32 | assert_recieve(:foo) 33 | end 34 | 35 | Most adapters will provide a mirrored "mock" adapter to use with your tests. 36 | """ 37 | 38 | @doc """ 39 | Sets the queue reciever as the current process for a GenQueue. 40 | """ 41 | @spec setup_test_queue(GenQueue.t()) :: :ok 42 | def setup_test_queue(gen_queue) do 43 | set_queue_receiver(gen_queue, :self) 44 | end 45 | 46 | @doc """ 47 | Removes any current queue receiver for a GenQueue. 48 | """ 49 | @spec reset_test_queue(GenQueue.t()) :: :ok 50 | def reset_test_queue(gen_queue) do 51 | set_queue_receiver(gen_queue, nil) 52 | end 53 | 54 | @doc """ 55 | Sets the queue reciever as the current process for a GenQueue. 56 | 57 | The current process is also given a name. This ensures queues that run outside 58 | of the current process are able to send items to the correct mailbox. 59 | """ 60 | @spec setup_global_test_queue(GenQueue.t(), atom) :: :ok 61 | def setup_global_test_queue(gen_queue, process_name) when is_atom(process_name) do 62 | Process.register(self(), process_name) 63 | set_queue_receiver(gen_queue, process_name) 64 | end 65 | 66 | @doc """ 67 | Sends an item to the mailbox of a process set for a GenQueue. 68 | """ 69 | @spec send_item(GenQueue.t(), any) :: any 70 | def send_item(gen_queue, item) do 71 | case get_queue_receiver(gen_queue) do 72 | nil -> nil 73 | :self -> send(self(), item) 74 | process_name when is_atom(process_name) -> send(process_name, item) 75 | end 76 | end 77 | 78 | defp set_queue_receiver(gen_queue, process_name) when is_atom(process_name) do 79 | Application.put_env(:gen_queue, gen_queue, process_name) 80 | end 81 | 82 | defp get_queue_receiver(gen_queue) do 83 | Application.get_env(:gen_queue, gen_queue) 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /test/gen_queue/test_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GenQueue.TestTest do 2 | use ExUnit.Case 3 | 4 | import GenQueue.Test 5 | 6 | defmodule Adapter do 7 | use GenQueue.Adapter 8 | 9 | def handle_push(gen_queue, item, _opts) do 10 | GenQueue.Test.send_item(gen_queue, item) 11 | end 12 | end 13 | 14 | defmodule AppQueue do 15 | Application.put_env(:gen_queue, __MODULE__, adapter: GenQueue.TestTest.Adapter) 16 | 17 | use GenQueue, otp_app: :gen_queue 18 | end 19 | 20 | defmodule ModQueue do 21 | Application.put_env(:gen_queue, __MODULE__, adapter: GenQueue.TestTest.Adapter) 22 | 23 | use GenQueue, adapter: GenQueue.TestTest.Adapter 24 | end 25 | 26 | describe "app_config: setup_test_queue/1" do 27 | test "will return the item back to the current process" do 28 | setup_test_queue(AppQueue) 29 | AppQueue.push(:foo) 30 | assert_receive(:foo) 31 | end 32 | end 33 | 34 | describe "app_config: setup_global_test_queue/2" do 35 | test "will name the current process" do 36 | setup_global_test_queue(AppQueue, :test) 37 | assert self() == Process.whereis(:test) 38 | end 39 | 40 | test "will return the item back to the named process" do 41 | setup_global_test_queue(AppQueue, :test) 42 | AppQueue.push(:foo) 43 | assert_receive(:foo) 44 | end 45 | end 46 | 47 | describe "app_config: reset_test_queue/1" do 48 | test "will remove any current return processes" do 49 | setup_test_queue(AppQueue) 50 | AppQueue.push(:foo) 51 | assert_receive(:foo) 52 | 53 | reset_test_queue(AppQueue) 54 | AppQueue.push(:foo) 55 | 56 | assert {:message_queue_len, 0} = Process.info(self(), :message_queue_len) 57 | end 58 | end 59 | 60 | describe "mod_config: setup_test_queue/1" do 61 | test "will return the item back to the current process" do 62 | setup_test_queue(ModQueue) 63 | ModQueue.push(:foo) 64 | assert_receive(:foo) 65 | end 66 | end 67 | 68 | describe "mod_config: setup_global_test_queue/2" do 69 | test "will name the current process" do 70 | setup_global_test_queue(ModQueue, :test) 71 | assert self() == Process.whereis(:test) 72 | end 73 | 74 | test "will return the item back to the named process" do 75 | setup_global_test_queue(ModQueue, :test) 76 | ModQueue.push(:foo) 77 | assert_receive(:foo) 78 | end 79 | end 80 | 81 | describe "mod_config: reset_test_queue/1" do 82 | test "will remove any current return processes" do 83 | setup_test_queue(ModQueue) 84 | ModQueue.push(:foo) 85 | assert_receive(:foo) 86 | 87 | reset_test_queue(ModQueue) 88 | ModQueue.push(:foo) 89 | 90 | assert {:message_queue_len, 0} = Process.info(self(), :message_queue_len) 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/gen_queue.ex: -------------------------------------------------------------------------------- 1 | defmodule GenQueue do 2 | @moduledoc """ 3 | A behaviour module for implementing queues. 4 | 5 | GenQueue relies on adapters to handle the specifics of how the queues 6 | are run. At its most simple, this can mean basic memory FIFO queues. At its 7 | most advanced, this can mean full async job queues with retries and 8 | backoffs. By providing a standard interface for such tools - ease in 9 | switching between different implementations is assured. 10 | 11 | ## Example 12 | 13 | The GenQueue behaviour abstracts the common queue interactions. 14 | Developers are only required to implement the callbacks and functionality 15 | they are interested in via adapters. 16 | 17 | Let's start with a simple FIFO queue: 18 | 19 | defmodule Queue do 20 | use GenQueue 21 | end 22 | 23 | # Start the queue 24 | Queue.start_link() 25 | 26 | # Push items into the queue 27 | Queue.push(:hello) 28 | #=> {:ok, :hello} 29 | Queue.push(:world) 30 | #=> {:ok, :world} 31 | 32 | # Pop items from the queue 33 | Queue.pop() 34 | #=> {:ok, :hello} 35 | Queue.pop() 36 | #=> {:ok, :world} 37 | 38 | We start our enqueuer by calling `start_link/1`. This call is then 39 | forwarded to our adapter. In this case, we dont specify an adapter 40 | anywhere, so it defaults to the simple FIFO queue implemented with 41 | the included `GenQueue.Adapters.Simple`. 42 | 43 | We can then add items into our simple FIFO queues with `push/2`, as 44 | well as remove them with `pop/1`. 45 | 46 | ## use GenQueue and adapters 47 | 48 | As we can see from above - implementing a simple queue is easy. But 49 | we can further extend our queues by creating our own adapters or by using 50 | external libraries. Simply specify the adapter name in your config. 51 | 52 | config :my_app, MyApp.Enqueuer, [ 53 | adapter: MyApp.MyAdapter 54 | ] 55 | 56 | defmodule MyApp.Enqueuer do 57 | use GenQueue, otp_app: :my_app 58 | end 59 | 60 | The adapter can also be specified for the module in line: 61 | 62 | defmodule MyApp.Enqueuer do 63 | use GenQueue, adapter: MyApp.MyAdapter 64 | end 65 | 66 | We can then create our own adapter by creating an adapter module that handles 67 | the callbacks specified by `GenQueue.Adapter`. 68 | 69 | defmodule MyApp.MyAdapter do 70 | use GenQueue.Adapter 71 | 72 | def handle_push(gen_queue, item) do 73 | IO.inspect(item) 74 | {:ok, item} 75 | end 76 | end 77 | 78 | ## Current adapters 79 | 80 | Currently, the following adapters are available: 81 | 82 | * [GenQueue Exq](https://github.com/nsweeting/gen_queue_exq) - Redis-backed job queue. 83 | * [GenQueue TaskBunny](https://github.com/nsweeting/gen_queue_task_bunny) - RabbitMQ-backed job queue. 84 | * [GenQueue Verk](https://github.com/nsweeting/gen_queue_verk) - Redis-backed job queue. 85 | * [GenQueue OPQ](https://github.com/nsweeting/gen_queue_opq) - GenStage-backed job queue. 86 | 87 | ## Job queues 88 | 89 | One of the benefits of using `GenQueue` is that it can abstract common tasks 90 | like job enqueueing. We can then provide a common API for the various forms 91 | of job enqueing we would like to implement, as well as easily swap 92 | implementations. 93 | 94 | Please refer to the documentation for each adapter for more details. 95 | """ 96 | 97 | @callback start_link(opts :: Keyword.t()) :: GenServer.on_start() 98 | 99 | @doc """ 100 | Pushes an item to a queue 101 | 102 | ## Example 103 | 104 | case MyQueue.push(value) do 105 | {:ok, value} -> # Pushed with success 106 | {:error, _} -> # Something went wrong 107 | end 108 | """ 109 | @callback push(item :: any, opts :: Keyword.t()) :: {:ok, any} | {:error, any} 110 | 111 | @doc """ 112 | Same as `push/2` but returns the item or raises if an error occurs. 113 | """ 114 | @callback push!(item :: any, opts :: Keyword.t()) :: any | no_return 115 | 116 | @doc """ 117 | Pops an item from a queue 118 | 119 | ## Example 120 | 121 | case MyQueue.pop() do 122 | {:ok, value} -> # Popped with success 123 | {:error, _} -> # Something went wrong 124 | end 125 | """ 126 | @callback pop(opts :: Keyword.t()) :: {:ok, any} | {:error, any} 127 | 128 | @doc """ 129 | Same as `pop/1` but returns the item or raises if an error occurs. 130 | """ 131 | @callback pop!(opts :: Keyword.t()) :: any | no_return 132 | 133 | @doc """ 134 | Removes all items from a queue 135 | 136 | ## Example 137 | 138 | case MyQueue.flush() do 139 | {:ok, number_of_items} -> # Flushed with success 140 | {:error, _} -> # Something went wrong 141 | end 142 | """ 143 | @callback flush(opts :: Keyword.t()) :: {:ok, integer} | {:error, any} 144 | 145 | @doc """ 146 | Gets the number of items in a queue 147 | 148 | ## Example 149 | 150 | case MyQueue.length() do 151 | {:ok, number_of_items} -> # Counted with success 152 | {:error, _} -> # Something went wrong 153 | end 154 | """ 155 | @callback length(opts :: Keyword.t()) :: {:ok, integer} | {:error, any} 156 | 157 | @doc """ 158 | Returns the application config for a queue 159 | """ 160 | @callback config :: Keyword.t() 161 | 162 | @doc """ 163 | Returns the adapter for a queue 164 | """ 165 | @callback adapter :: GenQueue.Adapter.t() 166 | 167 | @type t :: module 168 | 169 | @default_adapter GenQueue.Adapters.Simple 170 | 171 | defmacro __using__(opts) do 172 | quote bind_quoted: [opts: opts] do 173 | @behaviour GenQueue 174 | 175 | @adapter GenQueue.adapter(__MODULE__, opts) 176 | @config GenQueue.config(__MODULE__, opts) 177 | 178 | def child_spec(arg) do 179 | %{ 180 | id: __MODULE__, 181 | start: {__MODULE__, :start_link, [arg]} 182 | } 183 | end 184 | 185 | defoverridable child_spec: 1 186 | 187 | def start_link(opts \\ []) do 188 | apply(@adapter, :start_link, [__MODULE__, opts]) 189 | end 190 | 191 | def push(item, opts \\ []) do 192 | apply(@adapter, :handle_push, [__MODULE__, item, opts]) 193 | end 194 | 195 | def push!(item, opts \\ []) do 196 | case push(item, opts) do 197 | {:ok, item} -> item 198 | _ -> raise GenQueue.Error, "Failed to push item." 199 | end 200 | end 201 | 202 | def pop(opts \\ []) do 203 | apply(@adapter, :handle_pop, [__MODULE__, opts]) 204 | end 205 | 206 | def pop!(opts \\ []) do 207 | case pop(opts) do 208 | {:ok, item} -> item 209 | _ -> raise GenQueue.Error, "Failed to pop item." 210 | end 211 | end 212 | 213 | def flush(opts \\ []) do 214 | apply(@adapter, :handle_flush, [__MODULE__, opts]) 215 | end 216 | 217 | def length(opts \\ []) do 218 | apply(@adapter, :handle_length, [__MODULE__, opts]) 219 | end 220 | 221 | def config do 222 | @config 223 | end 224 | 225 | def adapter do 226 | @adapter 227 | end 228 | end 229 | end 230 | 231 | @doc false 232 | @deprecated "Use adapter/2 instead" 233 | @spec config_adapter(GenQueue.t(), opts :: Keyword.t()) :: GenQueue.Adapter.t() 234 | def config_adapter(gen_queue, opts \\ []) 235 | 236 | def config_adapter(_gen_queue, adapter: adapter) when is_atom(adapter), do: adapter 237 | 238 | def config_adapter(gen_queue, otp_app: app) when is_atom(app) do 239 | app 240 | |> Application.get_env(gen_queue, []) 241 | |> Keyword.get(:adapter, @default_adapter) 242 | end 243 | 244 | def config_adapter(_gen_queue, _opts), do: @default_adapter 245 | 246 | @doc """ 247 | Get the adapter for a GenQueue module based on the options provided. 248 | 249 | If no adapter if specified, the default `GenQueue.Adapters.Simple` is returned. 250 | 251 | ## Options: 252 | 253 | * `:adapter` - The adapter to be returned. 254 | * `:otp_app` - An OTP application that has your GenQueue adapter configuration. 255 | 256 | ## Example 257 | 258 | GenQueue.adapter(MyQueue, [otp_app: :my_app]) 259 | """ 260 | @since "0.1.7" 261 | @spec adapter(GenQueue.t(), opts :: Keyword.t()) :: GenQueue.Adapter.t() 262 | def adapter(gen_queue, opts \\ []) 263 | 264 | def adapter(_gen_queue, adapter: adapter) when is_atom(adapter), do: adapter 265 | 266 | def adapter(gen_queue, otp_app: app) when is_atom(app) do 267 | app 268 | |> Application.get_env(gen_queue, []) 269 | |> Keyword.get(:adapter, @default_adapter) 270 | end 271 | 272 | def adapter(_gen_queue, _opts), do: @default_adapter 273 | 274 | @doc """ 275 | Get the config for a GenQueue module based on the options provided. 276 | 277 | If an `:otp_app` option is provided, this will return the application config. 278 | Otherwise, it will return the options given. 279 | 280 | ## Options 281 | 282 | * `:otp_app` - An OTP application that has your GenQueue configuration. 283 | 284 | ## Example 285 | 286 | # Get the application config 287 | GenQueue.config(MyQueue, [otp_app: :my_app]) 288 | 289 | # Returns the provided options 290 | GenQueue.config(MyQueue, [adapter: MyAdapter]) 291 | """ 292 | @since "0.1.7" 293 | @spec config(GenQueue.t(), opts :: Keyword.t()) :: GenQueue.Adapter.t() 294 | def config(gen_queue, opts \\ []) 295 | 296 | def config(gen_queue, otp_app: app) when is_atom(app) do 297 | Application.get_env(app, gen_queue, []) 298 | end 299 | 300 | def config(_gen_queue, opts) when is_list(opts), do: opts 301 | end 302 | --------------------------------------------------------------------------------