├── test ├── test_helper.exs └── off_broadway │ └── redis │ └── producer_test.exs ├── .formatter.exs ├── .editorconfig ├── .travis.yml ├── CHANGELOG.md ├── lib └── off_broadway │ └── redis │ ├── redis_client.ex │ ├── producer.ex │ └── redix_client.ex ├── .gitignore ├── LICENSE.txt ├── mix.exs ├── mix.lock └── README.md /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 2 8 | 9 | end_of_line = lf 10 | charset = utf-8 11 | trim_trailing_whitespace = true 12 | insert_final_newline = true 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | os: linux 3 | script: 4 | - if [ "$CHECK_FORMATTED" = true ]; then mix format --check-formatted; fi 5 | - mix test 6 | jobs: 7 | include: 8 | - elixir: "1.8.2" 9 | otp_release: "20.3" 10 | - elixir: "1.9.4" 11 | otp_release: "20.3" 12 | - elixir: "1.10.3" 13 | otp_release: "21.3" 14 | - elixir: "1.10.3" 15 | otp_release: "22.3" 16 | env: CHECK_FORMATTED=true 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.4.2 (2020-04-03) 4 | 5 | * Upgrade to Broadway v0.6.0 6 | * Upgrade to Redix v0.11.1 7 | 8 | ## v0.4.1 (2019-09-06) 9 | 10 | * Fix namespace(s) to adhere to Broadway community standards 11 | 12 | ## v0.4.0 (2019-09-06) 13 | 14 | * Migration from old repository for name change 15 | * Upgrade to Broadway v0.4.0 16 | * Upgrade to Redix v0.10.2 17 | 18 | ## v0.3.0 (2019-04-28) 19 | 20 | * Upgrade to Broadway v0.3.0 21 | * Upgrade to Redix v0.10.0 22 | 23 | ## v0.2.0 (2019-04-05) 24 | 25 | * Upgrade to Broadway v0.2.0 26 | 27 | ## v0.1.0 (2019-04-03) 28 | 29 | * Initial release 30 | -------------------------------------------------------------------------------- /lib/off_broadway/redis/redis_client.ex: -------------------------------------------------------------------------------- 1 | defmodule OffBroadway.Redis.RedisClient do 2 | @moduledoc """ 3 | A generic behaviour to implement Redis Clients for `OffBroadway.Redis.Producer`. 4 | 5 | This module defines callbacks to normalize options and receive items 6 | from a Redis list. 7 | 8 | Modules that implement this behaviour should be passed 9 | as the `:redis_client` option from `OffBroadway.Redis.Producer`. 10 | """ 11 | 12 | alias Broadway.Message 13 | 14 | @type messages :: [Message.t()] 15 | 16 | @callback init(opts :: any) :: {:ok, normalized_opts :: any} | {:error, message :: binary} 17 | @callback receive_messages(demand :: pos_integer, opts :: any) :: messages 18 | end 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | off_broadway_redis-*.tar 24 | 25 | .elixir_ls 26 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Adam Mokan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule OffBroadwayRedis.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.4.3" 5 | @description "An opinionated Redis list connector for Broadway" 6 | @repo_url "https://github.com/amokan/off_broadway_redis" 7 | 8 | def project do 9 | [ 10 | app: :off_broadway_redis, 11 | version: @version, 12 | elixir: "~> 1.7", 13 | name: "OffBroadwayRedis", 14 | description: @description, 15 | start_permanent: Mix.env() == :prod, 16 | deps: deps(), 17 | docs: docs(), 18 | package: package() 19 | ] 20 | end 21 | 22 | # Run "mix help compile.app" to learn about applications. 23 | def application do 24 | [ 25 | extra_applications: [:logger] 26 | ] 27 | end 28 | 29 | # Run "mix help deps" to learn about dependencies. 30 | defp deps do 31 | [ 32 | {:broadway, "~> 0.6.2"}, 33 | {:ex_doc, ">= 0.23.0", only: [:dev, :docs], runtime: false}, 34 | {:redix, ">= 0.11.1 and < 1.1.0"} 35 | ] 36 | end 37 | 38 | defp docs do 39 | [ 40 | main: "readme", 41 | source_ref: "v#{@version}", 42 | source_url: @repo_url, 43 | extras: [ 44 | "README.md", 45 | "CHANGELOG.md", 46 | "LICENSE.txt" 47 | ] 48 | ] 49 | end 50 | 51 | defp package do 52 | %{ 53 | licenses: ["MIT"], 54 | links: %{"GitHub" => @repo_url}, 55 | maintainers: ["Adam Mokan"] 56 | } 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "broadway": {:hex, :broadway, "0.6.2", "ef8e0d257420c72f0e600958cf95556835d9921ad14be333493083226458791a", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f4f93704304a736c984cd6ed884f697415f68eb50906f4dc5d641926366ad8fa"}, 3 | "earmark": {:hex, :earmark, "1.4.5", "62ffd3bd7722fb7a7b1ecd2419ea0b458c356e7168c1f5d65caf09b4fbdd13c8", [:mix], [], "hexpm", "b7d0e6263d83dc27141a523467799a685965bf8b13b6743413f19a7079843f4f"}, 4 | "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, 5 | "ex_doc": {:hex, :ex_doc, "0.23.0", "a069bc9b0bf8efe323ecde8c0d62afc13d308b1fa3d228b65bca5cf8703a529d", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f5e2c4702468b2fd11b10d39416ddadd2fcdd173ba2a0285ebd92c39827a5a16"}, 6 | "gen_stage": {:hex, :gen_stage, "1.0.0", "51c8ae56ff54f9a2a604ca583798c210ad245f415115453b773b621c49776df5", [:mix], [], "hexpm", "1d9fc978db5305ac54e6f5fec7adf80cd893b1000cf78271564c516aa2af7706"}, 7 | "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, 8 | "makeup_elixir": {:hex, :makeup_elixir, "0.15.0", "98312c9f0d3730fde4049985a1105da5155bfe5c11e47bdc7406d88e01e4219b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "75ffa34ab1056b7e24844c90bfc62aaf6f3a37a15faa76b07bc5eba27e4a8b4a"}, 9 | "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, 10 | "redix": {:hex, :redix, "1.0.0", "4f310341744ffceab3031394450a4e603d4d1001a697c3f18ae57ae776cbd3fb", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8c8d9b33b5491737adcd5bb9e0f43b85212a384ac0042f64c156113518266ecb"}, 11 | "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, 12 | } 13 | -------------------------------------------------------------------------------- /lib/off_broadway/redis/producer.ex: -------------------------------------------------------------------------------- 1 | defmodule OffBroadway.Redis.Producer do 2 | @moduledoc """ 3 | A GenStage producer that continuously receives messages from a Redis list. 4 | 5 | This implementation follows the [Reliable Queue](https://redis.io/commands/rpoplpush#pattern-reliable-queue) pattern 6 | outlined in the Redis documentation. 7 | 8 | ## Options 9 | 10 | * `:redis_instance` - Required. An atom representing the redis instance/connection. 11 | * `:list_name` - Required. The name of the redis list containing items you want to process. 12 | * `:working_list_name` - Required. The name of the redis 'working' or 'processing' list. 13 | * `:max_number_of_items` - Optional. The maximum number of items to be fetched per pipelined request. 14 | This value generally should be between `1` and `20`. Default is `10`. 15 | 16 | ## Additional Options 17 | 18 | * `:redis_client` - Optional. A module that implements the `OffBroadway.Redis.RedisClient` 19 | behaviour. This module is responsible for fetching and acknowledging the 20 | messages. Pay attention that all options passed to the producer will be forwarded 21 | to the client. It's up to the client to normalize the options it needs. Default 22 | is `RedixClient`. 23 | * `:receive_interval` - Optional. The duration (in milliseconds) for which the producer waits 24 | before making a request for more items. Default is `5000`. 25 | 26 | """ 27 | 28 | use GenStage 29 | 30 | @default_receive_interval 5_000 31 | 32 | @impl true 33 | def init(opts) do 34 | client = opts[:redis_client] || OffBroadway.Redis.RedixClient 35 | receive_interval = opts[:receive_interval] || @default_receive_interval 36 | 37 | case client.init(opts) do 38 | {:error, message} -> 39 | raise ArgumentError, "invalid options given to #{inspect(client)}.init/1, " <> message 40 | 41 | {:ok, opts} -> 42 | {:producer, 43 | %{ 44 | demand: 0, 45 | receive_timer: nil, 46 | receive_interval: receive_interval, 47 | redis_client: {client, opts} 48 | }} 49 | end 50 | end 51 | 52 | @impl true 53 | def handle_demand(incoming_demand, %{demand: demand} = state) do 54 | handle_receive_messages(%{state | demand: demand + incoming_demand}) 55 | end 56 | 57 | @impl true 58 | def handle_info(:receive_messages, state) do 59 | handle_receive_messages(%{state | receive_timer: nil}) 60 | end 61 | 62 | @impl true 63 | def handle_info(_, state) do 64 | {:noreply, [], state} 65 | end 66 | 67 | def handle_receive_messages(%{receive_timer: nil, demand: demand} = state) when demand > 0 do 68 | messages = receive_messages_from_redis(state, demand) 69 | new_demand = demand - length(messages) 70 | 71 | receive_timer = 72 | case {messages, new_demand} do 73 | {[], _} -> schedule_receive_messages(state.receive_interval) 74 | {_, 0} -> nil 75 | _ -> schedule_receive_messages(0) 76 | end 77 | 78 | {:noreply, messages, %{state | demand: new_demand, receive_timer: receive_timer}} 79 | end 80 | 81 | def handle_receive_messages(state) do 82 | {:noreply, [], state} 83 | end 84 | 85 | defp receive_messages_from_redis(state, total_demand) do 86 | %{redis_client: {client, opts}} = state 87 | client.receive_messages(total_demand, opts) 88 | end 89 | 90 | defp schedule_receive_messages(interval) do 91 | Process.send_after(self(), :receive_messages, interval) 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OffBroadway.Redis 2 | 3 | [![Build Status](https://travis-ci.org/amokan/off_broadway_redis.svg?branch=master)](https://travis-ci.org/amokan/off_broadway_redis) 4 | [![Hex.pm](https://img.shields.io/hexpm/v/off_broadway_redis.svg)](https://hex.pm/packages/off_broadway_redis) 5 | 6 | An _opinionated_ Redis connector for [Broadway](https://github.com/plataformatec/broadway) to process work from a Redis list structure. 7 | 8 | Documentation can be found at [https://hexdocs.pm/off_broadway_redis](https://hexdocs.pm/off_broadway_redis). 9 | 10 | This project provides: 11 | 12 | * `OffBroadway.Redis.Producer` - A GenStage producer that continuously pops items from a Redis list and acknowledges them after being successfully processed. 13 | * `OffBroadway.Redis.RedisClient` - A generic behaviour to implement Redis clients. 14 | * `OffBroadway.Redis.RedixClient` - Default Redis client used by `OffBroadway.Redis.Producer`. 15 | 16 | ## What is opinionated about this library? 17 | 18 | Because Redis lists do not support the concept of acknowledgements, this project utilizes the [RPOPLPUSH](https://redis.io/commands/rpoplpush) command available in Redis to atomically pop an item from the list while moving the item to a 'working' or 'processing' list for a later pseudo-acknowledgement using the `LREM` command. 19 | 20 | This idea follows the blueprint of the _Reliable Queue_ pattern outlined in the Redis documentation found [here](https://redis.io/commands/rpoplpush#pattern-reliable-queue). 21 | 22 | Because `RPOPLPUSH` is used, the other assumption is that the head of your list will be on the right side, so you will likely want to push work into your list using `LPUSH` (for FIFO processing). If you want to prioritize an item to be processed next, you could push that to the right (head) by using a `RPUSH`. 23 | 24 | ## Redis Client 25 | 26 | The default Redis client uses the [Redix](https://github.com/whatyouhide/redix) library. See that project for other features. 27 | 28 | I have not attempted to use any other Redis libraries in the community at this point. I expect there may need to be changes made to this producer to accomodate others. 29 | 30 | ## Caveats 31 | 32 | * You are responsible for maintaining your own named connection to Redis outside the scope of this library. See the [Real-World Usage](https://hexdocs.pm/redix/real-world-usage.html) docs for Redix for setting up a named instance/connection. 33 | * At this point, no testing has been done with a pooling strategy around Redis. I am using a single connection dedicated for my broadway pipeline in a small system. Ideally I would like to improve this to the point where just the Redis host, port, and credentials are provided to this provider for handling it's own connections/pooling. 34 | * You are responsible for monitoring your working/processing list in Redis. If something goes wrong and an acknowledgement (`LREM`) is not handled - you will want some logic or process in place to move an item from the working list back to the main list. 35 | * The Redis `LREM` command is _O(N)_ - so the performance on this operation during acknowledgement will be based on the length of the list. I have been using this pattern for a number of years without problem, but be aware and do research on your own use-case to ensure this is not going to be a problem for you. 36 | 37 | ---- 38 | 39 | ## Installation 40 | 41 | Add `:off_broadway_redis` to the list of dependencies in `mix.exs`: 42 | 43 | ```elixir 44 | def deps do 45 | [ 46 | {:off_broadway_redis, "~> 0.4.3"} 47 | ] 48 | end 49 | ``` 50 | 51 | ## Usage 52 | 53 | Configure Broadway with one or more producers using `OffBroadway.Redis.Producer`: 54 | 55 | ```elixir 56 | Broadway.start_link(MyBroadway, 57 | name: MyBroadway, 58 | producers: [ 59 | default: [ 60 | module: { 61 | OffBroadway.Redis.Producer, 62 | redis_instance: :some_redis_instance, 63 | list_name: "some_list", 64 | working_list_name: "some_list_processing" 65 | } 66 | ] 67 | ] 68 | ) 69 | ``` 70 | 71 | ---- 72 | 73 | ## Other Info 74 | 75 | This library was created using the [Broadway Custom Producers documentation](https://hexdocs.pm/broadway/custom-producers.html) for reference. I would encourage you to view that as well as the [Broadway Architecture documentation](https://hexdocs.pm/broadway/architecture.html) for more information. 76 | 77 | ---- 78 | 79 | ## License 80 | 81 | MIT License 82 | 83 | See the [license file](LICENSE.txt) for details. 84 | -------------------------------------------------------------------------------- /lib/off_broadway/redis/redix_client.ex: -------------------------------------------------------------------------------- 1 | defmodule OffBroadway.Redis.RedixClient do 2 | @moduledoc """ 3 | Default Redis client used by `OffBroadway.Redis.Producer` to communicate with Redis. 4 | 5 | This client implements the `OffBroadway.Redis.RedisClient` behaviour which 6 | defines callbacks for receiving and acknowledging items popped from a list. 7 | """ 8 | 9 | alias Broadway.{Message, Acknowledger} 10 | require Logger 11 | 12 | @behaviour OffBroadway.Redis.RedisClient 13 | @behaviour Acknowledger 14 | 15 | @default_pipeline_timeout 10_000 16 | @default_max_number_of_messages 10 17 | @max_num_messages_allowed 20 18 | 19 | @impl true 20 | def init(opts) do 21 | with {:ok, redis_instance} <- validate(opts, :redis_instance), 22 | {:ok, list_name} <- validate(opts, :list_name), 23 | {:ok, working_list_name} <- validate(opts, :working_list_name), 24 | {:ok, receive_messages_opts} <- validate_receive_messages_opts(opts), 25 | {:ok, config} <- validate(opts, :config, []) do 26 | ack_ref = 27 | Broadway.TermStorage.put(%{ 28 | redis_instance: redis_instance, 29 | list_name: list_name, 30 | working_list_name: working_list_name, 31 | config: config 32 | }) 33 | 34 | {:ok, 35 | %{ 36 | redis_instance: redis_instance, 37 | list_name: list_name, 38 | working_list_name: working_list_name, 39 | receive_messages_opts: receive_messages_opts, 40 | config: config, 41 | ack_ref: ack_ref 42 | }} 43 | end 44 | end 45 | 46 | @impl true 47 | def receive_messages(demand, opts) do 48 | receive_messages_opts = put_max_number_of_items(opts.receive_messages_opts, demand) 49 | 50 | opts.redis_instance 51 | |> pop_messages( 52 | opts.list_name, 53 | opts.working_list_name, 54 | receive_messages_opts[:max_number_of_items] 55 | ) 56 | |> wrap_received_messages(opts.ack_ref) 57 | end 58 | 59 | @impl true 60 | def ack(ack_ref, successful, _failed) do 61 | successful 62 | |> Enum.chunk_every(@max_num_messages_allowed) 63 | |> Enum.each(fn messages -> delete_messages(messages, ack_ref) end) 64 | end 65 | 66 | defp delete_messages(messages, ack_ref) do 67 | receipts = Enum.map(messages, &extract_message_receipt/1) 68 | opts = Broadway.TermStorage.get!(ack_ref) 69 | 70 | delete_message_batch(opts.redis_instance, opts.working_list_name, receipts) 71 | end 72 | 73 | defp wrap_received_messages([], _ack_ref), do: [] 74 | 75 | defp wrap_received_messages(messages, ack_ref) when is_list(messages) do 76 | Enum.map(messages, fn message -> 77 | ack_data = %{receipt: %{id: message}} 78 | %Message{data: message, acknowledger: {__MODULE__, ack_ref, ack_data}} 79 | end) 80 | end 81 | 82 | defp put_max_number_of_items(receive_messages_opts, demand) do 83 | max_number_of_items = min(demand, receive_messages_opts[:max_number_of_items]) 84 | Keyword.put(receive_messages_opts, :max_number_of_items, max_number_of_items) 85 | end 86 | 87 | defp extract_message_receipt(message) do 88 | {_, _, %{receipt: receipt}} = message.acknowledger 89 | receipt 90 | end 91 | 92 | defp validate(opts, key, default \\ nil) when is_list(opts) do 93 | validate_option(key, opts[key] || default) 94 | end 95 | 96 | defp validate_option(:config, value) when not is_list(value), 97 | do: validation_error(:config, "a keyword list", value) 98 | 99 | defp validate_option(:list_name, value) when not is_binary(value) or value == "", 100 | do: validation_error(:list_name, "a non empty string", value) 101 | 102 | defp validate_option(:working_list_name, value) when not is_binary(value) or value == "", 103 | do: validation_error(:working_list_name, "a non empty string", value) 104 | 105 | defp validate_option(:redis_instance, nil), 106 | do: validation_error(:redis_instance, "a atom", nil) 107 | 108 | defp validate_option(:redis_instance, value) when not is_atom(value), 109 | do: validation_error(:redis_instance, "a atom", value) 110 | 111 | defp validate_option(:max_number_of_items, value) when value not in 1..20, 112 | do: validation_error(:max_number_of_items, "a integer between 1 and 20", value) 113 | 114 | defp validate_option(_, value), do: {:ok, value} 115 | 116 | defp validation_error(option, expected, value) do 117 | {:error, "expected #{inspect(option)} to be #{expected}, got: #{inspect(value)}"} 118 | end 119 | 120 | defp validate_receive_messages_opts(opts) do 121 | with {:ok, max_number_of_items} <- 122 | validate(opts, :max_number_of_items, @default_max_number_of_messages) do 123 | {:ok, [max_number_of_items: max_number_of_items]} 124 | end 125 | end 126 | 127 | # atomically batch rpoplpush items from the source list to the working list using a pipelined command 128 | defp pop_messages(redis_instance, source_list, dest_list, number_to_receive) 129 | when number_to_receive > 1 do 130 | pipeline_commands = for _ <- 1..number_to_receive, do: ["RPOPLPUSH", source_list, dest_list] 131 | 132 | case Redix.pipeline(redis_instance, pipeline_commands, timeout: @default_pipeline_timeout) do 133 | {:ok, []} -> 134 | [] 135 | 136 | {:ok, items} -> 137 | items |> Enum.filter(&(&1 != nil)) 138 | 139 | {:error, reason} -> 140 | Logger.warn("Error popping items from Redis list '#{source_list}'. " <> inspect(reason)) 141 | [] 142 | end 143 | end 144 | 145 | defp pop_messages(_, _, _, _), do: [] 146 | 147 | # atomically batch remove items from the working list using a pipelined command 148 | defp delete_message_batch(redis_instance, working_list, receipts) do 149 | pipeline_commands = Enum.map(receipts, fn %{id: id} -> ["LREM", working_list, -1, id] end) 150 | 151 | case Redix.pipeline(redis_instance, pipeline_commands, timeout: @default_pipeline_timeout) do 152 | {:error, reason} -> 153 | Logger.warn( 154 | "Error acknowledging items in Redis working list '#{working_list}'. " <> inspect(reason) 155 | ) 156 | 157 | {:error, reason} 158 | 159 | result -> 160 | result 161 | end 162 | end 163 | end 164 | -------------------------------------------------------------------------------- /test/off_broadway/redis/producer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule OffBroadway.Redis.ProducerTest do 2 | use ExUnit.Case 3 | 4 | alias Broadway.Message 5 | 6 | defmodule MessageServer do 7 | def start_link() do 8 | Agent.start_link(fn -> [] end) 9 | end 10 | 11 | def push_messages(server, messages) do 12 | Agent.update(server, fn queue -> queue ++ messages end) 13 | end 14 | 15 | def take_messages(server, amount) do 16 | Agent.get_and_update(server, &Enum.split(&1, amount)) 17 | end 18 | end 19 | 20 | defmodule FakeRedisClient do 21 | @behaviour OffBroadway.Redis.RedisClient 22 | @behaviour Broadway.Acknowledger 23 | 24 | @impl true 25 | def init(opts), do: {:ok, opts} 26 | 27 | @impl true 28 | def receive_messages(amount, opts) do 29 | messages = MessageServer.take_messages(opts[:message_server], amount) 30 | send(opts[:test_pid], {:messages_received, length(messages)}) 31 | 32 | for msg <- messages do 33 | ack_data = %{ 34 | receipt: %{id: "Id_#{msg}", receipt_handle: "ReceiptHandle_#{msg}"}, 35 | test_pid: opts[:test_pid] 36 | } 37 | 38 | %Message{data: msg, acknowledger: {__MODULE__, :ack_ref, ack_data}} 39 | end 40 | end 41 | 42 | @impl true 43 | def ack(_ack_ref, successful, _failed) do 44 | [%Message{acknowledger: {_, _, %{test_pid: test_pid}}} | _] = successful 45 | send(test_pid, {:messages_deleted, length(successful)}) 46 | end 47 | end 48 | 49 | defmodule Forwarder do 50 | use Broadway 51 | 52 | def handle_message(_, message, %{test_pid: test_pid}) do 53 | send(test_pid, {:message_handled, message.data}) 54 | message 55 | end 56 | 57 | def handle_batch(_, messages, _, _) do 58 | messages 59 | end 60 | end 61 | 62 | test "raise an ArgumentError with proper message when the redis_instance option is nil" do 63 | assert_raise( 64 | ArgumentError, 65 | "invalid options given to OffBroadway.Redis.RedixClient.init/1, expected :redis_instance to be a atom, got: nil", 66 | fn -> 67 | OffBroadway.Redis.Producer.init( 68 | redis_instance: nil, 69 | list_name: "foo", 70 | working_list_name: "bar" 71 | ) 72 | end 73 | ) 74 | end 75 | 76 | test "raise an ArgumentError with proper message when the redis_instance option is invalid" do 77 | assert_raise( 78 | ArgumentError, 79 | "invalid options given to OffBroadway.Redis.RedixClient.init/1, expected :redis_instance to be a atom, got: \"my_redis_instance\"", 80 | fn -> 81 | OffBroadway.Redis.Producer.init( 82 | redis_instance: "my_redis_instance", 83 | list_name: "foo", 84 | working_list_name: "bar" 85 | ) 86 | end 87 | ) 88 | end 89 | 90 | test "raise an ArgumentError with proper message when the list_name option is nil" do 91 | assert_raise( 92 | ArgumentError, 93 | "invalid options given to OffBroadway.Redis.RedixClient.init/1, expected :list_name to be a non empty string, got: nil", 94 | fn -> 95 | OffBroadway.Redis.Producer.init( 96 | redis_instance: :my_redis_instance, 97 | list_name: nil, 98 | working_list_name: "bar" 99 | ) 100 | end 101 | ) 102 | end 103 | 104 | test "raise an ArgumentError with proper message when the list_name option is empty" do 105 | assert_raise( 106 | ArgumentError, 107 | "invalid options given to OffBroadway.Redis.RedixClient.init/1, expected :list_name to be a non empty string, got: \"\"", 108 | fn -> 109 | OffBroadway.Redis.Producer.init( 110 | redis_instance: :my_redis_instance, 111 | list_name: "", 112 | working_list_name: "bar" 113 | ) 114 | end 115 | ) 116 | end 117 | 118 | test "raise an ArgumentError with proper message when the working_list_name option is nil" do 119 | assert_raise( 120 | ArgumentError, 121 | "invalid options given to OffBroadway.Redis.RedixClient.init/1, expected :working_list_name to be a non empty string, got: nil", 122 | fn -> 123 | OffBroadway.Redis.Producer.init( 124 | redis_instance: :my_redis_instance, 125 | list_name: "foo", 126 | working_list_name: nil 127 | ) 128 | end 129 | ) 130 | end 131 | 132 | test "raise an ArgumentError with proper message when the working_list_name option is empty" do 133 | assert_raise( 134 | ArgumentError, 135 | "invalid options given to OffBroadway.Redis.RedixClient.init/1, expected :working_list_name to be a non empty string, got: \"\"", 136 | fn -> 137 | OffBroadway.Redis.Producer.init( 138 | redis_instance: :my_redis_instance, 139 | list_name: "foo", 140 | working_list_name: "" 141 | ) 142 | end 143 | ) 144 | end 145 | 146 | test "receive messages when the queue has less than the demand" do 147 | {:ok, message_server} = MessageServer.start_link() 148 | {:ok, pid} = start_broadway(message_server) 149 | 150 | MessageServer.push_messages(message_server, 1..5) 151 | 152 | assert_receive {:messages_received, 5} 153 | 154 | for msg <- 1..5 do 155 | assert_receive {:message_handled, ^msg} 156 | end 157 | 158 | stop_broadway(pid) 159 | end 160 | 161 | test "keep receiving messages when the queue has more than the demand" do 162 | {:ok, message_server} = MessageServer.start_link() 163 | MessageServer.push_messages(message_server, 1..20) 164 | {:ok, pid} = start_broadway(message_server) 165 | 166 | assert_receive {:messages_received, 10} 167 | 168 | for msg <- 1..10 do 169 | assert_receive {:message_handled, ^msg} 170 | end 171 | 172 | assert_receive {:messages_received, 5} 173 | 174 | for msg <- 11..15 do 175 | assert_receive {:message_handled, ^msg} 176 | end 177 | 178 | assert_receive {:messages_received, 5} 179 | 180 | for msg <- 16..20 do 181 | assert_receive {:message_handled, ^msg} 182 | end 183 | 184 | assert_receive {:messages_received, 0} 185 | 186 | stop_broadway(pid) 187 | end 188 | 189 | test "keep trying to receive new messages when the queue is empty" do 190 | {:ok, message_server} = MessageServer.start_link() 191 | {:ok, pid} = start_broadway(message_server) 192 | 193 | MessageServer.push_messages(message_server, [13]) 194 | assert_receive {:messages_received, 1} 195 | assert_receive {:message_handled, 13} 196 | 197 | assert_receive {:messages_received, 0} 198 | refute_receive {:message_handled, _} 199 | 200 | MessageServer.push_messages(message_server, [14, 15]) 201 | assert_receive {:messages_received, 2} 202 | assert_receive {:message_handled, 14} 203 | assert_receive {:message_handled, 15} 204 | 205 | stop_broadway(pid) 206 | end 207 | 208 | test "delete acknowledged messages" do 209 | {:ok, message_server} = MessageServer.start_link() 210 | {:ok, pid} = start_broadway(message_server) 211 | 212 | MessageServer.push_messages(message_server, 1..20) 213 | 214 | assert_receive {:messages_deleted, 10} 215 | assert_receive {:messages_deleted, 10} 216 | 217 | stop_broadway(pid) 218 | end 219 | 220 | defp start_broadway(message_server) do 221 | Broadway.start_link(Forwarder, 222 | name: new_unique_name(), 223 | context: %{test_pid: self()}, 224 | producer: [ 225 | module: 226 | {OffBroadway.Redis.Producer, 227 | redis_client: FakeRedisClient, 228 | receive_interval: 0, 229 | redis_instance: :fake_redis_instance, 230 | list_name: "some_list", 231 | working_list_name: "some_list_processing", 232 | test_pid: self(), 233 | message_server: message_server}, 234 | concurrency: 1 235 | ], 236 | processors: [ 237 | default: [concurrency: 1] 238 | ], 239 | batchers: [ 240 | default: [ 241 | batch_size: 10, 242 | batch_timeout: 50, 243 | concurrency: 1 244 | ] 245 | ] 246 | ) 247 | end 248 | 249 | defp new_unique_name() do 250 | :"Broadway#{System.unique_integer([:positive, :monotonic])}" 251 | end 252 | 253 | defp stop_broadway(pid) do 254 | ref = Process.monitor(pid) 255 | Process.exit(pid, :normal) 256 | 257 | receive do 258 | {:DOWN, ^ref, _, _, _} -> :ok 259 | end 260 | end 261 | end 262 | --------------------------------------------------------------------------------