├── .gitignore ├── .tool-versions ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── README ├── api.md ├── caveats.md ├── dispatchers.md ├── job_lifecycle.md ├── queues.md ├── success_and_failure_modes.md └── workers.md ├── config ├── config.exs ├── dev.exs └── test.exs ├── diagrams └── diagrams.key ├── examples ├── delayed_job.exs ├── ecto_poll_queue │ ├── .formatter.exs │ ├── .gitignore │ ├── README.md │ ├── config │ │ ├── cockroach.exs │ │ ├── config.exs │ │ ├── postgres.exs │ │ └── test.exs │ ├── ecto_poll_queue.png │ ├── lib │ │ └── ecto_poll_queue_example │ │ │ ├── application.ex │ │ │ ├── book.ex │ │ │ ├── classify_photo.ex │ │ │ ├── notify.ex │ │ │ ├── ocr_book.ex │ │ │ ├── photo.ex │ │ │ ├── repo.ex │ │ │ └── user.ex │ ├── mix.cockroach.lock │ ├── mix.exs │ ├── mix.postgres.lock │ ├── priv │ │ └── repo │ │ │ └── migrations │ │ │ ├── 20180408023001_create_photos_and_users.exs │ │ │ ├── 20181228093856_add_prefix_schema.exs │ │ │ ├── 20181228095107_add_prefixed_job_tables.exs │ │ │ └── 20190609185555_add_books_table.exs │ └── test │ │ ├── compound_keys_test.exs │ │ ├── ecto_poll_queue_example_test.exs │ │ └── test_helper.exs ├── exponential_retry.exs ├── filter_and_cancel.exs ├── global │ ├── README.md │ ├── global.exs │ └── global.png ├── initialized_worker │ ├── README.md │ └── initialized_worker.exs ├── job_replies.exs ├── local │ ├── README.md │ ├── local.exs │ └── local.png ├── mnesia.exs └── progress_and_queue_status.exs ├── lib ├── honeydew.ex └── honeydew │ ├── application.ex │ ├── crash.ex │ ├── dispatcher.ex │ ├── dispatcher │ ├── lru.ex │ ├── lru_node.ex │ └── mru.ex │ ├── ecto_poll_queue.ex │ ├── failure_mode.ex │ ├── failure_mode │ ├── abandon.ex │ ├── exponential_retry.ex │ ├── move.ex │ └── retry.ex │ ├── job.ex │ ├── job_monitor.ex │ ├── job_runner.ex │ ├── logger.ex │ ├── logger │ └── metadata.ex │ ├── node_monitor.ex │ ├── node_monitor_supervisor.ex │ ├── poll_queue.ex │ ├── process_group_scope_supervisor.ex │ ├── processes.ex │ ├── progress.ex │ ├── queue.ex │ ├── queue │ ├── erlang_queue.ex │ ├── mnesia.ex │ └── mnesia │ │ └── wrapped_job.ex │ ├── queue_monitor.ex │ ├── queues.ex │ ├── sources │ ├── ecto │ │ ├── erlang_term.ex │ │ ├── sql.ex │ │ ├── sql │ │ │ ├── cockroach.ex │ │ │ └── postgres.ex │ │ └── state.ex │ └── ecto_source.ex │ ├── success_mode.ex │ ├── success_mode │ └── log.ex │ ├── worker.ex │ ├── worker_group_supervisor.ex │ ├── worker_root_supervisor.ex │ ├── worker_starter.ex │ ├── worker_supervisor.ex │ ├── workers.ex │ └── workers_per_queue_supervisor.ex ├── mix.exs ├── mix.lock └── test ├── hammer ├── hammer.exs └── hammer.sh ├── honeydew ├── dispatcher │ ├── lru_node_test.exs │ ├── lru_test.exs │ └── mru_test.exs ├── ecto_poll_queue_test.exs ├── failure_mode │ ├── abandon_test.exs │ ├── exponential_retry_test.exs │ ├── move_test.exs │ └── retry_test.exs ├── global_test.exs ├── logger │ └── metadata_test.exs ├── logger_test.exs ├── queue │ ├── ecto_poll_queue_integration_test.exs │ ├── erlang_queue_integration_test.exs │ └── mnesia_queue_integration_test.exs ├── queues_test.exs └── worker_test.exs ├── honeydew_test.exs ├── support ├── cluster.ex ├── cluster_setups.ex ├── crash_logger_helpers.ex └── workers │ ├── doc_test_worker.ex │ ├── fail_init_once_worker.ex │ ├── failed_init_worker.ex │ ├── stateful.ex │ └── stateless.ex └── test_helper.exs /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | /doc 4 | erl_crash.dump 5 | *.ez 6 | /Mnesia.* 7 | .DS_Store 8 | .elixir_ls -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | erlang 24.0 2 | elixir 1.12.0-otp-24 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | elixir: 3 | - 1.8 4 | otp_release: 5 | - 21.3 6 | # matrix: 7 | # exclude: 8 | # - otp_release: 20.3 9 | # elixir: 1.7 10 | 11 | services: 12 | - postgresql 13 | 14 | addons: 15 | postgresql: "9.5" 16 | 17 | cache: 18 | directories: 19 | - _build 20 | - examples/ecto_poll_queue/_build 21 | - examples/ecto_poll_queue/deps 22 | - deps 23 | 24 | before_install: 25 | - wget -q https://binaries.cockroachdb.com/cockroach-v2.0.0.linux-amd64.tgz 26 | - tar xvf cockroach-v2.0.0.linux-amd64.tgz 27 | - cd cockroach-v2.0.0.linux-amd64 && nohup ./cockroach start --insecure --host=localhost & 28 | 29 | before_script: 30 | - epmd -daemon 31 | - MIX_ENV=test mix compile --warnings-as-errors 32 | - cd examples/ecto_poll_queue && MIX_ENV=cockroach mix do deps.get, compile --warnings-as-errors && cd - 33 | - cd examples/ecto_poll_queue && MIX_ENV=cockroach mix dialyzer --plt && cd - 34 | - cd examples/ecto_poll_queue && MIX_ENV=postgres mix do deps.get, compile --warnings-as-errors && cd - 35 | - cd examples/ecto_poll_queue && MIX_ENV=postgres mix dialyzer --plt && cd - 36 | - MIX_ENV=test mix dialyzer --plt 37 | - cd cockroach-v2.0.0.linux-amd64 && ./cockroach sql --insecure --host=localhost --execute "create database postgres" && cd - 38 | - psql -c 'create database honeydew_test;' -U postgres 39 | # Check for changed files like mix.lock 40 | - git diff --exit-code 41 | 42 | script: 43 | - elixir --erl "+C multi_time_warp" -S mix test 44 | - cd examples/ecto_poll_queue && MIX_ENV=cockroach mix do deps.get && cd - 45 | - cd examples/ecto_poll_queue && MIX_ENV=cockroach mix dialyzer --halt-exit-status && cd - 46 | - cd examples/ecto_poll_queue && MIX_ENV=postgres mix do deps.get && cd - 47 | - cd examples/ecto_poll_queue && MIX_ENV=postgres mix dialyzer --halt-exit-status && cd - 48 | - MIX_ENV=test mix dialyzer --halt-exit-status 49 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 1.4.5 (2019-9-17) 2 | 3 | ### Enhancements 4 | 5 | * Return {:error, reason} tuple for Honeydew.start_queue/2 + start_workers/3. Thanks @hauleth! 6 | 7 | ## 1.4.4 (2019-8-1) 8 | 9 | ### Bug Fixes 10 | 11 | * Job results are now correctly passed to Success Modes 12 | 13 | ## 1.4.3 (2019-6-10) 14 | 15 | ### Enhancements 16 | 17 | * Support for compound primary keys in ecto tables. 18 | 19 | ### Bug Fixes 20 | 21 | * No longer assumes that ecto tables have a single primary key named `id` 22 | 23 | ## 1.4.2 (2019-6-7) 24 | 25 | ### Bug Fixes 26 | 27 | * Don't ignore mnesia table options provided by the user. Thanks @X4lldux! 28 | 29 | ## 1.4.1 (2019-5-9) 30 | 31 | ### Enhancements 32 | 33 | * Job execution filtering for Ecto Poll Queue with `:run_if` option, taking a boolean SQL fragment 34 | * Adding `:timeout` option to `Honeydew.status/2` 35 | 36 | ## 1.4.0 (2019-4-8) 37 | 38 | ### Enhancements 39 | * __Delayed Jobs__ 40 | 41 | You may now pass the `:delay_secs` argument to `async/3` to execute a job when the given number of seconds has passed. 42 | 43 | See the [Delayed Job Example](https://github.com/koudelka/honeydew/blob/master/examples/delayed_job.exs) 44 | 45 | Queue Support: 46 | - `Mnesia` 47 | 48 | Fully supported, uses the system's montonic clock. It's recommended to use [Multi Time Warp Mode](http://erlang.org/doc/apps/erts/time_correction.html#multi-time-warp-mode), to prevent the monotonic clock from freezing for extended periods during a time correction, with `--erl "+C multi_time_warp"`. 49 | - `EctoPollQueue` 50 | 51 | Unsupported, since the Ecto queue doesn't use `async/3`. However, delayed retries are supported. 52 | 53 | It's technically feasible to delay Ecto jobs. As Honeydew wants nothing to do with your model's insertion transaction (to limit its impact on your application), its job ordering is handled by default values in the migration. In order to delay Ecto jobs, you'll need to manually add a number of milliseconds to the `DEFAULT` value of honeydew's lock field in your insertion transaction. 54 | 55 | - `ErlangQueue` 56 | 57 | Unsupported, pending a move to a priority queue. See "Breaking Changes" below to use delayed jobs with an in-memory queue. 58 | 59 | * __Exponential Retry (backoff)__ 60 | 61 | Honeydew now supports exponential retries with the `ExponentialRetry` failure mode. You can optionally set the 62 | exponential base with the `:base` option, the default is `2`. 63 | 64 | See the [Exponential Retry example](https://github.com/koudelka/honeydew/blob/master/examples/exponential_retry.exs) and [docs](https://hexdocs.pm/honeydew/1.4.0/Honeydew.FailureMode.ExponentialRetry.html) 65 | 66 | * __Customizable Retry Strategies__ 67 | 68 | The `Retry` failure mode is now far more customizable, you can provide your own function to determine if, and when, you want 69 | to retry the job (by returning either `{:cont, state, delay_secs}` or `:halt`). 70 | 71 | See the [Exponential Retry Implementation](https://github.com/koudelka/honeydew/blob/master/lib/honeydew/failure_mode/exponential_retry.ex) and [docs](https://hexdocs.pm/honeydew/1.4.0/Honeydew.FailureMode.Retry.html) 72 | 73 | 74 | ### Breaking Changes 75 | * [Mnesia] The schema for the Mnesia queue has been simplified to allow for new features and 76 | future backward compatibility. Unfortuntaely, this change itself isn't backward compatible. 77 | You'll need to drain your queues and delete your on-disk mnesia schema files (`Mnesia.*`), 78 | if you're using `:on_disc`, before upgrading and restarting your queues. 79 | 80 | * [Mnesia] The arguments for the Mnesia queue have been simplified, you no longer need to explicitly 81 | provide a separate list of nodes, simply provide the standard mnesia persistence arguments: 82 | `:ram_copies`, `:disc_copies` and `:disc_only_copies`. 83 | 84 | See the [Mnesia Example](https://github.com/koudelka/honeydew/blob/master/examples/mnesia.exs) 85 | 86 | * [ErlangQueue] The in-memory ErlangQueue is no longer the default queue, since it doesn't currently 87 | support delayed jobs. If you still want to use it, you'll need to explicitly ask for it when starting 88 | your queue, with the `:queue` argument. Instead, the default queue is now an Mnesia queue using `:ram_copies` 89 | and the `:ets` access mode. 90 | 91 | ## 1.3.0 (2019-2-13) 92 | 93 | ### Enhancements 94 | * Ecto 3 support 95 | 96 | ## 1.2.7 (2019-1-8) 97 | 98 | ### Enhancements 99 | * Adding table prefixes to Ecto Poll Queue (thanks @jfornoff!) 100 | 101 | ## 1.2.6 (2018-9-19) 102 | 103 | ### Enhancements 104 | * Honeydew crash log statements now include the following metadata 105 | `:honeydew_crash_reason` and `:honeydew_job`. These metadata entries 106 | can be used for building a LoggerBackend that could forward failures 107 | to an error logger integration like Honeybadger or Bugsnag. 108 | 109 | ## 1.2.5 (2018-8-24) 110 | 111 | ### Bug fixes 112 | * Don't restart workers when linked process terminates normally 113 | 114 | ## 1.2.4 (2018-8-23) 115 | 116 | ### Bug fixes 117 | * Catch thrown signals on user's init/1 118 | 119 | ## 1.2.3 (2018-8-23) 120 | 121 | ### Bug fixes 122 | * Gracefully restart workers when an unhandled message is received. 123 | 124 | ## 1.2.2 (2018-8-23) 125 | 126 | ### Bug fixes 127 | * Catch thrown signals from user's job code 128 | 129 | ## 1.2.1 (2018-8-20) 130 | 131 | ### Bug fixes 132 | * Stop ignoring `init_retry_secs` worker option 133 | * Fixed `Honeydew.worker_opts` typespecs. 134 | * Fixed `Honeydew.start_workers` specs. 135 | 136 | ## 1.2.0 (2018-8-17) 137 | 138 | Honeydew now supervises your queues and workers for you, you no longer need to 139 | add them to your supervision trees. 140 | 141 | ### Breaking Changes 142 | * `Honeydew.queue_spec/2` and `Honeydew.worker_spec/3` are now hard deprecated 143 | in favor of `Honeydew.start_queue/2` and `Honeydew.start_workers/3` 144 | 145 | ### Bug fixes 146 | * Rapidly failing jobs no longer have a chance to take down the worker supervisor. 147 | 148 | ### Enhancements 149 | * `Honeydew.queues/0` and `Honeydew.workers/0` to list queues and workers running 150 | on the local node. 151 | * `Honeydew.stop_queue/1` and `Honeydew.stop_workers/1` to stop local queues and 152 | workers 153 | * Workers can now use the `failed_init/0` callback in combination with 154 | `Honeydew.reinitialize_worker` to re-init workers if their init fails. 155 | * Many other things I'm forgetting... 156 | 157 | ## ? 158 | 159 | ### Breaking Changes 160 | 161 | * Updated `Honeydew.cancel/1` to return `{:error, :not_found}` instead of `nil` 162 | when a job is not found on the queue. This makes it simpler to pattern match 163 | against error conditions, since the other condition is 164 | `{:error, :in_progress}`. 165 | * Changed `Honeydew.Queue.cancel/2` callback to return `{:error, :not_found}` 166 | instead of `nil` when a job isn't found. This makes the return types the same 167 | as `Honeydew.cancel/1`. 168 | 169 | ### Bug fixes 170 | 171 | * Fixed issue where new workers would process jobs from suspended queues (#35) 172 | 173 | ### Enhancements 174 | 175 | * Docs and typespecs for the following functions 176 | * `Honeydew.async/3` 177 | * `Honeydew.filter/2` 178 | * `Honeydew.resume/1` 179 | * `Honeydew.suspend/1` 180 | * `Honeydew.yield/2` 181 | 182 | ## 1.0.4 (2017-11-29) 183 | 184 | ### Breaking Changes 185 | 186 | * Removed `use Honeydew.Queue` in favor of `@behaviour Honeydew.Queue` callbacks 187 | 188 | ### Enhancements 189 | 190 | * Added Honeydew.worker behaviour 191 | * Relaxed typespec for `Honeydew.worker_spec/3` `module_and_args` param (#27) 192 | * New docs for 193 | * `Honeydew.FailureMode` 194 | * `Honeydew.FailureMode.Abandon` 195 | * `Honeydew.FailureMode.Move` 196 | * `Honeydew.FailureMode.Retry` 197 | * `Honeydew.Queue.ErlangQueue` 198 | * `Honeydew.Queue.Mnesia` 199 | * Validate arguments for success and failure modes 200 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Michael Shapiro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Honeydew 💪🏻🍈 2 | ======== 3 | [![Build Status](https://travis-ci.org/koudelka/honeydew.svg?branch=master)](https://travis-ci.org/koudelka/honeydew) 4 | [![Hex pm](https://img.shields.io/hexpm/v/honeydew.svg?style=flat)](https://hex.pm/packages/honeydew) 5 | 6 | Honeydew (["Honey, do!"](http://en.wiktionary.org/wiki/honey_do_list)) is a pluggable job queue and worker pool for Elixir, focused on at-least-once execution. 7 | 8 | ```elixir 9 | defmodule MyWorker do 10 | def do_a_thing do 11 | IO.puts "doing a thing!" 12 | end 13 | end 14 | 15 | :ok = Honeydew.start_queue(:my_queue) 16 | :ok = Honeydew.start_workers(:my_queue, MyWorker) 17 | 18 | :do_a_thing |> Honeydew.async(:my_queue) 19 | 20 | # => "doing a thing!" 21 | ``` 22 | 23 | __Isolation__ 24 | - Jobs are run in isolated one-time-use processes. 25 | - Optionally stores immutable state loaned to each worker (a database connection, for example). 26 | - [Initialized Worker](https://github.com/koudelka/honeydew/tree/master/examples/initialized_worker) 27 | 28 | __Strong Job Custody__ 29 | - Jobs don't leave the queue until either they succeed, are explicitly abandoned or are moved to another queue. 30 | - Workers are issued only one job at a time, no batching. 31 | - If a worker crashes while processing a job, the job is reset and a "failure mode" (e.g. abandon, move, retry) is executed. (The default failure mode [is to abandon the job](https://hexdocs.pm/honeydew/Honeydew.html#start_queue/2).) 32 | - [Job Lifecycle](https://github.com/koudelka/honeydew/blob/master/README/job_lifecycle.md) 33 | 34 | __Clusterable Components__ 35 | - Queues, workers and your enqueuing processes can exist anywhere in the BEAM cluster. 36 | - [Global Queues](https://github.com/koudelka/honeydew/tree/master/examples/global) 37 | 38 | __Plugability__ 39 | - [Queues](https://github.com/koudelka/honeydew/blob/master/README/queues.md), [workers](https://github.com/koudelka/honeydew/blob/master/README/workers.md), [dispatch strategies](https://github.com/koudelka/honeydew/blob/master/README/dispatchers.md), [failure modes and success modes](https://github.com/koudelka/honeydew/blob/master/README/success_and_failure_modes.md) are all plugable with user modules. 40 | - No forced dependency on external queue services. 41 | 42 | __Batteries Included__ 43 | - [Mnesia Queue](https://github.com/koudelka/honeydew/tree/master/examples/mnesia.exs), for in-memory/persistence and simple distribution scenarios. (default) 44 | - [Ecto Queue](#ecto), to turn an Ecto schema into its own work queue, using your database. 45 | - [Fast In-Memory Queue](https://github.com/koudelka/honeydew/tree/master/examples/local), for fast processing of recreatable jobs without delay requirements. 46 | - Can optionally heal the cluster after a disconnect or downed node when using a [Global Queue](https://github.com/koudelka/honeydew/tree/master/examples/global). 47 | - [Delayed Jobs](https://github.com/koudelka/honeydew/tree/master/examples/delayed_job.exs) 48 | - [Exponential Retry](https://github.com/koudelka/honeydew/tree/master/lib/honeydew/failure_mode/exponential_retry.ex), even works with Ecto queues! 49 | 50 | 51 | __Easy API__ 52 | - Jobs are enqueued using `async/3` and you can receive replies with `yield/2`, somewhat like [Task](https://hexdocs.pm/elixir/Task.html). 53 | - [API Overview](https://github.com/koudelka/honeydew/blob/master/README/api.md) 54 | - [Hex Docs](https://hexdocs.pm/honeydew/Honeydew.html) 55 | 56 | 57 | ### Ecto Queue 58 | 59 | The Ecto Queue is designed to painlessly turn your Ecto schema into a queue, using your repo as the backing store. 60 | 61 | - You don't need to explicitly enqueue jobs, that's handled for you (for example, sending a welcome email when a new User is inserted). 62 | - Eliminates the possibility of your database and work queue becoming out of sync 63 | - As the database is the queue, you don't need to run a separate queue node. 64 | - You get all of the high-availability, consistency and distribution semantics of your chosen database. 65 | 66 | Check out the included [example project](https://github.com/koudelka/honeydew/tree/master/examples/ecto_poll_queue), and its README. 67 | 68 | 69 | ## Getting Started 70 | 71 | In your mix.exs file: 72 | 73 | ```elixir 74 | defp deps do 75 | [{:honeydew, "~> 1.5.0"}] 76 | end 77 | ``` 78 | 79 | ## Deployment 80 | 81 | If you're using the Mnesia queue (the default), you'll need tell your release system to include the `:mnesia` application, and you'll have to decide how you're going to create your on-disk schema files, which needs to be done while mnesia is *not* running. 82 | 83 | If you use mnesia outside of Honeydew, you'll want to use the `:extra_applications` configuration key in your mix.exs file, as well as manually creating your mnesia schema with `:mnesia.create_schema(nodes)` in an iex session in production: 84 | 85 | ```elixir 86 | def application do 87 | [ 88 | extra_applications: [:mnesia] 89 | ] 90 | end 91 | ``` 92 | 93 | Otherwise, if Honeydew is the only user of mnesia, you can let Honeydew manage it by simply using the `:included_applications` key instead. 94 | 95 | ```elixir 96 | def application do 97 | [ 98 | included_applications: [:mnesia] 99 | ] 100 | end 101 | ``` 102 | 103 | ### tl;dr 104 | - Check out the [examples](https://github.com/koudelka/honeydew/tree/master/examples). 105 | - Enqueue jobs with `Honeydew.async/3`, delay jobs by passing `delay_secs: `. 106 | - Receive responses with `Honeydew.yield/2`. 107 | - Emit job progress with `progress/1` 108 | - Queue/Worker status with `Honeydew.status/1` 109 | - Suspend and resume with `Honeydew.suspend/1` and `Honeydew.resume/1` 110 | - List jobs with `Honeydew.filter/2` 111 | - Move jobs with `Honeydew.move/2` 112 | - Cancel jobs with `Honeydew.cancel/2` 113 | 114 | 115 | ### README 116 | The rest of the README is broken out into slightly more digestible [sections](https://github.com/koudelka/honeydew/tree/master/README). 117 | 118 | Also, check out the README files included with each of the [examples](https://github.com/koudelka/honeydew/tree/master/examples). 119 | 120 | ### CHANGELOG 121 | It's worth keeping abreast with the [CHANGELOG](https://github.com/koudelka/honeydew/blob/master/CHANGELOG.md) 122 | -------------------------------------------------------------------------------- /README/api.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | Please see the [Honeydew](https://hexdocs.pm/honeydew/Honeydew.html) module's hexdocs for Honeydew's complete API. 4 | 5 | ### Suspend and Resume 6 | You can suspend a queue (halt the distribution of new jobs to workers), by calling [suspend/1](https://hexdocs.pm/honeydew/Honeydew.html#suspend/1), then resume with [resume/1](https://hexdocs.pm/honeydew/Honeydew.html#resume/1). 7 | 8 | ### Cancelling Jobs 9 | To cancel a job that hasn't yet run, use [cancel/1|2](https://hexdocs.pm/honeydew/Honeydew.html#cancel/1). 10 | 11 | See the included [example](https://github.com/koudelka/honeydew/blob/centralize/examples/filter_and_cancel.exs). 12 | 13 | ### Moving Jobs 14 | You can move a job from one queue to another, if it hasn't been started yet, with [move/2](https://hexdocs.pm/honeydew/Honeydew.html#move/2). 15 | 16 | ### Listing Jobs 17 | You can list (and optionally filter the list) of jobs in a queue with [filter/2](https://hexdocs.pm/honeydew/Honeydew.html#filter/2). 18 | 19 | See the included [example](https://github.com/koudelka/honeydew/blob/centralize/examples/filter_and_cancel.exs). 20 | 21 | ### Job Progress 22 | Your jobs can emit their current status, i.e. "downloaded 10/50 items", using the `progress/1` function given to your job module by `use Honeydew.Progress`. 23 | 24 | See the included [example](https://github.com/koudelka/honeydew/blob/centralize/examples/progress_and_queue_status.exs). 25 | 26 | ### Job Replies 27 | By passing `reply: true` to [async/3](https://hexdocs.pm/honeydew/Honeydew.html#async/3), you can recieve replies from your jobs with [yield/2](https://hexdocs.pm/honeydew/Honeydew.html#yield/2). 28 | 29 | See the included [example](https://github.com/koudelka/honeydew/blob/centralize/examples/job_replies.exs) 30 | -------------------------------------------------------------------------------- /README/caveats.md: -------------------------------------------------------------------------------- 1 | # Caveats 2 | - Honeydew attempts to provide "at least once" job execution, it's possible that circumstances could conspire to execute a job, and prevent Honeydew from reporting that success back to the queue. I encourage you to write your jobs idempotently. 3 | 4 | - Honeydew isn't intended as a simple resource pool, the user's code isn't executed in the requesting process. Though you may use it as such, there are likely other alternatives that would fit your situation better, perhaps try [sbroker](https://github.com/fishcakez/sbroker). 5 | 6 | - It's important that you choose a queue implementation to match your needs, see the [Queues section](https://github.com/koudelka/honeydew/blob/master/README/queues.md). 7 | 8 | - It's not recommended to use multiple queue processes spanning the cluster with the Mnesia queue, as its execution order is determined by the system's monotonic clock, which isn't meaningful across nodes. 9 | -------------------------------------------------------------------------------- /README/dispatchers.md: -------------------------------------------------------------------------------- 1 | # Dispatchers 2 | Honeydew provides the following dispatchers: 3 | 4 | - [LRUNode](https://hexdocs.pm/honeydew/Honeydew.Dispatcher.LRUNode.html) - Least Recently Used Node (sends jobs to the least recently used worker on the least recently used node, the default for global queues) 5 | - [LRU](https://hexdocs.pm/honeydew/Honeydew.Dispatcher.LRU.html) - Least Recently Used Worker (FIFO, the default for local queues) 6 | - [MRU](https://hexdocs.pm/honeydew/Honeydew.Dispatcher.MRU.html) - Most Recently Used Worker (LIFO) 7 | 8 | You can also use your own dispatching strategy by passing it to `Honeydew.start_queue/2`. Check out the [built-in dispatchers](https://github.com/koudelka/honeydew/tree/master/lib/honeydew/dispatcher) for reference. 9 | -------------------------------------------------------------------------------- /README/job_lifecycle.md: -------------------------------------------------------------------------------- 1 | # Job Lifecycle 2 | In general, a job goes through the following stages: 3 | 4 | ``` 5 | |─ The user's process calls `async/3`, which packages the task tuple/fn up into a Job and sends it to a 6 | | member of the queue group. 7 | | 8 | ├─ The queue process recieves the Job and enqueues it 9 | | ├─ If there is a Worker available, the queue will dispatch the Job immediately to the waiting Worker 10 | | | via the selected dispatch strategy. 11 | | └─ If there aren't any Workers available, the Job will remain in the queue until a Worker announces 12 | | that it's ready. 13 | | 14 | ├─ Upon dispatch, the queue "reserves" the Job (marks it as in-progress), then spawns a local JobMonitor 15 | | process to watch the Worker. 16 | └─ The JobMonitor starts a timer, if the Worker doesn't claim the Job in time, the queue will assume 17 | there's a problem with the Worker and find another. 18 | ├─ When the Worker receives the Job, it informs the JobMonitor. The JobMonitor then watches the Worker 19 | | in case it crashes (bug, cluster disconnect, etc). 20 | └─ The Worker spawns a JobRunner, providing it with the Job and the user's state from init/1. 21 | | 22 | ├─ If the Job crashes 23 | | | ├─ And the error was trapable (rescue/catch), the JobRunner reports the issue to the Worker and 24 | | | | gracefully stops. 25 | | | └─ If it caused the JobRunner to brutally terminate, the Worker takes note. 26 | | ├─ The Worker informs the JobMonitor of the failure, the JobMonitor executes the selected 27 | | | FailureMode gracefully stops. 28 | | └─ The Worker then executes a controlled restart, because it assumes something is wrong with the 29 | | state the user provided (a dead db connection, for example). 30 | | 31 | └─ If the Job succeeds 32 | ├─ The JobRunner reports the success to the Worker along with the result, and gracefully stops. 33 | ├─ If the Job was enqueued with `reply: true`, the Worker sends the result to the user's process. 34 | ├─ The Worker sends an acknowledgement message to the JobMonitor. 35 | | └─ The JobMonitor sends an acknowledgement to the queue to remove the Job, executes the 36 | | selected SuccessMode and gracefully stops. 37 | └─ The Worker informs the queue that it's ready for a new Job. The queue checks the Worker in 38 | with the dispatcher and the cycle starts again. 39 | ``` 40 | -------------------------------------------------------------------------------- /README/queues.md: -------------------------------------------------------------------------------- 1 | # Queues 2 | Queues are the most critical location of state in Honeydew, a job will not be removed from the queue unless it has either been successfully executed, or been dealt with by the configured failure mode. 3 | 4 | Honeydew includes a few basic queue modules: 5 | - An Mnesia queue, configurable in all the ways mnesia is, for example: 6 | * Run with replication (with queues running on multiple nodes) 7 | * Persist jobs to disk (dets) 8 | * Follow various safety modes ("access contexts"). 9 | - A fast FIFO queue implemented with the `:queue` and `Map` modules. 10 | - An Ecto-backed queue that automatically enqueues jobs when a new row is inserted. 11 | 12 | If you don't explicitly specify a queue to use, Honeydew will use an in-memory Mnesia store. 13 | 14 | ### Queue Options 15 | There are various options you can pass to `start_queue/2`, see the [Honeydew](https://hexdocs.pm/honeydew/Honeydew.html) module docs. 16 | 17 | ### API Support Differences 18 | | | async/3 + yield/2 | filter/2 | cancel/2 | async/3 + `delay_secs` | exponential retry | 19 | |------------------------|:-----------------:|:------------------:|:--------------:|:----------------------:|:-----------------:| 20 | | ErlangQueue (`:queue`) | ✅ | ✅1 | ✅1 | ✅ | ✅ | 21 | | Mnesia | ✅ | ✅ | ✅ | ✅ | ✅ | 22 | | Ecto Poll Queue | ❌ | ❌ | ✅ | ❌ | ✅ | 23 | 24 | [1] this is "slow", O(num_job) 25 | 26 | ### Queue Comparison 27 | | | disk-backed1 | replicated2 | datastore-coordinated | auto-enqueue | 28 | |------------------------|:-----------------------:|:----------------------:|----------------------:|-------------:| 29 | | ErlangQueue (`:queue`) | ❌ | ❌ |❌ |❌ | 30 | | Mnesia | ✅ (dets) | ❌ |❌ |❌ | 31 | | Ecto Poll Queue | ✅ | ✅ |✅ |✅ | 32 | 33 | [1] survives node crashes 34 | 35 | [2] assuming you chose a replicated database to back ecto (tested with cockroachdb and postgres). 36 | Mnesia replication may require manual intevention after a significant netsplit 37 | 38 | ### Plugability 39 | If you want to implement your own queue, check out the included queues as a guide. Try to keep in mind where exactly your queue state lives, is your queue process(es) where jobs live, or is it a completely stateless connector for some external broker? A mix of the two? 40 | -------------------------------------------------------------------------------- /README/success_and_failure_modes.md: -------------------------------------------------------------------------------- 1 | # Failure Modes 2 | When a worker crashes, a monitoring process runs the `handle_failure/3` function from the selected module on the queue's node. Honeydew ships with two failure modes, at present: 3 | 4 | - [Abandon](https://hexdocs.pm/honeydew/Honeydew.FailureMode.Abandon.html) - Simply forgets about the job. 5 | - [Move](https://hexdocs.pm/honeydew/Honeydew.FailureMode.Move.html) - Removes the job from the original queue, and places it on another. 6 | - [Retry](https://hexdocs.pm/honeydew/Honeydew.FailureMode.Retry.html) - Re-attempts the job on its original queue a number of times, then calls another failure mode after the final failure. 7 | 8 | See [Honeydew.start_queue/3](https://hexdocs.pm/honeydew/Honeydew.html#start_queue/3) to select a failure mode. 9 | 10 | # Success Modes 11 | When a job completes successfully, the monitoring process runs the `handle_success/2` function from the selected module on the queue's node. You'll likely want to use this callback for monitoring purposes. You can use a job's `:enqueued_at`, `:started_at` and `:completed_at` fields to calculate various time intervals. 12 | 13 | See [Honeydew.start_queue/3](https://hexdocs.pm/honeydew/Honeydew.html#start_queue/3) to select a success mode. 14 | -------------------------------------------------------------------------------- /README/workers.md: -------------------------------------------------------------------------------- 1 | # Workers 2 | 3 | Workers can be completely stateless, or initialized with state by implementing the `init/1` callback. 4 | 5 | Worker state is immutable, the only way to change it is to cause the worker to crash and let Honeydew restart it. 6 | 7 | Your worker module's `init/1` function must return `{:ok, state}`. If anything else is returned or your callback raises an error, the worker will execute your `failed_init/0` callback, if you've implemented it. If not, the worker will attempt to re-initialize in five seconds. 8 | 9 | Check out the [initialized worker example](https://github.com/koudelka/honeydew/examples/initialized_worker). 10 | 11 | If you'd like to re-initialize the worker from within your `failed_init/0` callback, you can do it like so: 12 | 13 | ```elixir 14 | defmodule FailedInitWorker do 15 | def init(_) do 16 | raise "init failed" 17 | end 18 | 19 | def failed_init do 20 | Honeydew.reinitialize_worker() 21 | end 22 | end 23 | ``` 24 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for third- 9 | # party users, it should be done in your mix.exs file. 10 | 11 | # Sample configuration: 12 | # 13 | # config :logger, :console, 14 | # level: :info, 15 | # format: "$date $time [$level] $metadata$message\n", 16 | # metadata: [:user_id] 17 | 18 | # It is also possible to import configuration files, relative to this 19 | # directory. For example, you can emulate configuration per environment 20 | # by uncommenting the line below and defining dev.exs, test.exs and such. 21 | # Configuration from the imported file will override the ones defined 22 | # here (which is why it is important to import them last). 23 | # 24 | import_config "#{Mix.env}.exs" 25 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :logger, :console, 4 | level: :info 5 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :logger, 4 | compile_time_purge_matching: [ 5 | [level_lower_than: :warn] 6 | ] 7 | 8 | config :logger, :console, 9 | level: :warn 10 | -------------------------------------------------------------------------------- /diagrams/diagrams.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koudelka/honeydew/7c0e825c70ef4b72c82d02ca95491e7365d6b2e8/diagrams/diagrams.key -------------------------------------------------------------------------------- /examples/delayed_job.exs: -------------------------------------------------------------------------------- 1 | # 2 | # iex --erl "+C multi_time_warp" -S mix run examples/delayed_job.exs 3 | # 4 | 5 | defmodule Worker do 6 | @behaviour Honeydew.Worker 7 | 8 | def hello(enqueued_at) do 9 | secs_later = DateTime.diff(DateTime.utc_now(), enqueued_at, :millisecond) / 1_000 10 | IO.puts "I was delayed by #{secs_later}s!" 11 | end 12 | end 13 | 14 | defmodule App do 15 | def start do 16 | :ok = Honeydew.start_queue(:my_queue) 17 | :ok = Honeydew.start_workers(:my_queue, Worker) 18 | end 19 | end 20 | 21 | App.start 22 | {:hello, [DateTime.utc_now()]} |> Honeydew.async(:my_queue, delay_secs: 2) 23 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | /doc 4 | erl_crash.dump 5 | *.ez 6 | Mnesia.* 7 | .DS_Store 8 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/README.md: -------------------------------------------------------------------------------- 1 | # Ecto Poll Queue 2 | ![ecto poll queue](ecto_poll_queue.png) 3 | 4 | The Ecto Poll Queue turns your Ecto schema into a queue, for jobs that need to be run every time you insert a new row into your database. For example, if a user uploads a song file, and you want to transcode it into a number of different formats, or if a user uploads a photo and you want to run your object classifier against it. 5 | 6 | ## Advantages 7 | 8 | 1. State unification, your database is the queue itself. This queue removes the risk that your database and work queue become out of sync. For instance, if you save a model to your database, and before your process can enqueue its job, it suffers a failure that prevents it from contacting the queue, be it a crash, a hardware failure or a network segmentation. If Ecto inserted your row, the job will run. 9 | 10 | 2. You don't need to run an entirely separate queue deployable and manage its interconnection with distributed Erlang. In a typical scenario, you may have application nodes serving user traffic, a queue node, and background wokers. The Ecto queue eliminates the need for the queue node, and the use of distributed Erlang (for work queue purposes) entirely. 11 | 12 | 3. Honeydew automatically "enqueues" jobs for you, to reflect the state of your database, you don't enqueue jobs in your application code. 13 | You don't need to use `enqueue/2` and `yield/2`, in fact, they're unsupported. 14 | 15 | With this queue type, your database is the sole point of coordination, so any availability and replication guarantees are now shared by Honeydew. As such, all your nodes are independent, and don't need to be connected via distributed erlang, as they would with a normal `:global` Honeydew queue. If you choose to use distributed erlang, and make this a global queue, you'll be able to use dispatchers to send jobs to specific nodes for processing. 16 | 17 | This kind of queue process doesn't store any job data, if a queue process crashes for whatever reason, the worst that will happen is that jobs may be re-run, which is within Honeydew's goal of at-least-once job execution. 18 | 19 | ## Tradeoffs 20 | 21 | 1. This is a "poll queue", Honeydew will poll your database occasionally to look for new jobs. You can configure the interval. That being said, the poll itself will only ever fire if the queue is entirely silent. When a worker finishes with a job, it automatically asks your database for a new one. So for a silent queue, the maximum latency between inserting your model and its jobs starting is the polling interval you've configured. 22 | 23 | 2. In order to prevent multiple queues from running the same job, Honeydew uses an expiring lock per job. If a lock has expired, Honeydew assumes that it's because the node processing it has encountered a critical failure and the job needs to be run elsewhere. The lock expiration time is configurable, and should be set to the maximum time that you expect a job to take. Setting it higher is fine, too, it'll just take longer for the job to be re-tried. 24 | 25 | 3. It touches your domain model. Honeydew is writing data directly to your Ecto schema, there's always the chance that something could go horrifically wrong and the consequences are higher. That being said, this code actively used in a business-critical production environment, and we haven't seen any such issues. 26 | 27 | 28 | ## Getting Started 29 | 30 | 1. Add honeydew's fields to your schema. 31 | ```elixir 32 | defmodule MyApp.Photo do 33 | use Ecto.Schema 34 | import Honeydew.EctoPollQueue.Schema 35 | 36 | schema "photos" do 37 | field(:tag) 38 | 39 | honeydew_fields(:classify_photos) 40 | end 41 | end 42 | ``` 43 | 44 | 2. Add honeydew's columns and indexes to your migration. 45 | ```elixir 46 | defmodule MyApp.Repo.Migrations.CreatePhotos do 47 | use Ecto.Migration 48 | import Honeydew.EctoPollQueue.Migration 49 | 50 | def change do 51 | create table(:photos) do 52 | add :tag, :string 53 | 54 | # You can have as many queues as you'd like, they just need unique names. 55 | honeydew_fields(:classify_photos) 56 | end 57 | 58 | honeydew_indexes(:photos, :classify_photos) 59 | end 60 | end 61 | ``` 62 | 63 | 3. Create a Job. 64 | ```elixir 65 | defmodule MyApp.ClassifyPhoto do 66 | alias MyApp.Photo 67 | alias MyApp.Repo 68 | 69 | # By default, Honeydew will call the `run/1` function with the primary key of your newly inserted row. 70 | # 71 | # If your table uses compound keys, a keyword list suitable for passing to `Repo.get_by/2` will be given 72 | # as an arguement. For exaxmple: `[first_name: "Darwin", last_name: "Shapiro"]` 73 | def run(id) do 74 | photo = Repo.get(Photo, id) 75 | 76 | tag = Enum.random(["newt", "ripley", "jonesey", "xenomorph"]) 77 | 78 | IO.puts "Photo contained a #{tag}!" 79 | 80 | photo 81 | |> Ecto.Changeset.change(%{tag: tag}) 82 | |> Repo.update!() 83 | end 84 | end 85 | 86 | ``` 87 | 88 | 4. On your worker nodes, specify your schema and repo modules in the queue spec, and the job module in the worker spec. 89 | 90 | ```elixir 91 | defmodule MyApp.Application do 92 | use Application 93 | 94 | alias Honeydew.EctoPollQueue 95 | alias EctoPollQueue.Repo 96 | alias EctoPollQueue.Photo 97 | alias EctoPollQueue.ClassifyPhoto 98 | 99 | def start(_type, _args) do 100 | children = [Repo] 101 | opts = [strategy: :one_for_one, name: MyApp.Supervisor] 102 | {:ok, supervisor} = Supervisor.start_link(children, opts) 103 | 104 | :ok = Honeydew.start_queue(:classify_photos, queue: {EctoPollQueue, [schema: Photo, repo: Repo]}) 105 | :ok = Honeydew.start_workers(:classify_photos, ClassifyPhoto) 106 | 107 | {:ok, supervisor} 108 | end 109 | end 110 | ``` 111 | 112 | 5. You can also configure the arguments to the queue by specifying a configurable poll interval depending on your requirements. 113 | ```elixir 114 | :ok = Honeydew.start_queue(:classify_photos, queue: {EctoPollQueue, queue_args(Photo, Repo)}) 115 | 116 | defp queue_args(schema, repo) do 117 | # Note that the interval is in seconds to poll the database for new jobs 118 | [schema: schema, repo: repo, poll_interval: Application.get_env(:ecto_poll_queue, :interval, 2)] 119 | end 120 | ``` 121 | 122 | 6. Try inserting an instance of your schema from any of your nodes. The job will be picked up and executed by one of your worker nodes. 123 | ```elixir 124 | iex(1)> {:ok, _photo} = %MyApp.Photo{} |> MyApp.Repo.insert 125 | 126 | #=> "Photo contained a xenomorph!" 127 | ``` 128 | 129 | ## Execution Criteria 130 | 131 | You can filter which rows are selected for execution by providing a boolean SQL fragment with the `:run_if` option, for example: 132 | 133 | ```elixir 134 | # run if the User's name is NULL or the name isn't "dont run me" 135 | :ok = Honeydew.start_queue(:classify_photos, queue: {EctoPollQueue, [schema: User, 136 | repo: Repo, 137 | run_if: ~s{NAME IS NULL OR NAME != 'dont run me'}]}) 138 | ``` 139 | 140 | ## CockroachDB 141 | 142 | Honeydew auto-selects the correct set of SQL queries for your database, depending on which Ecto adapter your repository is using. However, since CockroachDB uses the Postgres adapter, Honeydew can't tell it apart. You'll need to tell Honeydew that you're using Cockroach in two places: 143 | 144 | - When you start your queue: 145 | ```elixir 146 | :ok = Honeydew.start_queue(:classify_photos, queue: {EctoPollQueue, [schema: Photo, 147 | repo: Repo, 148 | database: :cockroachdb]}) 149 | ``` 150 | 151 | - Your migration: 152 | 153 | ```elixir 154 | honeydew_fields(:classify_photos, database: :cockroachdb) 155 | ``` 156 | 157 | ### Queue Status 158 | CockroachDB's execution of `count(*)` is quite slow, so `Honeydew.status/2` will also be slow. You may need to pass the `:timeout` option to increase the amount of time Honeydew will wait for the database to return the status query. 159 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/config/cockroach.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :ecto_poll_queue_example, EctoPollQueueExample.Repo, 4 | adapter: Ecto.Adapters.Postgres, 5 | database: "honeydew_test", 6 | username: "root", 7 | password: "", 8 | hostname: "localhost", 9 | port: 26257, 10 | # removes lock on migration table for cockroach compat 11 | # https://github.com/cockroachdb/cockroach/issues/6583 12 | migration_lock: nil 13 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/config/config.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :ecto_poll_queue_example, ecto_repos: [EctoPollQueueExample.Repo] 4 | config :ecto_poll_queue_example, interval: 0.5 5 | 6 | config :logger, 7 | compile_time_purge_matching: [ 8 | [level_lower_than: :warn] 9 | ], 10 | console: [level: :warn] 11 | 12 | import_config "#{Mix.env()}.exs" 13 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/config/postgres.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :ecto_poll_queue_example, EctoPollQueueExample.Repo, 4 | adapter: Ecto.Adapters.Postgres, 5 | database: "honeydew_test", 6 | username: "postgres", 7 | password: "", 8 | hostname: "localhost" 9 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :logger, 4 | compile_time_purge_matching: [ 5 | [level_lower_than: :warn] 6 | ], 7 | console: [level: : warn] 8 | 9 | config :ecto_poll_queue_example, interval: 0.5 10 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/ecto_poll_queue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koudelka/honeydew/7c0e825c70ef4b72c82d02ca95491e7365d6b2e8/examples/ecto_poll_queue/ecto_poll_queue.png -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/application.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Application do 2 | # See https://hexdocs.pm/elixir/Application.html 3 | # for more information on OTP Applications 4 | @moduledoc false 5 | 6 | use Application 7 | 8 | alias Honeydew.EctoPollQueue 9 | alias Honeydew.FailureMode.Retry 10 | alias Honeydew.FailureMode.ExponentialRetry 11 | alias EctoPollQueueExample.Repo 12 | 13 | alias EctoPollQueueExample.Photo 14 | alias EctoPollQueueExample.ClassifyPhoto 15 | 16 | alias EctoPollQueueExample.Book 17 | alias EctoPollQueueExample.OCRBook 18 | 19 | alias EctoPollQueueExample.User 20 | alias EctoPollQueueExample.Notify 21 | 22 | import User, only: [notify_queue: 0] 23 | import Photo, only: [classify_queue: 0] 24 | import Book, only: [ocr_queue: 0] 25 | 26 | def start(_type, _args) do 27 | children = [Repo] 28 | opts = [strategy: :one_for_one, name: EctoPollQueueExample.Supervisor] 29 | {:ok, supervisor} = Supervisor.start_link(children, opts) 30 | 31 | notify_queue_args = queue_args(User) ++ [run_if: ~s{NAME IS NULL OR NAME != 'dont run'}] 32 | :ok = Honeydew.start_queue(notify_queue(), queue: {EctoPollQueue, notify_queue_args}, failure_mode: {ExponentialRetry, base: 3, times: 3}) 33 | :ok = Honeydew.start_workers(notify_queue(), Notify) 34 | 35 | :ok = Honeydew.start_queue(classify_queue(), queue: {EctoPollQueue, queue_args(Photo)}, failure_mode: {Retry, [times: 1]}) 36 | :ok = Honeydew.start_workers(classify_queue(), ClassifyPhoto, num: 20) 37 | 38 | :ok = Honeydew.start_queue(ocr_queue(), queue: {EctoPollQueue, queue_args(Book)}, failure_mode: {Retry, [times: 1]}) 39 | :ok = Honeydew.start_workers(ocr_queue(), OCRBook, num: 5) 40 | 41 | {:ok, supervisor} 42 | end 43 | 44 | defp queue_args(schema) do 45 | poll_interval = Application.get_env(:ecto_poll_queue, :interval, 1) 46 | 47 | queue_args = [schema: schema, repo: Repo, poll_interval: poll_interval] 48 | 49 | if Mix.env == :cockroach do 50 | Keyword.put(queue_args, :database, :cockroachdb) 51 | else 52 | queue_args 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/book.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Book do 2 | use Ecto.Schema 3 | import Honeydew.EctoPollQueue.Schema 4 | alias Honeydew.EctoSource.ErlangTerm 5 | 6 | @primary_key false 7 | 8 | @ocr_queue :ocr 9 | 10 | schema "books" do 11 | field :author, :string, primary_key: true 12 | field :title, :string, primary_key: true 13 | 14 | field :from, ErlangTerm 15 | field :should_fail, :boolean 16 | 17 | honeydew_fields(@ocr_queue) 18 | 19 | timestamps() 20 | end 21 | 22 | def ocr_queue, do: @ocr_queue 23 | end 24 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/classify_photo.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.ClassifyPhoto do 2 | alias EctoPollQueueExample.Photo 3 | alias EctoPollQueueExample.Repo 4 | 5 | def run(id) do 6 | photo = Repo.get(Photo, id) 7 | 8 | if photo.sleep do 9 | Process.sleep(photo.sleep) 10 | end 11 | 12 | if photo.from do 13 | send(photo.from, {:classify_job_ran, id}) 14 | end 15 | 16 | if photo.should_fail do 17 | raise "classifier's totally busted dude!" 18 | end 19 | 20 | tag = Enum.random(["newt", "ripley", "jonesey", "xenomorph"]) 21 | 22 | photo 23 | |> Ecto.Changeset.change(%{tag: tag}) 24 | |> Repo.update!() 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/notify.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Notify do 2 | alias EctoPollQueueExample.User 3 | alias EctoPollQueueExample.Repo 4 | 5 | def run(id) do 6 | user = Repo.get(User, id) 7 | 8 | if user.sleep do 9 | Process.sleep(user.sleep) 10 | end 11 | 12 | if user.from do 13 | send(user.from, {:notify_job_ran, id}) 14 | end 15 | 16 | if user.should_fail do 17 | raise "notifier's totally busted dude!" 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/ocr_book.ex: -------------------------------------------------------------------------------- 1 | # "Optical Character Recognition" 2 | 3 | defmodule EctoPollQueueExample.OCRBook do 4 | alias EctoPollQueueExample.Repo 5 | alias EctoPollQueueExample.Book 6 | 7 | def run(primary_keys) do 8 | book = Repo.get_by(Book, primary_keys) 9 | 10 | if book.from do 11 | send(book.from, {:ocr_job_ran, primary_keys}) 12 | end 13 | 14 | if book.should_fail do 15 | raise "ocr's totally busted dude!" 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/photo.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Photo do 2 | use Ecto.Schema 3 | import Honeydew.EctoPollQueue.Schema 4 | alias Honeydew.EctoSource.ErlangTerm 5 | 6 | if Mix.env == :cockroach do 7 | @primary_key {:id, :binary_id, autogenerate: false, read_after_writes: true} 8 | @foreign_key_type :binary_id 9 | else 10 | @primary_key {:id, :binary_id, autogenerate: true} 11 | end 12 | 13 | if System.get_env("prefixed_tables") do 14 | @schema_prefix "theprefix" 15 | end 16 | 17 | @classify_queue :classify_photos 18 | 19 | schema "photos" do 20 | field(:tag) 21 | field(:should_fail, :boolean) 22 | field(:sleep, :integer) 23 | field(:from, ErlangTerm) 24 | 25 | honeydew_fields(@classify_queue) 26 | 27 | timestamps() 28 | end 29 | 30 | def classify_queue, do: @classify_queue 31 | end 32 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/repo.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Repo do 2 | use Ecto.Repo, 3 | otp_app: :ecto_poll_queue_example, 4 | adapter: Ecto.Adapters.Postgres 5 | end 6 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/lib/ecto_poll_queue_example/user.ex: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.User do 2 | use Ecto.Schema 3 | import Honeydew.EctoPollQueue.Schema 4 | alias Honeydew.EctoSource.ErlangTerm 5 | 6 | if Mix.env == :cockroach do 7 | @primary_key {:id, :binary_id, autogenerate: false, read_after_writes: true} 8 | @foreign_key_type :binary_id 9 | else 10 | @primary_key {:id, :binary_id, autogenerate: true} 11 | end 12 | 13 | if System.get_env("prefixed_tables") do 14 | @schema_prefix "theprefix" 15 | end 16 | 17 | @notify_queue :notify_user 18 | 19 | schema "users" do 20 | field(:name) 21 | field(:should_fail, :boolean) 22 | field(:sleep, :integer) 23 | field(:from, ErlangTerm) 24 | 25 | honeydew_fields(@notify_queue) 26 | 27 | timestamps() 28 | end 29 | 30 | def honeydew_task(id, _queue) do 31 | {:run, [id]} 32 | end 33 | 34 | def notify_queue, do: @notify_queue 35 | end 36 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/mix.cockroach.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, 3 | "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, 4 | "decimal": {:hex, :decimal, "1.6.0", "bfd84d90ff966e1f5d4370bdd3943432d8f65f07d3bab48001aebd7030590dcc", [:mix], [], "hexpm"}, 5 | "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm"}, 6 | "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"}, 7 | "ecto_sql": {:git, "https://github.com/activeprospect/ecto_sql.git", "d10f10e6c6c47ec21bb8deb6c21a8a357e31e08d", [branch: "v3.0.5-cdb"]}, 8 | "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, 9 | "postgrex": {:git, "https://github.com/activeprospect/postgrex.git", "c249ad706f2b4151fc3d56d874649c8ef0ae9fbe", [branch: "v0.14.1-cdb"]}, 10 | "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"}, 11 | } 12 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :ecto_poll_queue_example, 7 | version: "0.1.0", 8 | lockfile: lockfile(Mix.env()), 9 | elixir: "~> 1.5", 10 | start_permanent: Mix.env() == :prod, 11 | aliases: aliases(), 12 | deps: deps(Mix.env), 13 | dialyzer: [ 14 | flags: [ 15 | :unmatched_returns, 16 | :error_handling, 17 | :race_conditions, 18 | :no_opaque 19 | ] 20 | ] 21 | ] 22 | end 23 | 24 | # Run "mix help compile.app" to learn about applications. 25 | def application do 26 | [ 27 | extra_applications: [:logger, :mix], 28 | mod: {EctoPollQueueExample.Application, []} 29 | ] 30 | end 31 | 32 | defp deps do 33 | [ 34 | {:honeydew, path: "../.."}, 35 | {:dialyxir, "~> 0.5", only: [:cockroach, :postgres], runtime: false} 36 | ] 37 | end 38 | 39 | defp deps(:cockroach) do 40 | [ 41 | {:ecto, "~> 3.0"}, 42 | {:postgrex, github: "activeprospect/postgrex", branch: "v0.14.1-cdb", override: true}, 43 | {:ecto_sql, github: "activeprospect/ecto_sql", branch: "v3.0.5-cdb", override: true}, 44 | {:jason, "~> 1.0"}, 45 | ] ++ deps() 46 | end 47 | 48 | defp deps(:postgres) do 49 | [{:ecto_sql, "~> 3.0"}, 50 | {:postgrex, "~> 0.13"}] ++ deps() 51 | end 52 | 53 | defp aliases do 54 | [ 55 | "ecto.setup": ["ecto.create --quiet", "ecto.migrate --quiet"], 56 | "ecto.reset": ["ecto.drop --quiet", "ecto.setup"], 57 | test: ["ecto.reset", "test"] 58 | ] 59 | end 60 | 61 | defp lockfile(:cockroach), do: "mix.cockroach.lock" 62 | defp lockfile(:postgres), do: "mix.postgres.lock" 63 | defp lockfile(_), do: "mix.lock" 64 | 65 | end 66 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/mix.postgres.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, 3 | "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "ced0780bed50430f770b74fcde870c4a50c815124ecf9fee20d67a465966eb4f"}, 4 | "decimal": {:hex, :decimal, "1.6.0", "bfd84d90ff966e1f5d4370bdd3943432d8f65f07d3bab48001aebd7030590dcc", [:mix], [], "hexpm", "bbd124e240e3ff40f407d50fced3736049e72a73d547f69201484d3a624ab569"}, 5 | "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm", "6c32a70ed5d452c6650916555b1f96c79af5fc4bf286997f8b15f213de786f73"}, 6 | "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "8acd54c5c92c7dbe5a9e76adc22ffb4e2e76e5298989eed2068a4f04bc8e6fef"}, 7 | "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5bd47a499d27084afaa3c2154cfedb478ea2fcc926ef59fa515ee089701e390"}, 8 | "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "a20f189bdd5a219c484818fde18e09ace20cd15fe630a828fde70bd6efdeb23b"}, 9 | "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm", "63d9f37d319ff331a51f6221310deb5aac8ea3dcf5e0369d689121b5e52f72d4"}, 10 | } 11 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/priv/repo/migrations/20180408023001_create_photos_and_users.exs: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Repo.Migrations.CreatePhotosAndUsers do 2 | use Ecto.Migration 3 | import Honeydew.EctoPollQueue.Migration 4 | import EctoPollQueueExample.User, only: [notify_queue: 0] 5 | import EctoPollQueueExample.Photo, only: [classify_queue: 0] 6 | alias Honeydew.EctoSource.ErlangTerm 7 | 8 | def change do 9 | create table(:photos, primary_key: false) do 10 | add :tag, :string 11 | add :should_fail, :boolean 12 | add :sleep, :integer 13 | add :from, ErlangTerm.type() 14 | 15 | if Mix.env == :cockroach do 16 | add :id, :uuid, primary_key: true, default: fragment("gen_random_uuid()") 17 | honeydew_fields(classify_queue(), database: :cockroachdb) 18 | else 19 | add :id, :binary_id, primary_key: true 20 | honeydew_fields(classify_queue()) 21 | end 22 | 23 | timestamps() 24 | end 25 | honeydew_indexes(:photos, classify_queue()) 26 | 27 | create table(:users, primary_key: false) do 28 | add :name, :string 29 | add :should_fail, :boolean 30 | add :sleep, :integer 31 | add :from, ErlangTerm.type() 32 | 33 | if Mix.env == :cockroach do 34 | add :id, :uuid, primary_key: true, default: fragment("gen_random_uuid()") 35 | honeydew_fields(notify_queue(), database: :cockroachdb) 36 | else 37 | add :id, :binary_id, primary_key: true 38 | honeydew_fields(notify_queue()) 39 | end 40 | 41 | timestamps() 42 | end 43 | honeydew_indexes(:users, notify_queue()) 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/priv/repo/migrations/20181228093856_add_prefix_schema.exs: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Repo.Migrations.AddPrefixSchema do 2 | use Ecto.Migration 3 | 4 | if Mix.env() == :postgres do 5 | @prefix "theprefix" 6 | 7 | def up, do: execute("CREATE SCHEMA #{@prefix}") 8 | def down, do: execute("DROP SCHEMA #{@prefix}") 9 | else 10 | def change do 11 | # Cockroach does not support custom schemas 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/priv/repo/migrations/20181228095107_add_prefixed_job_tables.exs: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Repo.Migrations.AddPrefixedJobTables do 2 | use Ecto.Migration 3 | 4 | if Mix.env() == :postgres do 5 | import Honeydew.EctoPollQueue.Migration 6 | import EctoPollQueueExample.User, only: [notify_queue: 0] 7 | import EctoPollQueueExample.Photo, only: [classify_queue: 0] 8 | alias Honeydew.EctoSource.ErlangTerm 9 | 10 | @prefix "theprefix" 11 | 12 | def change do 13 | create table(:photos, primary_key: false, prefix: @prefix) do 14 | add(:tag, :string) 15 | add(:should_fail, :boolean) 16 | add(:sleep, :integer) 17 | add(:from, ErlangTerm.type()) 18 | 19 | if Mix.env() == :cockroach do 20 | add(:id, :uuid, primary_key: true, default: fragment("gen_random_uuid()")) 21 | honeydew_fields(classify_queue(), database: :cockroachdb) 22 | else 23 | add(:id, :binary_id, primary_key: true) 24 | honeydew_fields(classify_queue()) 25 | end 26 | 27 | timestamps() 28 | end 29 | 30 | honeydew_indexes(:photos, classify_queue(), prefix: @prefix) 31 | 32 | create table(:users, primary_key: false, prefix: @prefix) do 33 | add(:name, :string) 34 | add(:should_fail, :boolean) 35 | add(:sleep, :integer) 36 | add(:from, ErlangTerm.type()) 37 | 38 | if Mix.env() == :cockroach do 39 | add(:id, :uuid, primary_key: true, default: fragment("gen_random_uuid()")) 40 | honeydew_fields(notify_queue(), database: :cockroachdb) 41 | else 42 | add(:id, :binary_id, primary_key: true) 43 | honeydew_fields(notify_queue()) 44 | end 45 | 46 | timestamps() 47 | end 48 | 49 | honeydew_indexes(:users, notify_queue(), prefix: @prefix) 50 | end 51 | else 52 | def change do 53 | # Cockroach does not support custom schemas 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/priv/repo/migrations/20190609185555_add_books_table.exs: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExample.Repo.Migrations.AddBooksTable do 2 | use Ecto.Migration 3 | 4 | import Honeydew.EctoPollQueue.Migration 5 | import EctoPollQueueExample.Book, only: [ocr_queue: 0] 6 | 7 | alias Honeydew.EctoSource.ErlangTerm 8 | 9 | def change do 10 | create table(:books, primary_key: false) do 11 | add :author, :string, primary_key: true 12 | add :title, :string, primary_key: true 13 | add :from, ErlangTerm.type() 14 | add :should_fail, :boolean 15 | 16 | if Mix.env == :cockroach do 17 | honeydew_fields(ocr_queue(), database: :cockroachdb) 18 | else 19 | honeydew_fields(ocr_queue()) 20 | end 21 | 22 | timestamps() 23 | end 24 | honeydew_indexes(:books, ocr_queue()) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/test/compound_keys_test.exs: -------------------------------------------------------------------------------- 1 | defmodule CompoundKeysTest do 2 | use ExUnit.Case, async: false 3 | alias EctoPollQueueExample.Repo 4 | alias EctoPollQueueExample.Book 5 | alias Honeydew.EctoSource 6 | 7 | @moduletag :capture_log 8 | 9 | setup do 10 | Repo.delete_all(Book) 11 | :ok 12 | end 13 | 14 | test "automatically enqueues when a new row is saved" do 15 | author = "Yuval Noah Harari" 16 | title = "Sapiens" 17 | 18 | {:ok, %Book{}} = %Book{author: author, title: title, from: self()} |> Repo.insert() 19 | Process.sleep(2_000) 20 | 21 | keys = [author: author, title: title] 22 | 23 | %Book{ 24 | honeydew_ocr_lock: lock, 25 | honeydew_ocr_private: private 26 | } = Repo.get_by(Book, keys) 27 | 28 | assert_receive {:ocr_job_ran, ^keys}, 1_000 29 | 30 | # clears lock 31 | assert is_nil(lock) 32 | # shouldn't be populated, as the job never failed 33 | assert is_nil(private) 34 | end 35 | 36 | test "cancel/2" do 37 | Honeydew.suspend(Book.ocr_queue()) 38 | {:ok, %Book{}} = %Book{author: "Neil Stephenson", title: "Anathem", from: self()} |> Repo.insert() 39 | {:ok, %Book{}} = %Book{author: "Issac Asimov", title: "Rendezvous with Rama", from: self()} |> Repo.insert() 40 | 41 | cancel_keys = [author: "Issac Asimov", title: "Rendezvous with Rama"] 42 | Honeydew.cancel(cancel_keys, Book.ocr_queue()) 43 | 44 | Honeydew.resume(Book.ocr_queue()) 45 | assert_receive {:ocr_job_ran, [author: "Neil Stephenson", title: "Anathem"]}, 1_000 46 | refute_receive {:ocr_job_ran, ^cancel_keys, 500} 47 | end 48 | 49 | test "support inter-job persistent state (retry count, etc)" do 50 | {:ok, %Book{}} = %Book{author: "Hugh D. Young", title: "University Physics", from: self(), should_fail: true} |> Repo.insert() 51 | 52 | primary_keys = [author: "Hugh D. Young", title: "University Physics"] 53 | 54 | Process.sleep(1_000) 55 | assert_receive {:ocr_job_ran, ^primary_keys}, 1_000 56 | 57 | Process.sleep(1_000) 58 | assert_receive {:ocr_job_ran, ^primary_keys}, 1_000 59 | 60 | Process.sleep(1_000) 61 | 62 | %Book{ 63 | honeydew_ocr_lock: lock, 64 | honeydew_ocr_private: private 65 | } = Repo.get_by(Book, primary_keys) 66 | 67 | # job never ran successfully 68 | assert lock == EctoSource.abandoned() 69 | # cleared when job is abandonded 70 | assert is_nil(private) 71 | end 72 | 73 | test "hammer" do 74 | keys = 75 | Enum.map(1..2_000, fn i -> 76 | author = "author_#{i}" 77 | title = "title_#{i}" 78 | 79 | {:ok, %Book{}} = %Book{author: author, title: title, from: self()} |> Repo.insert() 80 | [author: author, title: title] 81 | end) 82 | 83 | Enum.each(keys, fn key -> 84 | assert_receive({:ocr_job_ran, ^key}, 400) 85 | end) 86 | 87 | refute_receive({:ocr_job_ran, _}, 200) 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/test/ecto_poll_queue_example_test.exs: -------------------------------------------------------------------------------- 1 | defmodule EctoPollQueueExampleTest do 2 | use ExUnit.Case, async: false 3 | alias EctoPollQueueExample.Repo 4 | alias EctoPollQueueExample.Photo 5 | alias EctoPollQueueExample.User 6 | alias Honeydew.Job 7 | alias Honeydew.EctoSource 8 | alias Honeydew.EctoSource.State 9 | alias Honeydew.PollQueue.State, as: PollQueueState 10 | alias Honeydew.Queue.State, as: QueueState 11 | alias Honeydew.Processes 12 | 13 | @moduletag :capture_log 14 | 15 | setup do 16 | Honeydew.resume(User.notify_queue()) 17 | Honeydew.resume(Photo.classify_queue()) 18 | Repo.delete_all(Photo) 19 | Repo.delete_all(User) 20 | :ok 21 | end 22 | 23 | test "automatically enqueues when a new row is saved" do 24 | {:ok, %Photo{id: id}} = %Photo{} |> Repo.insert() 25 | Process.sleep(2_000) 26 | 27 | %Photo{ 28 | tag: tag, 29 | honeydew_classify_photos_lock: lock, 30 | honeydew_classify_photos_private: private 31 | } = Repo.get(Photo, id) 32 | 33 | assert is_binary(tag) 34 | # clears lock 35 | assert is_nil(lock) 36 | # shouldn't be populated, as the job never failed 37 | assert is_nil(private) 38 | end 39 | 40 | test "status/1" do 41 | {:ok, _} = %User{from: self(), sleep: 3_000} |> Repo.insert() 42 | {:ok, _} = %User{from: self(), sleep: 3_000} |> Repo.insert() 43 | {:ok, _} = %User{from: self(), sleep: 3_000} |> Repo.insert() 44 | {:ok, _} = %User{from: self(), should_fail: true} |> Repo.insert() 45 | Process.sleep(1_000) 46 | Honeydew.suspend(User.notify_queue()) 47 | 48 | {:ok, _} = %User{from: self(), sleep: 3_000} |> Repo.insert() 49 | {:ok, _} = %User{from: self(), sleep: 3_000} |> Repo.insert() 50 | 51 | assert %{queue: %{count: 6, 52 | abandoned: 0, 53 | ready: 2, 54 | in_progress: 3, 55 | delayed: 1, 56 | stale: 0}} = Honeydew.status(User.notify_queue()) 57 | end 58 | 59 | test "filter/2 abandoned" do 60 | {:ok, _} = %Photo{from: self(), sleep: 10_000} |> Repo.insert() 61 | {:ok, _} = %Photo{from: self(), sleep: 10_000} |> Repo.insert() 62 | {:ok, _} = %Photo{from: self(), sleep: 10_000} |> Repo.insert() 63 | 64 | failed_ids = 65 | Enum.map(1..2, fn _ -> 66 | {:ok, %Photo{id: failed_id}} = %Photo{from: self(), should_fail: true} |> Repo.insert() 67 | failed_id 68 | end) 69 | |> Enum.sort 70 | 71 | Process.sleep(1000) 72 | Honeydew.suspend(Photo.classify_queue()) 73 | {:ok, _} = %Photo{from: self(), sleep: 1_000} |> Repo.insert() 74 | 75 | assert failed_jobs = Honeydew.filter(Photo.classify_queue(), :abandoned) 76 | 77 | ids = 78 | Enum.map(failed_jobs, fn %Job{private: [id: id], queue: queue} -> 79 | assert queue == Photo.classify_queue 80 | id 81 | end) 82 | |> Enum.sort 83 | 84 | assert ids == failed_ids 85 | end 86 | 87 | test "cancel/2" do 88 | Honeydew.suspend(User.notify_queue()) 89 | {:ok, %User{id: id}} = %User{from: self()} |> Repo.insert() 90 | {:ok, %User{id: cancel_id}} = %User{from: self()} |> Repo.insert() 91 | 92 | Honeydew.cancel(cancel_id, User.notify_queue()) 93 | 94 | Honeydew.resume(User.notify_queue()) 95 | assert_receive {:notify_job_ran, ^id}, 1_000 96 | refute_receive {:notify_job_ran, ^cancel_id, 500} 97 | end 98 | 99 | test "resets stale jobs" do 100 | original_state = get_source_state(User.notify_queue()) 101 | 102 | update_source_state(User.notify_queue(), fn state -> 103 | %State{state | stale_timeout: 0} 104 | end) 105 | 106 | {:ok, _user} = %User{from: self(), sleep: 2_000} |> Repo.insert() 107 | 108 | Process.sleep(1_500) 109 | 110 | assert %{queue: %{stale: 1, ready: 0}} = Honeydew.status(User.notify_queue()) 111 | 112 | User.notify_queue() 113 | |> Processes.get_queue() 114 | |> send(:__reset_stale__) 115 | 116 | assert %{queue: %{stale: 0, ready: 1}} = Honeydew.status(User.notify_queue()) 117 | 118 | update_source_state(User.notify_queue(), fn _state -> 119 | original_state 120 | end) 121 | end 122 | 123 | test "support inter-job persistent state (retry count, etc)" do 124 | {:ok, %Photo{id: id}} = %Photo{from: self(), should_fail: true} |> Repo.insert() 125 | 126 | Process.sleep(1_000) 127 | assert_receive {:classify_job_ran, ^id} 128 | 129 | Process.sleep(1_000) 130 | assert_receive {:classify_job_ran, ^id} 131 | 132 | Process.sleep(1_000) 133 | 134 | %Photo{ 135 | tag: tag, 136 | honeydew_classify_photos_lock: lock, 137 | honeydew_classify_photos_private: private 138 | } = Repo.get(Photo, id) 139 | 140 | # job never ran successfully 141 | assert is_nil(tag) 142 | assert lock == EctoSource.abandoned() 143 | # cleared when job is abandonded 144 | assert is_nil(private) 145 | end 146 | 147 | test "delay via nack" do 148 | {:ok, %User{id: id}} = %User{from: self(), should_fail: true} |> Repo.insert() 149 | 150 | delays = 151 | Enum.map(0..3, fn _ -> 152 | receive do 153 | {:notify_job_ran, ^id} -> 154 | DateTime.utc_now() 155 | end 156 | end) 157 | |> Enum.chunk_every(2, 1, :discard) 158 | |> Enum.map(fn [a, b] -> DateTime.diff(b, a) end) 159 | 160 | # 3^0 - 1 -> 0 sec delay 161 | # 3^1 - 1 -> 2 sec delay 162 | # 3^2 - 1 -> 8 sec delay 163 | assert_in_delta Enum.at(delays, 0), 0, 1 164 | assert_in_delta Enum.at(delays, 1), 2, 1 165 | assert_in_delta Enum.at(delays, 2), 8, 1 166 | end 167 | 168 | test "supports :run_if" do 169 | {:ok, %User{id: run_id}} = %User{from: self(), name: "darwin"} |> Repo.insert() 170 | {:ok, %User{id: dont_run_id}} = %User{from: self(), name: "dont run"} |> Repo.insert() 171 | {:ok, %User{id: run_id_1}} = %User{from: self(), name: "odo"} |> Repo.insert() 172 | 173 | Process.sleep(1_000) 174 | 175 | assert_receive {:notify_job_ran, ^run_id} 176 | refute_receive {:notify_job_ran, ^dont_run_id} 177 | assert_receive {:notify_job_ran, ^run_id_1} 178 | end 179 | 180 | test "hammer" do 181 | ids = 182 | Enum.map(1..2_000, fn _ -> 183 | {:ok, %Photo{id: id}} = %Photo{from: self()} |> Repo.insert() 184 | id 185 | end) 186 | 187 | Enum.each(ids, fn id -> 188 | assert_receive({:classify_job_ran, ^id}, 400) 189 | end) 190 | 191 | refute_receive({:classify_job_ran, _}, 200) 192 | end 193 | 194 | defp get_source_state(queue) do 195 | %QueueState{private: %PollQueueState{source: {EctoSource, state}}} = 196 | queue 197 | |> Processes.get_queue() 198 | |> :sys.get_state 199 | 200 | state 201 | end 202 | 203 | defp update_source_state(queue, state_fn) do 204 | queue 205 | |> Processes.get_queue() 206 | |> :sys.replace_state(fn %QueueState{private: %PollQueueState{source: {EctoSource, state}} = poll_queue_state} = queue_state -> 207 | %QueueState{queue_state | 208 | private: %PollQueueState{poll_queue_state | 209 | source: {EctoSource, state_fn.(state)}}} 210 | end) 211 | end 212 | end 213 | -------------------------------------------------------------------------------- /examples/ecto_poll_queue/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /examples/exponential_retry.exs: -------------------------------------------------------------------------------- 1 | # 2 | # iex -S mix run examples/exponential_retry.exs 3 | # 4 | 5 | defmodule Worker do 6 | @behaviour Honeydew.Worker 7 | 8 | def crash(enqueued_at) do 9 | secs_later = DateTime.diff(DateTime.utc_now(), enqueued_at, :millisecond) / 1_000 10 | IO.puts "I ran #{secs_later}s after enqueue!" 11 | raise "crashing on purpose!" 12 | end 13 | end 14 | 15 | defmodule App do 16 | alias Honeydew.FailureMode.ExponentialRetry 17 | 18 | def start do 19 | :ok = Honeydew.start_queue(:my_queue, failure_mode: {ExponentialRetry, [times: 5]}) 20 | :ok = Honeydew.start_workers(:my_queue, Worker) 21 | end 22 | end 23 | 24 | App.start 25 | Logger.configure(level: :error) # don't fill the console with crash reports 26 | {:crash, [DateTime.utc_now()]} |> Honeydew.async(:my_queue, delay_secs: 1) 27 | -------------------------------------------------------------------------------- /examples/filter_and_cancel.exs: -------------------------------------------------------------------------------- 1 | # 2 | # If a job hasn't been started yet, you can cancel it with `Honeydew.cancel/1` 3 | # 4 | # If you don't already have the job struct, you can find it by passing a function to `Honeydew.filter/2` 5 | # 6 | 7 | defmodule Worker do 8 | @behaviour Honeydew.Worker 9 | 10 | def run(i) do 11 | Process.sleep(10_000) 12 | IO.puts "job #{i} finished" 13 | end 14 | end 15 | 16 | defmodule App do 17 | def start do 18 | :ok = Honeydew.start_queue(:my_queue) 19 | :ok = Honeydew.start_workers(:my_queue, Worker, num: 10) 20 | end 21 | end 22 | 23 | 24 | App.start 25 | 26 | # enqueue eleven jobs 27 | Enum.each(0..10, & {:run, [&1]} |> Honeydew.async(:my_queue)) 28 | 29 | # Status indicates that eleven jobs are queued 30 | Honeydew.status(:my_queue) 31 | |> Map.get(:queue) 32 | |> IO.inspect 33 | 34 | # find the job that would run with the argument `10`, as it wont have started yet 35 | # and cancel it 36 | :ok = 37 | Honeydew.filter(:my_queue, %{task: {:run, [10]}}) 38 | |> List.first 39 | |> Honeydew.cancel 40 | 41 | # You should see that there are only ten jobs enqueued now. 42 | Honeydew.status(:my_queue) 43 | |> Map.get(:queue) 44 | |> IO.inspect 45 | -------------------------------------------------------------------------------- /examples/global/README.md: -------------------------------------------------------------------------------- 1 | ### Global Queue Example 2 | ![global queue](global.png) 3 | 4 | Say we've got some pretty heavy tasks that we want to distribute over a farm of background job processing nodes, they're too heavy to process on our client-facing nodes. In a distributed Erlang scenario, you have the option of distributing Honeydew's various components around different nodes in your cluster. Honeydew is basically a simple collection of queue processes and worker processes. Honeydew detects when nodes go up and down, and reconnects workers. 5 | 6 | To start a global queue, pass a `{:global, name}` tuple when you start Honeydew's components 7 | 8 | In this example, we'll use the Mnesia queue with stateless workers. 9 | 10 | We'll start the queue on node `queue@dax` with: 11 | 12 | ```elixir 13 | defmodule QueueApp do 14 | def start do 15 | :ok = Honeydew.start_queue({:global, :my_queue}, queue: {Honeydew.Queue.Mnesia, [disc_copies: nodes]}) 16 | end 17 | end 18 | 19 | iex(queue@dax)1> QueueApp.start 20 | :ok 21 | ``` 22 | 23 | And we'll run our workers on `worker@dax` with: 24 | ```elixir 25 | defmodule HeavyTask do 26 | @behaviour Honeydew.Worker 27 | 28 | # note that in this case, our worker is stateless, so we left out `init/1` 29 | 30 | def work_really_hard(secs) do 31 | :timer.sleep(1_000 * secs) 32 | IO.puts "I worked really hard for #{secs} secs!" 33 | end 34 | end 35 | 36 | defmodule WorkerApp do 37 | def start do 38 | :ok = Honeydew.start_workers({:global, :my_queue}, HeavyTask, num: 10, nodes: [:clientfacing@dax, :queue@dax]) 39 | end 40 | end 41 | 42 | iex(background@dax)1> WorkerApp.start 43 | :ok 44 | ``` 45 | 46 | Note that we've provided a list of nodes to the worker spec, Honeydew will attempt to heal the cluster if any of these nodes go down. 47 | 48 | Then on any node in the cluster, we can enqueue a job: 49 | 50 | ```elixir 51 | iex(clientfacing@dax)1> {:work_really_hard, [5]} |> Honeydew.async({:global, :my_queue}) 52 | %Honeydew.Job{by: nil, failure_private: nil, from: nil, monitor: nil, 53 | private: {false, -576460752303423485}, queue: {:global, :my_queue}, 54 | result: nil, task: {:work_really_hard, [5]}} 55 | ``` 56 | 57 | The job will run on the worker node, five seconds later it'll print `I worked really hard for 5 secs!` 58 | -------------------------------------------------------------------------------- /examples/global/global.exs: -------------------------------------------------------------------------------- 1 | defmodule HeavyTask do 2 | @behaviour Honeydew.Worker 3 | 4 | # note that in this case, our worker is stateless, so we left out `init/1` 5 | 6 | def work_really_hard(secs) do 7 | :timer.sleep(1_000 * secs) 8 | IO.puts "I worked really hard for #{secs} secs!" 9 | end 10 | end 11 | 12 | defmodule QueueApp do 13 | def start do 14 | nodes = [node()] 15 | :ok = Honeydew.start_queue({:global, :my_queue}, queue: {Honeydew.Queue.Mnesia, [disc_copies: nodes]}) 16 | end 17 | end 18 | 19 | defmodule WorkerApp do 20 | def start do 21 | # 22 | # change me! 23 | # 24 | nodes = [:clientfacing@dax, :queue@dax] 25 | :ok = Honeydew.start_workers({:global, :my_queue}, HeavyTask, num: 10, nodes: nodes) 26 | end 27 | end 28 | 29 | # 30 | # - Change nodes above to your hostname. 31 | # 32 | # iex --sname queue -S mix run examples/global/global.exs 33 | # QueueApp.start 34 | # 35 | # iex --sname worker -S mix run examples/global/global.exs 36 | # WorkerApp.start 37 | # 38 | # iex --sname clientfacing -S mix run examples/global/global.exs 39 | # {:work_really_hard, [2]} |> Honeydew.async({:global, :my_queue}) 40 | -------------------------------------------------------------------------------- /examples/global/global.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koudelka/honeydew/7c0e825c70ef4b72c82d02ca95491e7365d6b2e8/examples/global/global.png -------------------------------------------------------------------------------- /examples/initialized_worker/README.md: -------------------------------------------------------------------------------- 1 | ### Initialized Worker Example 2 | 3 | Oftentimes, you'll want to initialize your worker with some kind of state (a database connection, for example). You can do this by implementing the `init/1` callback. The arguments passed to it are those you provided to `Honeydew.start_workers/3`. 4 | 5 | Let's create a worker module. Honeydew will call our worker's `init/1` and keep the `state` from an `{:ok, state}` return. 6 | 7 | ```elixir 8 | defmodule Worker do 9 | 10 | def init([ip, port]) do 11 | {:ok, db} = Database.connect(ip, port) 12 | {:ok, db} 13 | end 14 | 15 | def send_email(id, db) do 16 | %{name: name, email: email} = Database.find(id, db) 17 | 18 | IO.puts "sending email to #{email}" 19 | IO.inspect "hello #{name}, want to enlarge ur keyboard by 500%???" 20 | end 21 | 22 | end 23 | ``` 24 | 25 | If your `init/1` function returns anything other than `{:ok, state}` or raises an error, Honeydew will retry your init function in five seconds. 26 | 27 | 28 | We'll ask Honeydew to start both the queue and workers in its supervision tree. 29 | 30 | ```elixir 31 | defmodule App do 32 | def start do 33 | Honeydew.start_queue(:my_queue) 34 | Honeydew.start_workers(:my_queue, {Worker, ['127.0.0.1', 8087]}) 35 | end 36 | end 37 | ``` 38 | 39 | Add the task to your queue using `async/3`, Honeydew will append your state to the list of arguments. 40 | 41 | ```elixir 42 | iex(1)> {:run, [123]} |> Honeydew.async(:my_queue) 43 | Sending email to koudelka+honeydew@ryoukai.org! 44 | "hello koudelka, want to enlarge ur keyboard by 500%???" 45 | ``` 46 | -------------------------------------------------------------------------------- /examples/initialized_worker/initialized_worker.exs: -------------------------------------------------------------------------------- 1 | # 2 | # elixir -S mix run initialized_worker.exs 3 | # 4 | 5 | defmodule Database do 6 | def connect(ip, port) do 7 | {:ok, "connection to #{ip}:#{port}"} 8 | end 9 | 10 | def find(id, db) do 11 | IO.puts "finding #{id} in db #{inspect db}" 12 | %{id: id, name: "koudelka", email: "koudelka+honeydew@ryoukai.org"} 13 | end 14 | end 15 | 16 | defmodule Worker do 17 | @behaviour Honeydew.Worker 18 | 19 | def init([ip, port]) do 20 | {:ok, db} = Database.connect(ip, port) 21 | {:ok, db} 22 | end 23 | 24 | def send_email(id, db) do 25 | %{name: name, email: email} = Database.find(id, db) 26 | 27 | IO.puts "sending email to #{email}" 28 | IO.inspect "hello #{name}, want to enlarge ur keyboard by 500%???" 29 | end 30 | end 31 | 32 | defmodule App do 33 | def start do 34 | :ok = Honeydew.start_queue(:my_queue) 35 | :ok = Honeydew.start_workers(:my_queue, {Worker, ["database.host", 1234]}) 36 | end 37 | end 38 | 39 | App.start 40 | {:send_email, ["2145"]} |> Honeydew.async(:my_queue) 41 | Process.sleep(100) 42 | -------------------------------------------------------------------------------- /examples/job_replies.exs: -------------------------------------------------------------------------------- 1 | # 2 | # Optionally, you can receive a response from a job with `yield/2`. To tell Honeydew that we expect a response, you must specify `reply: true`, like so: 3 | # 4 | # iex(1)> {:run, [1]} |> Honeydew.async(:my_queue, reply: true) 5 | # {:ok, 2} 6 | # 7 | # If you pass `reply: true`, and you never call `yield/2` to read the result, your process' mailbox may fill up after multiple calls. Don't do that. 8 | 9 | # 10 | 11 | # 12 | # iex -S mix run examples/job_replies.exs 13 | # 14 | 15 | defmodule Worker do 16 | @behaviour Honeydew.Worker 17 | 18 | def run(num) do 19 | 1 + num 20 | end 21 | end 22 | 23 | defmodule App do 24 | def start do 25 | :ok = Honeydew.start_queue(:my_queue) 26 | :ok = Honeydew.start_workers(:my_queue, Worker) 27 | end 28 | end 29 | 30 | App.start 31 | job = {:run, [1]} |> Honeydew.async(:my_queue, reply: true) 32 | job |> Honeydew.yield |> IO.inspect 33 | -------------------------------------------------------------------------------- /examples/local/README.md: -------------------------------------------------------------------------------- 1 | ### Simple Local Example 2 | 3 | ![local queue](local.png) 4 | 5 | Here's a barebones example of a local, in-memory Honeydew queue. 6 | 7 | Let's create a basic worker module: 8 | 9 | ```elixir 10 | defmodule Worker do 11 | @behaviour Honeydew.Worker 12 | 13 | def hello(thing) do 14 | IO.puts "Hello #{thing}!" 15 | end 16 | end 17 | ``` 18 | 19 | Then we'll ask Honeydew to start both the queue and workers in its supervision tree. 20 | 21 | ```elixir 22 | defmodule App do 23 | def start do 24 | :ok = Honeydew.start_queue(:my_queue) 25 | :ok = Honeydew.start_workers(:my_queue, Worker) 26 | end 27 | end 28 | ``` 29 | 30 | A task is simply a tuple with the name of a function and arguments, or a `fn`. In our case, `{:hello, ["World"]}`. 31 | 32 | We'll add tasks to the queue using `async/3`, a worker will then pick up the job and execute it. 33 | 34 | 35 | ```elixir 36 | iex(1)> {:hello, ["World"]} |> Honeydew.async(:my_queue) 37 | Hello World! 38 | ``` 39 | 40 | The `async/3` function returns a `Honeydew.Job` struct. You can call `cancel/1` with it, if you want to try to kill the job. 41 | -------------------------------------------------------------------------------- /examples/local/local.exs: -------------------------------------------------------------------------------- 1 | # 2 | # iex -S mix run examples/local/local.exs 3 | # 4 | 5 | defmodule Worker do 6 | @behaviour Honeydew.Worker 7 | 8 | def hello(thing) do 9 | IO.puts "Hello #{thing}!" 10 | end 11 | end 12 | 13 | defmodule App do 14 | def start do 15 | :ok = Honeydew.start_queue(:my_queue) 16 | :ok = Honeydew.start_workers(:my_queue, Worker) 17 | end 18 | end 19 | 20 | App.start 21 | {:hello, ["World"]} |> Honeydew.async(:my_queue) 22 | -------------------------------------------------------------------------------- /examples/local/local.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/koudelka/honeydew/7c0e825c70ef4b72c82d02ca95491e7365d6b2e8/examples/local/local.png -------------------------------------------------------------------------------- /examples/mnesia.exs: -------------------------------------------------------------------------------- 1 | # 2 | # iex --erl "+C multi_time_warp" -S mix run examples/mnesia.exs 3 | # 4 | 5 | defmodule Worker do 6 | @behaviour Honeydew.Worker 7 | 8 | def hello(thing) do 9 | IO.puts "Hello #{thing}!" 10 | end 11 | end 12 | 13 | defmodule App do 14 | def start do 15 | nodes = [node()] 16 | :ok = Honeydew.start_queue(:my_queue, queue: {Honeydew.Queue.Mnesia, [disc_copies: nodes]}) 17 | :ok = Honeydew.start_workers(:my_queue, Worker) 18 | end 19 | end 20 | 21 | App.start 22 | {:hello, ["World"]} |> Honeydew.async(:my_queue) 23 | -------------------------------------------------------------------------------- /examples/progress_and_queue_status.exs: -------------------------------------------------------------------------------- 1 | # 2 | # Workers can update their current status with `Honeydew.Progress.progress/1`. 3 | # You can view the status of all your workers with `Honeydew.status/1`. 4 | # 5 | 6 | # 7 | # elixir -S mix run progress_and_queue_status.exs 8 | # 9 | 10 | defmodule HeavyTask do 11 | import Honeydew.Progress 12 | 13 | @behaviour Honeydew.Worker 14 | 15 | def work_really_hard(secs) do 16 | progress("Getting ready to do stuff!") 17 | Enum.each 0..secs, fn i -> 18 | Process.sleep(1000) 19 | progress("I've been working hard for #{i} secs!") 20 | end 21 | IO.puts "I worked really hard for #{secs} secs!" 22 | end 23 | end 24 | 25 | 26 | defmodule App do 27 | def start do 28 | :ok = Honeydew.start_queue(:my_queue) 29 | :ok = Honeydew.start_workers(:my_queue, HeavyTask, num: 10) 30 | end 31 | end 32 | 33 | 34 | App.start 35 | 36 | {:work_really_hard, [20]} |> Honeydew.async(:my_queue) 37 | Process.sleep(500) 38 | 39 | # 40 | # The :workers key maps from worker pids to their `{job, job_status}` 41 | # 42 | Honeydew.status(:my_queue) 43 | |> Map.get(:workers) 44 | |> Enum.each(fn 45 | {worker, nil} -> 46 | IO.puts "#{inspect worker} -> idle" 47 | {worker, {_job, status}} -> 48 | IO.puts "#{inspect worker} -> #{inspect status}" 49 | end) 50 | -------------------------------------------------------------------------------- /lib/honeydew/application.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Application do 2 | @moduledoc false 3 | 4 | alias Honeydew.Queues 5 | alias Honeydew.Workers 6 | alias Honeydew.ProcessGroupScopeSupervisor 7 | 8 | use Application 9 | 10 | def start(_type, _args) do 11 | children = [ 12 | {Queues, []}, 13 | {Workers, []}, 14 | {ProcessGroupScopeSupervisor, []} 15 | ] 16 | 17 | opts = [strategy: :one_for_one, name: Honeydew.Supervisor] 18 | Supervisor.start_link(children, opts) 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/honeydew/crash.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Crash do 2 | @moduledoc false 3 | 4 | @type type :: :exception | :throw | :bad_return_value | :exit 5 | 6 | @type t :: %__MODULE__{ 7 | type: type, 8 | reason: term, 9 | stacktrace: Exception.stacktrace() 10 | } 11 | 12 | defstruct [:type, :reason, :stacktrace] 13 | 14 | def new(type, reason, stacktrace) 15 | def new(:exception, %{__struct__: _} = exception, stacktrace) when is_list(stacktrace) do 16 | %__MODULE__{type: :exception, reason: exception, stacktrace: stacktrace} 17 | end 18 | 19 | def new(:throw, reason, stacktrace) when is_list(stacktrace) do 20 | %__MODULE__{type: :throw, reason: reason, stacktrace: stacktrace} 21 | end 22 | 23 | def new(:bad_return_value, value) do 24 | %__MODULE__{type: :bad_return_value, reason: value, stacktrace: []} 25 | end 26 | 27 | def new(:exit, reason) do 28 | %__MODULE__{type: :exit, reason: reason, stacktrace: []} 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/honeydew/dispatcher.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher do 2 | @moduledoc false 3 | #typespecs 4 | end 5 | -------------------------------------------------------------------------------- /lib/honeydew/dispatcher/lru.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher.LRU do 2 | @moduledoc false 3 | 4 | # TODO: docs 5 | 6 | def init do 7 | {:ok, {:queue.new, MapSet.new}} 8 | end 9 | 10 | def available?({free, _busy}) do 11 | !:queue.is_empty(free) 12 | end 13 | 14 | def check_in(worker, {free, busy}) do 15 | {:queue.in(worker, free), MapSet.delete(busy, worker)} 16 | end 17 | 18 | def check_out(_job, {free, busy}) do 19 | case :queue.out(free) do 20 | {{:value, worker}, free} -> 21 | {worker, {free, MapSet.put(busy, worker)}} 22 | {:empty, _free} -> 23 | {nil, {free, busy}} 24 | end 25 | end 26 | 27 | def remove(worker, {free, busy}) do 28 | {:queue.filter(&(&1 != worker), free), MapSet.delete(busy, worker)} 29 | end 30 | 31 | def known?(worker, {free, busy}) do 32 | :queue.member(worker, free) || MapSet.member?(busy, worker) 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/honeydew/dispatcher/lru_node.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher.LRUNode do 2 | @moduledoc false 3 | 4 | alias Honeydew.Dispatcher.LRU 5 | 6 | # TODO: docs 7 | 8 | def init do 9 | # {node_queue, node -> lrus} 10 | {:ok, {:queue.new, Map.new}} 11 | end 12 | 13 | def available?({_node_queue, lrus}) do 14 | lrus 15 | |> Map.values 16 | |> Enum.any?(&LRU.available?/1) 17 | end 18 | 19 | def check_in(worker, {node_queue, lrus}) do 20 | node = worker_node(worker) 21 | 22 | {node_queue, lru} = 23 | lrus 24 | |> Map.get(node) 25 | |> case do 26 | nil -> 27 | # this node isn't currently known 28 | {:ok, lru} = LRU.init 29 | {:queue.in(node, node_queue), lru} 30 | lru -> 31 | # there's already at least one worker from this node present 32 | {node_queue, lru} 33 | end 34 | 35 | {node_queue, Map.put(lrus, node, LRU.check_in(worker, lru))} 36 | end 37 | 38 | def check_out(job, {node_queue, lrus} = state) do 39 | with {{:value, node}, node_queue} <- :queue.out(node_queue), 40 | %{^node => lru} <- lrus, 41 | {worker, lru} when not is_nil(worker) <- LRU.check_out(job, lru) do 42 | unless LRU.available?(lru) do 43 | {worker, {node_queue, Map.delete(lrus, node)}} 44 | else 45 | {worker, {:queue.in(node, node_queue), Map.put(lrus, node, lru)}} 46 | end 47 | else _ -> 48 | {nil, state} 49 | end 50 | end 51 | 52 | def remove(worker, {node_queue, lrus}) do 53 | node = worker_node(worker) 54 | 55 | with %{^node => lru} <- lrus, 56 | lru <- LRU.remove(worker, lru) do 57 | if LRU.available?(lru) do 58 | {node_queue, Map.put(lrus, node, lru)} 59 | else 60 | {:queue.filter(&(&1 != node), node_queue), Map.delete(lrus, node)} 61 | end 62 | else _ -> 63 | # this means that we've been asked to remove a worker we don't know about 64 | # this should never happen :o 65 | {node_queue, lrus} 66 | end 67 | end 68 | 69 | def known?(worker, {_node_queue, lrus}) do 70 | Enum.any?(lrus, fn {_node, lru} -> LRU.known?(worker, lru) end) 71 | end 72 | 73 | # for testing 74 | defp worker_node({_worker, node}), do: node 75 | defp worker_node(worker) do 76 | :erlang.node(worker) 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/honeydew/dispatcher/mru.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher.MRU do 2 | @moduledoc false 3 | 4 | alias Honeydew.Dispatcher.LRU 5 | # TODO: docs 6 | 7 | defdelegate init, to: LRU 8 | defdelegate available?(state), to: LRU 9 | defdelegate check_out(job, state), to: LRU 10 | defdelegate remove(worker, state), to: LRU 11 | defdelegate known?(worker, state), to: LRU 12 | 13 | def check_in(worker, {free, busy}) do 14 | {:queue.in_r(worker, free), MapSet.delete(busy, worker)} 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/honeydew/ecto_poll_queue.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.EctoPollQueue do 2 | @moduledoc """ 3 | The following arguments can be provided when selecting the Ecto Poll Queue module: 4 | 5 | You *must* provide: 6 | 7 | - `repo`: is your Ecto.Repo module 8 | - `schema`: is your Ecto.Schema module 9 | 10 | You may provide: 11 | 12 | - `poll_interval`: is how often Honeydew will poll your database when the queue is silent, in seconds (default: 10) 13 | - `stale_timeout`: is the amount of time a job can take before it risks retry, in seconds (default: 300) 14 | 15 | For example: 16 | 17 | ```elixir 18 | Honeydew.start_queue(:classify_photos, {Honeydew.EctoPollQueue, 19 | repo: MyApp.Repo, 20 | schema: MyApp.Photo}) 21 | ``` 22 | ```elixir 23 | Honeydew.start_queue(:classify_photos, {Honeydew.EctoPollQueue, 24 | repo: MyApp.Repo, 25 | schema: MyApp.Photo}, 26 | failure_mode: {Honeydew.Retry, 27 | times: 3}) 28 | ``` 29 | 30 | """ 31 | 32 | alias Honeydew.PollQueue 33 | alias Honeydew.EctoSource 34 | 35 | @type queue_name :: Honeydew.queue_name() 36 | 37 | @type ecto_poll_queue_spec_opt :: 38 | Honeydew.queue_spec_opt | 39 | {:schema, module} | 40 | {:repo, module} | 41 | {:poll_interval, pos_integer} | 42 | {:stale_timeout, pos_integer} 43 | 44 | def validate_args!(args) do 45 | PollQueue.validate_args!(args) 46 | validate_module_loaded!(args, :schema) 47 | validate_module_loaded!(args, :repo) 48 | validate_stale_timeout!(args[:stale_timeout]) 49 | validate_run_if!(args[:run_if]) 50 | end 51 | 52 | defp validate_module_loaded!(args, type) do 53 | module = Keyword.get(args, type) 54 | 55 | unless module do 56 | raise ArgumentError, argument_not_given_error(args, type) 57 | end 58 | 59 | unless Code.ensure_loaded?(module) do 60 | raise ArgumentError, module_not_loaded_error(module, type) 61 | end 62 | end 63 | 64 | defp validate_stale_timeout!(interval) when is_integer(interval) and interval > 0, do: :ok 65 | defp validate_stale_timeout!(nil), do: :ok 66 | defp validate_stale_timeout!(arg), do: raise ArgumentError, invalid_stale_timeout_error(arg) 67 | 68 | defp validate_run_if!(sql) when is_binary(sql), do: :ok 69 | defp validate_run_if!(nil), do: :ok 70 | defp validate_run_if!(arg), do: raise ArgumentError, invalid_run_if_error(arg) 71 | 72 | defp invalid_stale_timeout_error(argument) do 73 | "Stale timeout must be an integer number of seconds. You gave #{inspect argument}" 74 | end 75 | 76 | defp invalid_run_if_error(argument) do 77 | "Run condition (:run_if) must be an SQL string. You gave #{inspect argument}" 78 | end 79 | 80 | defp argument_not_given_error(args, key) do 81 | "You didn't provide a required argument, #{inspect key}, you gave: #{inspect args}" 82 | end 83 | 84 | defp module_not_loaded_error(module, type) do 85 | "The #{type} module you provided, #{inspect module} couldn't be found" 86 | end 87 | 88 | @doc false 89 | def rewrite_opts([name, __MODULE__, args | rest]) do 90 | {database_override, args} = Keyword.pop(args, :database) 91 | 92 | sql = EctoSource.SQL.module(args[:repo], database_override) 93 | 94 | ecto_source_args = 95 | args 96 | |> Keyword.put(:sql, sql) 97 | |> Keyword.put(:poll_interval, args[:poll_interval] || 10) 98 | |> Keyword.put(:stale_timeout, args[:stale_timeout] || 300) 99 | 100 | [name, PollQueue, [EctoSource, ecto_source_args] | rest] 101 | end 102 | 103 | defmodule Schema do 104 | @moduledoc false 105 | 106 | defmacro honeydew_fields(queue) do 107 | quote do 108 | alias Honeydew.EctoSource.ErlangTerm 109 | 110 | unquote(queue) 111 | |> Honeydew.EctoSource.field_name(:lock) 112 | |> Ecto.Schema.field(:integer) 113 | 114 | unquote(queue) 115 | |> Honeydew.EctoSource.field_name(:private) 116 | |> Ecto.Schema.field(ErlangTerm) 117 | end 118 | end 119 | end 120 | 121 | defmodule Migration do 122 | @moduledoc false 123 | 124 | defmacro honeydew_fields(queue, opts \\ []) do 125 | quote do 126 | require unquote(__MODULE__) 127 | alias Honeydew.EctoSource.SQL 128 | alias Honeydew.EctoSource.ErlangTerm 129 | require SQL 130 | 131 | database = Keyword.get(unquote(opts), :database, nil) 132 | 133 | sql_module = 134 | Ecto.Migration.Runner.repo() 135 | |> SQL.module(database) 136 | 137 | unquote(queue) 138 | |> Honeydew.EctoSource.field_name(:lock) 139 | |> Ecto.Migration.add(sql_module.integer_type(), default: SQL.ready_fragment(sql_module)) 140 | 141 | unquote(queue) 142 | |> Honeydew.EctoSource.field_name(:private) 143 | |> Ecto.Migration.add(ErlangTerm.type()) 144 | end 145 | end 146 | 147 | defmacro honeydew_indexes(table, queue, opts \\ []) do 148 | quote do 149 | lock_field = unquote(queue) |> Honeydew.EctoSource.field_name(:lock) 150 | Ecto.Migration.create(index(unquote(table), [lock_field], unquote(opts))) 151 | end 152 | end 153 | end 154 | 155 | end 156 | -------------------------------------------------------------------------------- /lib/honeydew/failure_mode.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode do 2 | @moduledoc """ 3 | Behaviour module for implementing a job failure mechanism in Honeydew. 4 | 5 | Honeydew comes with the following built-in failure modes: 6 | 7 | * `Honeydew.FailureMode.Abandon` 8 | * `Honeydew.FailureMode.Move` 9 | * `Honeydew.FailureMode.Retry` 10 | """ 11 | alias Honeydew.Job 12 | 13 | @callback validate_args!(args :: list) :: any 14 | @callback handle_failure(job :: %Job{}, reason :: any, args :: list) :: any 15 | end 16 | -------------------------------------------------------------------------------- /lib/honeydew/failure_mode/abandon.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.Abandon do 2 | @moduledoc """ 3 | Instructs Honeydew to abandon a job on failure. 4 | 5 | ## Example: 6 | 7 | ```elixir 8 | Honeydew.start_queue(:my_queue, failure_mode: #{inspect __MODULE__}) 9 | ``` 10 | 11 | """ 12 | require Logger 13 | alias Honeydew.Job 14 | alias Honeydew.Queue 15 | alias Honeydew.Processes 16 | 17 | @behaviour Honeydew.FailureMode 18 | 19 | @impl true 20 | def validate_args!([]), do: :ok 21 | def validate_args!(args), do: raise ArgumentError, "You provided arguments (#{inspect args}) to the Abandon failure mode, it only accepts an empty list" 22 | 23 | @impl true 24 | def handle_failure(%Job{queue: queue, from: from} = job, reason, []) do 25 | Logger.warn "Job failed because #{inspect reason}, abandoning: #{inspect job}" 26 | 27 | # tell the queue that that job can be removed. 28 | queue 29 | |> Processes.get_queue() 30 | |> Queue.ack(job) 31 | 32 | # send the error to the awaiting process, if necessary 33 | with {owner, _ref} <- from, 34 | do: send(owner, %{job | result: {:error, reason}}) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/honeydew/failure_mode/exponential_retry.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.ExponentialRetry do 2 | alias Honeydew.Job 3 | alias Honeydew.FailureMode.Retry 4 | alias Honeydew.FailureMode.Move 5 | 6 | @moduledoc """ 7 | Instructs Honeydew to retry a job a number of times on failure, waiting an exponentially growing number 8 | of seconds between retry attempts. You may specify the base of exponential delay with the `:base` argument, 9 | it defaults to 2. 10 | 11 | Please note, this failure mode will not work with the ErlangQueue queue implementation at the moment. 12 | 13 | ## Examples 14 | 15 | Retry jobs in this queue 3 times, delaying exponentially between with a base of 2: 16 | 17 | ```elixir 18 | Honeydew.start_queue(:my_queue, failure_mode: {#{inspect __MODULE__}, 19 | times: 3, 20 | base: 2}) 21 | ``` 22 | 23 | Retry jobs in this queue 3 times, delaying exponentially between with a base of 2, and then move to another queue: 24 | 25 | ```elixir 26 | Honeydew.start_queue(:my_queue, 27 | failure_mode: {#{inspect __MODULE__}, 28 | times: 3, 29 | base: 2, 30 | finally: {#{inspect Move}, 31 | queue: :dead_letters}}) 32 | ``` 33 | """ 34 | 35 | require Logger 36 | 37 | @behaviour Honeydew.FailureMode 38 | 39 | @impl true 40 | def validate_args!(args) when is_list(args) do 41 | args 42 | |> Enum.into(%{}) 43 | |> validate_args! 44 | end 45 | 46 | def validate_args!(%{base: base}) when not is_integer(base) or base <= 0 do 47 | raise ArgumentError, "You provided a bad `:base` argument (#{inspect base}) to the ExponentialRetry failure mode, it's expecting a positive number." 48 | end 49 | 50 | def validate_args!(args), do: Retry.validate_args!(args, __MODULE__) 51 | 52 | @impl true 53 | def handle_failure(job, reason, args) do 54 | args = 55 | args 56 | |> Keyword.put(:fun, &exponential/3) 57 | |> Keyword.put_new(:base, 2) 58 | 59 | Retry.handle_failure(job, reason, args) 60 | end 61 | 62 | # 63 | # base ^ times_retried - 1, rounded to integer 64 | # 65 | 66 | def exponential(%Job{failure_private: nil} = job, reason, args) do 67 | exponential(%Job{job | failure_private: 0}, reason, args) 68 | end 69 | 70 | def exponential(%Job{failure_private: times_retried} = job, reason, %{times: max_retries, base: base}) when times_retried < max_retries do 71 | delay_secs = (:math.pow(base, times_retried) - 1) |> round() 72 | 73 | Logger.info "Job failed because #{inspect reason}, retrying #{max_retries - times_retried} more times, next attempt in #{delay_secs}s, job: #{inspect job}" 74 | 75 | {:cont, times_retried + 1, delay_secs} 76 | end 77 | def exponential(_, _, _), do: :halt 78 | end 79 | -------------------------------------------------------------------------------- /lib/honeydew/failure_mode/move.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.Move do 2 | @moduledoc """ 3 | Instructs Honeydew to move a job to another queue on failure. 4 | 5 | ## Example 6 | 7 | Move this job to the `:failed` queue, on failure. 8 | 9 | ```elixir 10 | Honeydew.start_queue(:my_queue, failure_mode: {#{inspect __MODULE__}, 11 | queue: :failed}) 12 | ``` 13 | """ 14 | 15 | alias Honeydew.Job 16 | alias Honeydew.Queue 17 | alias Honeydew.Processes 18 | 19 | require Logger 20 | 21 | @behaviour Honeydew.FailureMode 22 | 23 | @impl true 24 | def validate_args!([queue: {:global, queue}]) when is_atom(queue) or is_binary(queue), do: :ok 25 | def validate_args!([queue: queue]), do: validate_args!(queue: {:global, queue}) 26 | def validate_args!(args), do: raise ArgumentError, "You provided arguments (#{inspect args}) to the Move failure mode, it's expecting [queue: to_queue]" 27 | 28 | @impl true 29 | def handle_failure(%Job{queue: queue, from: from} = job, reason, [queue: to_queue]) do 30 | Logger.info "Job failed because #{inspect reason}, moving to #{inspect to_queue}: #{inspect job}" 31 | 32 | # tell the queue that that job can be removed. 33 | queue 34 | |> Processes.get_queue() 35 | |> Queue.ack(job) 36 | 37 | {:ok, job} = 38 | %{job | queue: to_queue} 39 | |> Honeydew.enqueue 40 | 41 | # send the error to the awaiting process, if necessary 42 | with {owner, _ref} <- from, 43 | do: send(owner, %{job | result: {:moved, reason}}) 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/honeydew/failure_mode/retry.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.Retry do 2 | alias Honeydew.Job 3 | alias Honeydew.Queue 4 | alias Honeydew.Processes 5 | alias Honeydew.FailureMode.Abandon 6 | alias Honeydew.FailureMode.Move 7 | 8 | @moduledoc """ 9 | Instructs Honeydew to retry a job a number of times on failure. 10 | 11 | ## Examples 12 | 13 | Retry jobs in this queue 3 times: 14 | 15 | ```elixir 16 | Honeydew.start_queue(:my_queue, failure_mode: {#{inspect __MODULE__}, 17 | times: 3}) 18 | ``` 19 | 20 | Retry jobs in this queue 3 times and then move to another queue: 21 | 22 | ```elixir 23 | Honeydew.start_queue(:my_queue, 24 | failure_mode: {#{inspect __MODULE__}, 25 | times: 3, 26 | finally: {#{inspect Move}, 27 | queue: :dead_letters}}) 28 | ``` 29 | """ 30 | require Logger 31 | 32 | @behaviour Honeydew.FailureMode 33 | 34 | @impl true 35 | def validate_args!(args) when is_list(args) do 36 | args 37 | |> Enum.into(%{}) 38 | |> validate_args!(__MODULE__) 39 | end 40 | 41 | def validate_args!(args, module \\ __MODULE__) 42 | 43 | def validate_args!(%{fun: fun}, _module) when is_function(fun, 3), do: :ok 44 | def validate_args!(%{fun: bad}, module) do 45 | raise ArgumentError, "You provided a bad `:fun` argument (#{inspect bad}) to the #{module} failure mode, it's expecting a function or function capture of arity three (job, failure_reason, args), for example: `&#{inspect __MODULE__}.immediate/3`" 46 | end 47 | 48 | def validate_args!(%{times: times}, module) when not is_integer(times) or times <= 0 do 49 | raise ArgumentError, "You provided a bad `:times` argument (#{inspect times}) to the #{module} failure mode, it's expecting a positive integer." 50 | end 51 | 52 | def validate_args!(%{finally: {module, args} = bad}, module) when not is_atom(module) or not is_list(args) do 53 | raise ArgumentError, "You provided a bad `:finally` argument (#{inspect bad}) to the #{module} failure mode, it's expecting `finally: {module, args}`" 54 | end 55 | 56 | def validate_args!(%{times: _times, finally: {m, a}}, _module) do 57 | m.validate_args!(a) 58 | end 59 | 60 | def validate_args!(%{times: _times}, _module), do: :ok 61 | 62 | def validate_args!(bad, module) do 63 | raise ArgumentError, "You provided bad arguments (#{inspect bad}) to the #{module} failure mode, at a minimum, it must be a list with a maximum number of retries specified, for example: `[times: 5]`" 64 | end 65 | 66 | 67 | @impl true 68 | def handle_failure(%Job{queue: queue, from: from} = job, reason, args) when is_list(args) do 69 | args = Enum.into(args, %{}) 70 | args = Map.merge(%{finally: {Abandon, []}, 71 | fun: &immediate/3}, args) 72 | 73 | %{fun: fun, finally: {finally_module, finally_args}} = args 74 | 75 | case fun.(job, reason, args) do 76 | {:cont, private, delay_secs} -> 77 | job = %Job{job | failure_private: private, delay_secs: delay_secs, result: {:retrying, reason}} 78 | 79 | queue 80 | |> Processes.get_queue() 81 | |> Queue.nack(job) 82 | 83 | # send the error to the awaiting process, if necessary 84 | with {owner, _ref} <- from, 85 | do: send(owner, %{job | result: {:retrying, reason}}) 86 | 87 | :halt -> 88 | finally_module.handle_failure(%{job | failure_private: nil, delay_secs: 0}, reason, finally_args) 89 | end 90 | end 91 | 92 | def immediate(%Job{failure_private: nil} = job, reason, args) do 93 | immediate(%Job{job | failure_private: 0}, reason, args) 94 | end 95 | 96 | def immediate(%Job{failure_private: times_retried} = job, reason, %{times: max_retries}) when times_retried < max_retries do 97 | Logger.info "Job failed because #{inspect reason}, retrying #{max_retries - times_retried} more times, job: #{inspect job}" 98 | 99 | {:cont, times_retried + 1, 0} 100 | end 101 | def immediate(_, _, _), do: :halt 102 | end 103 | -------------------------------------------------------------------------------- /lib/honeydew/job.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Job do 2 | @moduledoc """ 3 | A Honeydew job. 4 | """ 5 | 6 | @type private :: term() 7 | 8 | defstruct [:private, # queue's private state 9 | :failure_private, # failure mode's private state 10 | :task, 11 | :from, # if the requester wants the result, here's where to send it 12 | :result, 13 | :by, # node last processed the job 14 | :queue, 15 | :job_monitor, 16 | :enqueued_at, 17 | :started_at, 18 | :completed_at, 19 | {:delay_secs, 0}] 20 | 21 | @type t :: %__MODULE__{ 22 | task: Honeydew.task | nil, 23 | queue: Honeydew.queue_name, 24 | private: private, 25 | delay_secs: integer() 26 | } 27 | 28 | @doc false 29 | def new(task, queue) do 30 | %__MODULE__{task: task, queue: queue, enqueued_at: System.system_time(:millisecond)} 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/honeydew/job_monitor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.JobMonitor do 2 | @moduledoc false 3 | 4 | use GenServer, restart: :transient 5 | 6 | alias Honeydew.Crash 7 | alias Honeydew.Logger, as: HoneydewLogger 8 | alias Honeydew.Queue 9 | alias Honeydew.Job 10 | 11 | require Logger 12 | require Honeydew 13 | 14 | # when the queue casts a job to a worker, it spawns a local JobMonitor with the job as state, 15 | # the JobMonitor watches the worker, if the worker dies (or its node is disconnected), the JobMonitor returns the 16 | # job to the queue. it waits @claim_delay miliseconds for the worker to confirm receipt of the job. 17 | 18 | @claim_delay 1_000 # ms 19 | 20 | defmodule State do 21 | @moduledoc false 22 | 23 | defstruct [:queue_pid, :worker, :job, :failure_mode, :success_mode, :progress] 24 | end 25 | 26 | def start_link(job, queue_pid, failure_mode, success_mode) do 27 | GenServer.start_link(__MODULE__, [job, queue_pid, failure_mode, success_mode]) 28 | end 29 | 30 | def init([job, queue_pid, failure_mode, success_mode]) do 31 | Process.send_after(self(), :return_job, @claim_delay) 32 | 33 | {:ok, %State{job: job, 34 | queue_pid: queue_pid, 35 | failure_mode: failure_mode, 36 | success_mode: success_mode, 37 | progress: :awaiting_claim}} 38 | end 39 | 40 | 41 | # 42 | # Internal API 43 | # 44 | 45 | def claim(job_monitor, job), do: GenServer.call(job_monitor, {:claim, job}) 46 | def job_succeeded(job_monitor, result), do: GenServer.call(job_monitor, {:job_succeeded, result}) 47 | def job_failed(job_monitor, %Crash{} = reason), do: GenServer.call(job_monitor, {:job_failed, reason}) 48 | def status(job_monitor), do: GenServer.call(job_monitor, :status) 49 | def progress(job_monitor, progress), do: GenServer.call(job_monitor, {:progress, progress}) 50 | 51 | 52 | def handle_call({:claim, job}, {worker, _ref}, %State{worker: nil} = state) do 53 | Honeydew.debug "[Honeydew] Monitor #{inspect self()} had job #{inspect job.private} claimed by worker #{inspect worker}" 54 | Process.monitor(worker) 55 | job = %{job | started_at: System.system_time(:millisecond)} 56 | {:reply, :ok, %{state | job: job, worker: worker, progress: :running}} 57 | end 58 | 59 | def handle_call(:status, _from, %State{job: job, worker: worker, progress: progress} = state) do 60 | {:reply, {worker, {job, progress}}, state} 61 | end 62 | 63 | def handle_call({:progress, progress}, _from, state) do 64 | {:reply, :ok, %{state | progress: {:running, progress}}} 65 | end 66 | 67 | def handle_call({:job_succeeded, result}, {worker, _ref}, %State{job: %Job{from: from} = job, queue_pid: queue_pid, worker: worker, success_mode: success_mode} = state) do 68 | job = %{job | completed_at: System.system_time(:millisecond), result: {:ok, result}} 69 | 70 | with {owner, _ref} <- from, 71 | do: send(owner, job) 72 | 73 | Queue.ack(queue_pid, job) 74 | 75 | with {success_mode_module, success_mode_args} <- success_mode, 76 | do: success_mode_module.handle_success(job, success_mode_args) 77 | 78 | {:stop, :normal, :ok, reset(state)} 79 | end 80 | 81 | def handle_call({:job_failed, reason}, {worker, _ref}, %State{worker: worker} = state) do 82 | execute_failure_mode(reason, state) 83 | 84 | {:stop, :normal, :ok, reset(state)} 85 | end 86 | 87 | # no worker has claimed the job, return it 88 | def handle_info(:return_job, %State{job: job, queue_pid: queue_pid, worker: nil} = state) do 89 | Queue.nack(queue_pid, job) 90 | 91 | {:stop, :normal, reset(state)} 92 | end 93 | def handle_info(:return_job, state), do: {:noreply, state} 94 | 95 | # worker died while busy 96 | def handle_info({:DOWN, _ref, :process, worker, reason}, %State{worker: worker, job: job} = state) do 97 | crash = Crash.new(:exit, reason) 98 | HoneydewLogger.job_failed(job, crash) 99 | execute_failure_mode(crash, state) 100 | 101 | {:stop, :normal, reset(state)} 102 | end 103 | 104 | def handle_info(msg, state) do 105 | Logger.warn "[Honeydew] Monitor #{inspect self()} received unexpected message #{inspect msg}" 106 | {:noreply, state} 107 | end 108 | 109 | defp reset(state) do 110 | %{state | job: nil, progress: :about_to_die} 111 | end 112 | 113 | defp execute_failure_mode(%Crash{} = crash, %State{job: job, failure_mode: {failure_mode, failure_mode_args}}) do 114 | failure_mode.handle_failure(job, format_failure_reason(crash), failure_mode_args) 115 | end 116 | 117 | defp format_failure_reason(%Crash{type: :exception, reason: exception, stacktrace: stacktrace}) do 118 | {exception, stacktrace} 119 | end 120 | 121 | defp format_failure_reason(%Crash{type: :throw, reason: thrown, stacktrace: stacktrace}) do 122 | {thrown, stacktrace} 123 | end 124 | 125 | defp format_failure_reason(%Crash{type: :exit, reason: reason}), do: reason 126 | end 127 | -------------------------------------------------------------------------------- /lib/honeydew/job_runner.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.JobRunner do 2 | @moduledoc false 3 | 4 | use GenServer 5 | require Logger 6 | require Honeydew 7 | 8 | alias Honeydew.Job 9 | alias Honeydew.Crash 10 | alias Honeydew.Worker 11 | 12 | @doc false 13 | def start_link(opts) do 14 | GenServer.start_link(__MODULE__, opts) 15 | end 16 | 17 | defmodule State do 18 | @moduledoc false 19 | 20 | defstruct [:worker, :job, :module, :worker_private] 21 | end 22 | 23 | 24 | # 25 | # Internal API 26 | # 27 | 28 | def run_link(job, module, worker_private) do 29 | GenServer.start_link(__MODULE__, [self(), job, module, worker_private]) 30 | end 31 | 32 | @impl true 33 | def init([worker, %Job{job_monitor: job_monitor} = job, module, worker_private]) do 34 | Process.put(:job_monitor, job_monitor) 35 | 36 | {:ok, %State{job: job, 37 | module: module, 38 | worker: worker, 39 | worker_private: worker_private}, {:continue, :run}} 40 | end 41 | 42 | defp do_run(%State{job: %Job{task: task} = job, 43 | module: module, 44 | worker: worker, 45 | worker_private: worker_private} = state) do 46 | private_args = 47 | case worker_private do 48 | {:state, s} -> [s] 49 | :no_state -> [] 50 | end 51 | 52 | result = 53 | try do 54 | result = 55 | case task do 56 | f when is_function(f) -> apply(f, private_args) 57 | f when is_atom(f) -> apply(module, f, private_args) 58 | {f, a} -> apply(module, f, a ++ private_args) 59 | end 60 | {:ok, result} 61 | rescue e -> 62 | {:error, Crash.new(:exception, e, __STACKTRACE__)} 63 | catch 64 | :exit, reason -> 65 | # catch exit signals and shut down in an orderly manner 66 | {:error, Crash.new(:exit, reason)} 67 | e -> 68 | {:error, Crash.new(:throw, e, __STACKTRACE__)} 69 | end 70 | 71 | :ok = Worker.job_finished(worker, %{job | result: result}) 72 | 73 | state 74 | end 75 | 76 | 77 | @impl true 78 | def handle_continue(:run, state) do 79 | state = do_run(state) 80 | 81 | {:stop, :normal, state} 82 | end 83 | 84 | @impl true 85 | def terminate(:normal, _state), do: :ok 86 | def terminate(:shutdown, _state), do: :ok 87 | def terminate({:shutdown, _}, _state), do: :ok 88 | def terminate(reason, _state) do 89 | Logger.info "[Honeydew] JobRunner #{inspect self()} stopped because #{inspect reason}" 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/honeydew/logger.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Logger do 2 | @moduledoc false 3 | 4 | alias Honeydew.Crash 5 | alias Honeydew.Logger.Metadata 6 | alias Honeydew.Job 7 | 8 | require Logger 9 | 10 | def worker_init_crashed(module, %Crash{type: :exception, reason: exception} = crash) do 11 | Logger.warn(fn -> 12 | { 13 | "#{module}.init/1 must return {:ok, state :: any()}, but raised #{inspect(exception)}", 14 | honeydew_crash_reason: Metadata.build_crash_reason(crash) 15 | } 16 | end) 17 | end 18 | 19 | def worker_init_crashed(module, %Crash{type: :throw, reason: thrown} = crash) do 20 | Logger.warn(fn -> 21 | { 22 | "#{module}.init/1 must return {:ok, state :: any()}, but threw #{inspect(thrown)}", 23 | honeydew_crash_reason: Metadata.build_crash_reason(crash) 24 | } 25 | end) 26 | end 27 | 28 | def worker_init_crashed(module, %Crash{type: :bad_return_value, reason: value} = crash) do 29 | Logger.warn(fn -> 30 | { 31 | "#{module}.init/1 must return {:ok, state :: any()}, got: #{inspect value}", 32 | honeydew_crash_reason: Metadata.build_crash_reason(crash) 33 | } 34 | end) 35 | end 36 | 37 | def job_failed(%Job{} = job, %Crash{type: :exception} = crash) do 38 | Logger.warn(fn -> 39 | { 40 | """ 41 | Job failed due to exception. #{inspect(job)} 42 | #{format_crash_for_log(crash)} 43 | """, 44 | honeydew_crash_reason: Metadata.build_crash_reason(crash), 45 | honeydew_job: job 46 | } 47 | end) 48 | end 49 | 50 | def job_failed(%Job{} = job, %Crash{type: :throw} = crash) do 51 | Logger.warn(fn -> 52 | { 53 | """ 54 | Job failed due to uncaught throw. #{inspect job}", 55 | #{format_crash_for_log(crash)} 56 | """, 57 | honeydew_crash_reason: Metadata.build_crash_reason(crash), 58 | honeydew_job: job 59 | } 60 | end) 61 | end 62 | 63 | def job_failed(%Job{} = job, %Crash{type: :exit} = crash) do 64 | Logger.warn(fn -> 65 | { 66 | """ 67 | Job failed due unexpected exit. #{inspect job}", 68 | #{format_crash_for_log(crash)} 69 | """, 70 | honeydew_crash_reason: Metadata.build_crash_reason(crash), 71 | honeydew_job: job 72 | } 73 | end) 74 | end 75 | 76 | defp format_crash_for_log(%Crash{type: :exception, reason: exception, stacktrace: stacktrace}) do 77 | Exception.format(:error, exception, stacktrace) 78 | end 79 | 80 | defp format_crash_for_log(%Crash{type: :throw, reason: exception, stacktrace: stacktrace}) do 81 | Exception.format(:throw, exception, stacktrace) 82 | end 83 | 84 | defp format_crash_for_log(%Crash{type: :exit, reason: reason, stacktrace: []}) do 85 | Exception.format(:exit, reason, []) 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/honeydew/logger/metadata.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Logger.Metadata do 2 | @moduledoc false 3 | 4 | alias Honeydew.Crash 5 | 6 | def build_crash_reason(%Crash{type: :exception, reason: exception, stacktrace: stacktrace}) do 7 | {exception, stacktrace} 8 | end 9 | 10 | def build_crash_reason(%Crash{type: :throw, reason: thrown, stacktrace: stacktrace}) do 11 | {{:nocatch, thrown}, stacktrace} 12 | end 13 | 14 | def build_crash_reason(%Crash{type: :bad_return_value, reason: value, stacktrace: stacktrace}) do 15 | {{:bad_return_value, value}, stacktrace} 16 | end 17 | 18 | def build_crash_reason(%Crash{type: :exit, reason: value, stacktrace: stacktrace}) do 19 | {{:exit, value}, stacktrace} 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/honeydew/node_monitor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.NodeMonitor do 2 | @moduledoc false 3 | 4 | use GenServer, restart: :transient 5 | require Logger 6 | 7 | @interval 1_000 # ms 8 | 9 | def start_link(node) do 10 | GenServer.start_link(__MODULE__, node) 11 | end 12 | 13 | def init(node) do 14 | :ok = :net_kernel.monitor_nodes(true) 15 | 16 | GenServer.cast(self(), :ping) 17 | 18 | {:ok, node} 19 | end 20 | 21 | def handle_cast(:ping, node) do 22 | node 23 | |> Node.ping 24 | |> case do 25 | :pang -> :timer.apply_after(@interval, GenServer, :cast, [self(), :ping]) 26 | _ -> :noop 27 | end 28 | {:noreply, node} 29 | end 30 | 31 | def handle_info({:nodeup, node}, node) do 32 | Logger.info "[Honeydew] Connection to #{node} established." 33 | 34 | {:noreply, node} 35 | end 36 | def handle_info({:nodeup, _}, node), do: {:noreply, node} 37 | 38 | def handle_info({:nodedown, node}, node) do 39 | Logger.warn "[Honeydew] Lost connection to #{node}, attempting to reestablish..." 40 | 41 | GenServer.cast(self(), :ping) 42 | 43 | {:noreply, node} 44 | end 45 | def handle_info({:nodedown, _}, node), do: {:noreply, node} 46 | 47 | end 48 | -------------------------------------------------------------------------------- /lib/honeydew/node_monitor_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.NodeMonitorSupervisor do 2 | @moduledoc false 3 | 4 | use DynamicSupervisor 5 | alias Honeydew.NodeMonitor 6 | 7 | def start_link([_queue, nodes]) do 8 | {:ok, supervisor} = DynamicSupervisor.start_link(__MODULE__, [], []) 9 | 10 | Enum.each(nodes, fn node -> 11 | DynamicSupervisor.start_child(supervisor, {NodeMonitor, node}) 12 | end) 13 | 14 | {:ok, supervisor} 15 | end 16 | 17 | @impl true 18 | def init(_) do 19 | DynamicSupervisor.init(strategy: :one_for_one) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/honeydew/poll_queue.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.PollQueue do 2 | @moduledoc false 3 | 4 | require Logger 5 | alias Honeydew.Job 6 | alias Honeydew.Queue 7 | alias Honeydew.Queue.State, as: QueueState 8 | 9 | @behaviour Queue 10 | 11 | @type job :: Job.t() 12 | @type private :: any() 13 | @type name :: Honeydew.queue_name() 14 | @type filter :: atom() 15 | 16 | @callback init(name, arg :: any()) :: {:ok, private} 17 | @callback reserve(private) :: {job, private} 18 | @callback ack(job, private) :: private 19 | @callback nack(job, private) :: private 20 | @callback status(private) :: %{:count => number, :in_progress => number, optional(atom) => any} 21 | @callback cancel(job, private) :: {:ok | {:error, :in_progress | :not_found}, private} 22 | @callback filter(private, filter) :: [job] 23 | 24 | @callback handle_info(msg :: :timeout | term, state :: private) :: 25 | {:noreply, new_state} 26 | | {:noreply, new_state, timeout | :hibernate} 27 | | {:stop, reason :: term, new_state} 28 | when new_state: private 29 | 30 | defmodule State do 31 | @moduledoc false 32 | 33 | defstruct [:queue, :source, :poll_interval] 34 | end 35 | 36 | @impl true 37 | def validate_args!(args) do 38 | validate_poll_interval!(args[:poll_interval]) 39 | end 40 | 41 | defp validate_poll_interval!(interval) when is_integer(interval) and interval > 0, do: :ok 42 | defp validate_poll_interval!(nil), do: :ok 43 | defp validate_poll_interval!(arg), do: raise ArgumentError, invalid_poll_interval_error(arg) 44 | 45 | defp invalid_poll_interval_error(argument) do 46 | "Poll interval must be positive integer number of seconds. You gave #{inspect argument}" 47 | end 48 | 49 | @impl true 50 | def init(queue, [source, args]) do 51 | poll_interval = args[:poll_interval] * 1_000 |> trunc 52 | 53 | {:ok, source_state} = source.init(queue, args) 54 | source = {source, source_state} 55 | 56 | poll(poll_interval) 57 | 58 | {:ok, %State{queue: queue, source: source, poll_interval: poll_interval}} 59 | end 60 | 61 | @impl true 62 | def enqueue(job, state) do 63 | nack(job, state) 64 | {state, job} 65 | end 66 | 67 | @impl true 68 | def reserve(%State{source: {source, source_state}} = state) do 69 | case source.reserve(source_state) do 70 | {:empty, source_state} -> 71 | {:empty, %{state | source: {source, source_state}}} 72 | 73 | {{:value, {id, job}}, source_state} -> 74 | {%{job | private: id}, %{state | source: {source, source_state}}} 75 | end 76 | end 77 | 78 | @impl true 79 | def ack(job, %State{source: {source, source_state}} = state) do 80 | %{state | source: {source, source.ack(job, source_state)}} 81 | end 82 | 83 | @impl true 84 | def nack(job, %State{source: {source, source_state}} = state) do 85 | %{state | source: {source, source.nack(job, source_state)}} 86 | end 87 | 88 | @impl true 89 | def status(%State{source: {source, source_state}}) do 90 | source.status(source_state) 91 | end 92 | 93 | @impl true 94 | def filter(%State{source: {source, source_state}}, filter) when is_atom(filter) do 95 | source.filter(source_state, filter) 96 | end 97 | 98 | @impl true 99 | def filter(_state, _filter) do 100 | raise "Implementations of PollQueue only support predefined filters (atoms)" 101 | end 102 | 103 | @impl true 104 | def cancel(job, %State{source: {source, source_state}} = state) do 105 | {response, source_state} = source.cancel(job, source_state) 106 | {response, %{state | source: {source, source_state}}} 107 | end 108 | 109 | @impl true 110 | def handle_info(:__poll__, %QueueState{private: %State{poll_interval: poll_interval}} = queue_state) do 111 | poll(poll_interval) 112 | {:noreply, Queue.dispatch(queue_state)} 113 | end 114 | 115 | @impl true 116 | def handle_info(msg, %QueueState{private: %State{source: {source, _source_state}}} = queue_state) do 117 | source.handle_info(msg, queue_state) 118 | end 119 | 120 | defp poll(poll_interval) do 121 | {:ok, _} = :timer.send_after(poll_interval, :__poll__) 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /lib/honeydew/process_group_scope_supervisor.ex: -------------------------------------------------------------------------------- 1 | # 2 | # Dynamic supervision of :pg scopes (one per queue). 3 | # 4 | defmodule Honeydew.ProcessGroupScopeSupervisor do 5 | @moduledoc false 6 | 7 | use DynamicSupervisor 8 | 9 | def start_link([]) do 10 | DynamicSupervisor.start_link(__MODULE__, [], name: __MODULE__) 11 | end 12 | 13 | def init(extra_args) do 14 | DynamicSupervisor.init(strategy: :one_for_one, extra_arguments: extra_args) 15 | end 16 | 17 | def start_scope(name) do 18 | child_spec = %{ 19 | id: name, 20 | start: {:pg, :start_link, [name]} 21 | } 22 | 23 | DynamicSupervisor.start_child(__MODULE__, child_spec) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/honeydew/processes.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Processes do 2 | @moduledoc false 3 | 4 | alias Honeydew.ProcessGroupScopeSupervisor 5 | alias Honeydew.Queues 6 | alias Honeydew.WorkerGroupSupervisor 7 | alias Honeydew.WorkerStarter 8 | alias Honeydew.Workers 9 | 10 | @processes [WorkerGroupSupervisor, WorkerStarter] 11 | 12 | for process <- @processes do 13 | def process(queue, unquote(process)) do 14 | name(queue, unquote(process)) 15 | end 16 | end 17 | 18 | def start_process_group_scope(queue) do 19 | queue 20 | |> scope() 21 | |> ProcessGroupScopeSupervisor.start_scope() 22 | |> case do 23 | {:ok, _pid} -> 24 | :ok 25 | 26 | {:error, {:already_started, _pid}} -> 27 | :ok 28 | end 29 | end 30 | 31 | def join_group(component, queue, pid) do 32 | queue 33 | |> scope() 34 | |> :pg.join(component, pid) 35 | end 36 | 37 | # 38 | # this function may be in a hot path, so we don't call Processes.start_process_group/1 unless necessary 39 | # 40 | def get_queue(queue) do 41 | case get_queues(queue) do 42 | [queue_process | _rest] -> 43 | queue_process 44 | 45 | [] -> 46 | start_process_group_scope(queue) 47 | 48 | case get_queues(queue) do 49 | [queue_process | _rest] -> 50 | queue_process 51 | 52 | [] -> 53 | raise RuntimeError, Honeydew.no_queues_running_error(queue) 54 | end 55 | end 56 | end 57 | 58 | def get_queues(queue) do 59 | get_members(queue, Queues) 60 | end 61 | 62 | def get_workers(queue) do 63 | get_members(queue, Workers) 64 | end 65 | 66 | def get_local_members(queue, group) do 67 | queue 68 | |> scope() 69 | |> :pg.get_local_members(group) 70 | end 71 | 72 | defp get_members({:global, _} = queue, group) do 73 | queue 74 | |> scope() 75 | |> :pg.get_members(group) 76 | end 77 | 78 | defp get_members(queue, name) do 79 | get_local_members(queue, name) 80 | end 81 | 82 | defp scope(queue) do 83 | name(queue, "scope") 84 | end 85 | 86 | defp name({:global, queue}, component) do 87 | name([:global, queue], component) 88 | end 89 | 90 | defp name(queue, component) do 91 | [component, queue] 92 | |> List.flatten() 93 | |> Enum.join(".") 94 | |> String.to_atom() 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/honeydew/progress.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Progress do 2 | @moduledoc false 3 | 4 | alias Honeydew.JobMonitor 5 | 6 | def progress(update) do 7 | :ok = 8 | :job_monitor 9 | |> Process.get 10 | |> JobMonitor.progress(update) 11 | end 12 | 13 | end 14 | -------------------------------------------------------------------------------- /lib/honeydew/queue/erlang_queue.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Queue.ErlangQueue do 2 | @moduledoc """ 3 | An in-memory queue implementation. 4 | 5 | This is a simple FIFO queue implemented with the `:queue` and `Map` modules. 6 | """ 7 | require Logger 8 | alias Honeydew.Job 9 | alias Honeydew.Queue 10 | 11 | @behaviour Queue 12 | 13 | @impl true 14 | def validate_args!([]), do: :ok 15 | def validate_args!(args), do: raise ArgumentError, "You provided arguments (#{inspect args}) to the #{__MODULE__} queue, it's expecting an empty list, or just the bare module." 16 | 17 | @impl true 18 | def init(_name, []) do 19 | # {pending, in_progress} 20 | {:ok, {:queue.new, Map.new}} 21 | end 22 | 23 | # 24 | # Enqueue/Reservee 25 | # 26 | 27 | @impl true 28 | def enqueue(job, {pending, in_progress}) do 29 | job = %{job | private: :erlang.unique_integer} 30 | {{:queue.in(job, pending), in_progress}, job} 31 | end 32 | 33 | @impl true 34 | def reserve({pending, in_progress} = state) do 35 | case :queue.out(pending) do 36 | {:empty, _pending} -> 37 | {:empty, state} 38 | {{:value, job}, pending} -> 39 | {job, {pending, Map.put(in_progress, job.private, job)}} 40 | end 41 | end 42 | 43 | # 44 | # Ack/Nack 45 | # 46 | 47 | @impl true 48 | def ack(%Job{private: id}, {pending, in_progress}) do 49 | {pending, Map.delete(in_progress, id)} 50 | end 51 | 52 | @impl true 53 | def nack(%Job{private: id} = job, {pending, in_progress}) do 54 | {:queue.in_r(job, pending), Map.delete(in_progress, id)} 55 | end 56 | 57 | # 58 | # Helpers 59 | # 60 | 61 | @impl true 62 | def status({pending, in_progress}) do 63 | %{count: :queue.len(pending) + map_size(in_progress), 64 | in_progress: map_size(in_progress)} 65 | end 66 | 67 | @impl true 68 | def filter({pending, in_progress}, function) do 69 | (function |> :queue.filter(pending) |> :queue.to_list) ++ 70 | (in_progress |> Map.values |> Enum.filter(function)) 71 | end 72 | 73 | @impl true 74 | def cancel(%Job{private: private}, {pending, in_progress}) do 75 | filter = fn 76 | %Job{private: ^private} -> false; 77 | _ -> true 78 | end 79 | 80 | new_pending = :queue.filter(filter, pending) 81 | 82 | reply = cond do 83 | :queue.len(pending) > :queue.len(new_pending) -> :ok 84 | in_progress |> Map.values |> Enum.filter(&(!filter.(&1))) |> Enum.count > 0 -> {:error, :in_progress} 85 | true -> {:error, :not_found} 86 | end 87 | 88 | {reply, {new_pending, in_progress}} 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/honeydew/queue/mnesia/wrapped_job.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Queue.Mnesia.WrappedJob do 2 | alias Honeydew.Job 3 | 4 | @record_name :wrapped_job 5 | @record_fields [:key, :job] 6 | 7 | job_filter_map = 8 | %Job{} 9 | |> Map.from_struct() 10 | |> Enum.map(fn {k, _} -> 11 | {k, :_} 12 | end) 13 | 14 | @job_filter struct(Job, job_filter_map) 15 | 16 | defstruct [:run_at, 17 | :id, 18 | :job] 19 | 20 | def record_name, do: @record_name 21 | def record_fields, do: @record_fields 22 | 23 | def new(%Job{delay_secs: delay_secs} = job) do 24 | id = :erlang.unique_integer() 25 | run_at = now() + delay_secs 26 | 27 | job = %{job | private: id} 28 | 29 | %__MODULE__{run_at: run_at, 30 | id: id, 31 | job: job} 32 | end 33 | 34 | def from_record({@record_name, {run_at, id}, job}) do 35 | %__MODULE__{run_at: run_at, 36 | id: id, 37 | job: job} 38 | end 39 | 40 | def to_record(%__MODULE__{run_at: run_at, 41 | id: id, 42 | job: job}) do 43 | {@record_name, key(run_at, id), job} 44 | end 45 | 46 | def key({@record_name, key, _job}) do 47 | key 48 | end 49 | 50 | def key(run_at, id) do 51 | {run_at, id} 52 | end 53 | 54 | def id_from_key({_run_at, id}) do 55 | id 56 | end 57 | 58 | def id_pattern(id) do 59 | %__MODULE__{ 60 | id: id, 61 | run_at: :_, 62 | job: :_ 63 | } 64 | |> to_record 65 | end 66 | 67 | def filter_pattern(map) do 68 | job = struct(@job_filter, map) 69 | 70 | %__MODULE__{ 71 | id: :_, 72 | run_at: :_, 73 | job: job 74 | } 75 | |> to_record 76 | end 77 | 78 | def reserve_match_spec do 79 | pattern = 80 | %__MODULE__{ 81 | id: :_, 82 | run_at: :"$1", 83 | job: :_ 84 | } 85 | |> to_record 86 | 87 | [{ 88 | pattern, 89 | [{:"=<", :"$1", now()}], 90 | [:"$_"] 91 | }] 92 | end 93 | 94 | defp now do 95 | :erlang.monotonic_time(:second) 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /lib/honeydew/queue_monitor.ex: -------------------------------------------------------------------------------- 1 | # when a queue dies, this process hears about it and stops its associated worker group. 2 | defmodule Honeydew.QueueMonitor do 3 | @moduledoc false 4 | 5 | use GenServer 6 | require Logger 7 | 8 | defmodule State do 9 | @moduledoc false 10 | 11 | defstruct [:workers_per_queue_pid, :queue_pid] 12 | end 13 | 14 | def start_link(opts) do 15 | GenServer.start_link(__MODULE__, opts) 16 | end 17 | 18 | def init([workers_per_queue_pid, queue_pid]) do 19 | Process.monitor(queue_pid) 20 | {:ok, %State{workers_per_queue_pid: workers_per_queue_pid, queue_pid: queue_pid}} 21 | end 22 | 23 | def handle_info({:DOWN, _, _, queue_pid, _}, %State{workers_per_queue_pid: workers_per_queue_pid, queue_pid: queue_pid} = state) do 24 | Logger.info "[Honeydew] Queue process #{inspect queue_pid} on node #{node queue_pid} stopped, stopping local workers." 25 | 26 | Supervisor.stop(workers_per_queue_pid) 27 | 28 | {:noreply, state} 29 | end 30 | 31 | def handle_info(msg, state) do 32 | Logger.warn "[QueueMonitor] Received unexpected message #{inspect msg}" 33 | {:noreply, state} 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/honeydew/queues.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Queues do 2 | @moduledoc false 3 | 4 | use Supervisor 5 | alias Honeydew.Queue 6 | alias Honeydew.Queue.Mnesia 7 | alias Honeydew.Dispatcher.LRUNode 8 | alias Honeydew.Dispatcher.LRU 9 | alias Honeydew.FailureMode.Abandon 10 | alias Honeydew.Processes 11 | 12 | @type name :: Honeydew.queue_name() 13 | @type queue_spec_opt :: Honeydew.queue_spec_opt() 14 | 15 | @spec queues() :: [name] 16 | def queues do 17 | __MODULE__ 18 | |> Supervisor.which_children 19 | |> Enum.map(fn {queue, _, _, _} -> queue end) 20 | |> Enum.sort 21 | end 22 | 23 | @spec stop_queue(name) :: :ok | {:error, :not_running} 24 | def stop_queue(name) do 25 | with :ok <- Supervisor.terminate_child(__MODULE__, name) do 26 | Supervisor.delete_child(__MODULE__, name) 27 | end 28 | end 29 | 30 | @spec start_queue(name, [queue_spec_opt]) :: :ok | {:error, term()} 31 | def start_queue(name, opts) do 32 | {module, args} = 33 | case opts[:queue] do 34 | nil -> {Mnesia, [ram_copies: [node()]]} 35 | module when is_atom(module) -> {module, []} 36 | {module, args} -> {module, args} 37 | end 38 | 39 | dispatcher = 40 | opts[:dispatcher] || 41 | case name do 42 | {:global, _} -> {LRUNode, []} 43 | _ -> {LRU, []} 44 | end 45 | 46 | failure_mode = 47 | case opts[:failure_mode] do 48 | nil -> {Abandon, []} 49 | {module, args} -> {module, args} 50 | module when is_atom(module) -> {module, []} 51 | end 52 | 53 | {failure_module, failure_args} = failure_mode 54 | failure_module.validate_args!(failure_args) 55 | 56 | success_mode = 57 | case opts[:success_mode] do 58 | nil -> nil 59 | {module, args} -> {module, args} 60 | module when is_atom(module) -> {module, []} 61 | end 62 | 63 | with {success_module, success_args} <- success_mode do 64 | success_module.validate_args!(success_args) 65 | end 66 | 67 | suspended = Keyword.get(opts, :suspended, false) 68 | 69 | module.validate_args!(args) 70 | 71 | opts = [name, module, args, dispatcher, failure_mode, success_mode, suspended] 72 | 73 | opts = 74 | :functions 75 | |> module.__info__ 76 | |> Enum.member?({:rewrite_opts, 1}) 77 | |> if do 78 | module.rewrite_opts(opts) 79 | else 80 | opts 81 | end 82 | 83 | Processes.start_process_group_scope(name) 84 | 85 | with {:ok, _} <- Supervisor.start_child(__MODULE__, Queue.child_spec(name, opts)) do 86 | :ok 87 | end 88 | end 89 | 90 | def start_link(args) do 91 | Supervisor.start_link(__MODULE__, args, name: __MODULE__) 92 | end 93 | 94 | @impl true 95 | def init(_args) do 96 | Supervisor.init([], strategy: :one_for_one) 97 | end 98 | 99 | end 100 | -------------------------------------------------------------------------------- /lib/honeydew/sources/ecto/erlang_term.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(Ecto) do 2 | defmodule Honeydew.EctoSource.ErlangTerm do 3 | @moduledoc false 4 | 5 | if macro_exported?(Ecto.Type, :__using__, 1) do 6 | use Ecto.Type 7 | else 8 | @behaviour Ecto.Type 9 | end 10 | 11 | @impl true 12 | def type, do: :binary 13 | 14 | @impl true 15 | def cast(term) do 16 | {:ok, term} 17 | end 18 | 19 | @impl true 20 | def load(binary) when is_binary(binary) do 21 | {:ok, :erlang.binary_to_term(binary)} 22 | end 23 | 24 | @impl true 25 | def dump(term) do 26 | {:ok, :erlang.term_to_binary(term)} 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/honeydew/sources/ecto/sql.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.EctoSource.SQL do 2 | @moduledoc false 3 | 4 | alias Honeydew.EctoSource.State 5 | alias Honeydew.EctoSource.SQL.Cockroach 6 | alias Honeydew.EctoSource.SQL.Postgres 7 | 8 | # 9 | # you might be wondering "what's all this shitty sql for?", it's to make sure that the database is sole arbiter of "now", 10 | # in case of clock skew between the various nodes running this queue 11 | # 12 | 13 | @type sql :: String.t() 14 | @type msecs :: integer() 15 | @type repo :: module() 16 | @type override :: :cockroachdb | nil 17 | @type sql_module :: Postgres | Cockroach 18 | @type filter :: atom 19 | 20 | @callback integer_type :: atom() 21 | @callback reserve(State.t()) :: sql 22 | @callback cancel(State.t()) :: sql 23 | @callback ready :: sql 24 | @callback delay_ready(State.t()) :: sql 25 | @callback status(State.t()) :: sql 26 | @callback filter(State.t(), filter) :: sql 27 | @callback reset_stale(State.t()) :: sql 28 | @callback table_name(module()) :: String.t() 29 | 30 | @spec module(repo, override) :: sql_module | no_return 31 | def module(repo, override) do 32 | case override do 33 | :cockroachdb -> 34 | Cockroach 35 | 36 | nil -> 37 | case repo.__adapter__() do 38 | Ecto.Adapters.Postgres -> 39 | Postgres 40 | 41 | unsupported -> 42 | raise ArgumentError, unsupported_adapter_error(unsupported) 43 | end 44 | end 45 | end 46 | 47 | defmacro ready_fragment(module) do 48 | quote do 49 | unquote(module).ready() 50 | |> fragment() 51 | end 52 | end 53 | 54 | @doc false 55 | defp unsupported_adapter_error(adapter) do 56 | "your repo's ecto adapter, #{inspect(adapter)}, isn't currently supported, but it's probably not hard to implement, open an issue and we'll chat!" 57 | end 58 | 59 | # "I left in love, in laughter, and in truth. And wherever truth, love and laughter abide, I am there in spirit." 60 | @spec far_in_the_past() :: NaiveDateTime.t() 61 | def far_in_the_past do 62 | ~N[1994-03-26 04:20:00] 63 | end 64 | 65 | @spec where_keys_fragment(State.t(), pos_integer()) :: sql 66 | def where_keys_fragment(%State{key_fields: key_fields}, starting_index) do 67 | key_fields 68 | |> Enum.with_index 69 | |> Enum.map(fn {key_field, i} -> "#{key_field} = $#{i + starting_index}" end) 70 | |> Enum.join(" AND ") 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/honeydew/sources/ecto/sql/cockroach.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(Ecto) do 2 | defmodule Honeydew.EctoSource.SQL.Cockroach do 3 | @moduledoc false 4 | 5 | alias Honeydew.EctoSource 6 | alias Honeydew.EctoSource.SQL 7 | alias Honeydew.EctoSource.State 8 | 9 | @behaviour SQL 10 | 11 | @impl true 12 | def integer_type do 13 | :bigint 14 | end 15 | 16 | @impl true 17 | def table_name(schema) do 18 | schema.__schema__(:source) 19 | end 20 | 21 | @impl true 22 | def ready do 23 | "CAST('#{SQL.far_in_the_past()}' AS TIMESTAMP)" 24 | |> time_in_msecs 25 | |> msecs_ago 26 | end 27 | 28 | @impl true 29 | def delay_ready(state) do 30 | "UPDATE #{state.table} 31 | SET #{state.lock_field} = (#{ready()} + $1 * 1000), 32 | #{state.private_field} = $2 33 | WHERE #{SQL.where_keys_fragment(state, 3)}" 34 | end 35 | 36 | @impl true 37 | def reserve(state) do 38 | returning_fragment = [state.private_field | state.key_fields] |> Enum.join(", ") 39 | 40 | "UPDATE #{state.table} 41 | SET #{state.lock_field} = #{reserve_at(state)} 42 | WHERE #{state.lock_field} BETWEEN 0 AND #{ready()} #{run_if(state)} 43 | ORDER BY #{state.lock_field} 44 | LIMIT 1 45 | RETURNING #{returning_fragment}" 46 | end 47 | 48 | defp run_if(%State{run_if: nil}), do: nil 49 | defp run_if(%State{run_if: run_if}), do: "AND (#{run_if})" 50 | 51 | @impl true 52 | def cancel(state) do 53 | "UPDATE #{state.table} 54 | SET #{state.lock_field} = NULL 55 | WHERE 56 | #{SQL.where_keys_fragment(state, 1)} 57 | RETURNING #{state.lock_field}" 58 | end 59 | 60 | 61 | @impl true 62 | def status(state) do 63 | "SELECT COUNT(IF(#{state.lock_field} IS NOT NULL, 1, NULL)) AS count, 64 | 65 | COUNT(IF(#{state.lock_field} = #{EctoSource.abandoned()}, 1, NULL)) AS abandoned, 66 | 67 | COUNT(IF( 68 | 0 <= #{state.lock_field} AND #{state.lock_field} <= #{ready()}, 69 | 1, NULL)) AS ready, 70 | 71 | COUNT(IF( 72 | #{ready()} < #{state.lock_field} AND #{state.lock_field} < #{stale_at()}, 73 | 1, NULL)) AS delayed, 74 | 75 | COUNT(IF( 76 | #{stale_at()} < #{state.lock_field} AND #{state.lock_field} < #{now()}, 77 | 1, NULL)) AS stale, 78 | 79 | COUNT(IF( 80 | #{now()} < #{state.lock_field} AND #{state.lock_field} <= #{reserve_at(state)}, 81 | 1, NULL)) AS in_progress 82 | FROM #{state.table}" 83 | end 84 | 85 | @impl true 86 | def filter(state, :abandoned) do 87 | keys_fragment = Enum.join(state.key_fields, ", ") 88 | "SELECT #{keys_fragment} FROM #{state.table} WHERE #{state.lock_field} = #{EctoSource.abandoned()}" 89 | end 90 | 91 | @impl true 92 | def reset_stale(state) do 93 | "UPDATE #{state.table} 94 | SET #{state.lock_field} = DEFAULT, 95 | #{state.private_field} = DEFAULT 96 | WHERE 97 | #{stale_at()} < #{state.lock_field} 98 | AND 99 | #{state.lock_field} < #{now()}" 100 | end 101 | 102 | defp reserve_at(state) do 103 | "#{now()} + #{state.stale_timeout}" 104 | end 105 | 106 | defp stale_at do 107 | "(NOW() - INTERVAL '5 year')" 108 | |> time_in_msecs 109 | end 110 | 111 | defp msecs_ago(msecs) do 112 | "#{now()} - #{msecs}" 113 | end 114 | 115 | defp now do 116 | time_in_msecs("NOW()") 117 | end 118 | 119 | defp time_in_msecs(time) do 120 | "(EXTRACT('millisecond', (#{time})) + CAST((#{time}) AS INT) * 1000)" 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /lib/honeydew/sources/ecto/sql/postgres.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(Ecto) do 2 | defmodule Honeydew.EctoSource.SQL.Postgres do 3 | @moduledoc false 4 | 5 | alias Honeydew.EctoSource 6 | alias Honeydew.EctoSource.SQL 7 | alias Honeydew.EctoSource.State 8 | 9 | @behaviour SQL 10 | 11 | @impl true 12 | def integer_type do 13 | :bigint 14 | end 15 | 16 | @impl true 17 | def table_name(schema) do 18 | source = schema.__schema__(:source) 19 | prefix = schema.__schema__(:prefix) 20 | 21 | if prefix do 22 | prefix <> "." <> source 23 | else 24 | source 25 | end 26 | end 27 | 28 | @impl true 29 | def ready do 30 | SQL.far_in_the_past() 31 | |> timestamp_in_msecs 32 | |> msecs_ago 33 | end 34 | 35 | @impl true 36 | def delay_ready(state) do 37 | "UPDATE #{state.table} 38 | SET #{state.lock_field} = (#{ready()} + CAST($1 AS BIGINT) * 1000), 39 | #{state.private_field} = $2 40 | WHERE 41 | #{SQL.where_keys_fragment(state, 3)}" 42 | end 43 | 44 | @impl true 45 | def reserve(state) do 46 | key_list_fragment = Enum.join(state.key_fields, ", ") 47 | returning_fragment = [state.private_field | state.key_fields] |> Enum.join(", ") 48 | 49 | "UPDATE #{state.table} 50 | SET #{state.lock_field} = #{reserve_at(state)} 51 | WHERE ROW(#{key_list_fragment}) = ( 52 | SELECT #{key_list_fragment} 53 | FROM #{state.table} 54 | WHERE #{state.lock_field} BETWEEN 0 AND #{ready()} #{run_if(state)} 55 | ORDER BY #{state.lock_field} 56 | LIMIT 1 57 | FOR UPDATE SKIP LOCKED 58 | ) 59 | RETURNING #{returning_fragment}" 60 | end 61 | 62 | defp run_if(%State{run_if: nil}), do: nil 63 | defp run_if(%State{run_if: run_if}), do: "AND (#{run_if})" 64 | 65 | @impl true 66 | def cancel(state) do 67 | "UPDATE #{state.table} 68 | SET #{state.lock_field} = NULL 69 | WHERE 70 | #{SQL.where_keys_fragment(state, 1)} 71 | RETURNING #{state.lock_field}" 72 | end 73 | 74 | @impl true 75 | def status(state) do 76 | "SELECT COUNT(#{state.lock_field}) AS count, 77 | COUNT(*) FILTER (WHERE #{state.lock_field} = #{EctoSource.abandoned()}) AS abandoned, 78 | 79 | COUNT(*) FILTER (WHERE #{state.lock_field} BETWEEN 0 AND #{ready()}) AS ready, 80 | 81 | COUNT(*) FILTER (WHERE #{ready()} < #{state.lock_field} AND #{state.lock_field} < #{stale_at()}) AS delayed, 82 | 83 | COUNT(*) FILTER (WHERE #{stale_at()} <= #{state.lock_field} AND #{state.lock_field} < #{now()}) AS stale, 84 | 85 | COUNT(*) FILTER (WHERE #{now()} < #{state.lock_field} AND #{state.lock_field} <= #{reserve_at(state)}) AS in_progress 86 | FROM #{state.table}" 87 | end 88 | 89 | @impl true 90 | def reset_stale(state) do 91 | "UPDATE #{state.table} 92 | SET #{state.lock_field} = DEFAULT, 93 | #{state.private_field} = DEFAULT 94 | WHERE 95 | #{stale_at()} < #{state.lock_field} 96 | AND 97 | #{state.lock_field} < #{now()}" 98 | end 99 | 100 | @impl true 101 | def filter(state, :abandoned) do 102 | keys_fragment = Enum.join(state.key_fields, ", ") 103 | "SELECT #{keys_fragment} FROM #{state.table} WHERE #{state.lock_field} = #{EctoSource.abandoned()}" 104 | end 105 | 106 | def reserve_at(state) do 107 | "#{now()} + #{state.stale_timeout}" 108 | end 109 | 110 | def stale_at do 111 | time_in_msecs("(NOW() - INTERVAL '5 year')") 112 | end 113 | 114 | defp msecs_ago(msecs) do 115 | "#{now()} - #{msecs}" 116 | end 117 | 118 | def now do 119 | time_in_msecs("NOW()") 120 | end 121 | 122 | defp timestamp_in_msecs(time) do 123 | time_in_msecs("timestamp '#{time}'") 124 | end 125 | 126 | defp time_in_msecs(time) do 127 | "(CAST(EXTRACT(epoch from #{time}) * 1000 AS BIGINT))" 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /lib/honeydew/sources/ecto/state.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.EctoSource.State do 2 | @moduledoc false 3 | 4 | defstruct [ 5 | :schema, 6 | :repo, 7 | :sql, 8 | :table, 9 | :key_fields, 10 | :lock_field, 11 | :private_field, 12 | :task_fn, 13 | :queue, 14 | :stale_timeout, 15 | :reset_stale_interval, 16 | :run_if 17 | ] 18 | 19 | @type stale_timeout :: pos_integer 20 | 21 | @type t :: %__MODULE__{schema: module, 22 | repo: module, 23 | sql: module, 24 | table: String.t(), 25 | key_fields: [atom()], 26 | lock_field: String.t(), 27 | private_field: String.t(), 28 | stale_timeout: stale_timeout, 29 | task_fn: function(), 30 | queue: Honeydew.queue_name(), 31 | reset_stale_interval: pos_integer(), 32 | run_if: String.t()} 33 | end 34 | -------------------------------------------------------------------------------- /lib/honeydew/success_mode.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.SuccessMode do 2 | @moduledoc """ 3 | Behaviour module for implementing a job success mode. 4 | 5 | Honeydew comes with the following built-in failure modes: 6 | 7 | * `Honeydew.SuccessMode.Log` 8 | """ 9 | alias Honeydew.Job 10 | 11 | @callback validate_args!(args :: list) :: any 12 | @callback handle_success(job :: %Job{}, args :: list) :: any 13 | end 14 | -------------------------------------------------------------------------------- /lib/honeydew/success_mode/log.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.SuccessMode.Log do 2 | @moduledoc """ 3 | Instructs Honeydew to log the job when it succeeds. 4 | 5 | This logging might be too verbose for most needs, it's mostly an example of how to write a SuccessMode. 6 | 7 | ## Example 8 | 9 | ```elixir 10 | Honeydew.start_queue(:my_queue, success_mode: #{inspect __MODULE__}) 11 | ``` 12 | """ 13 | 14 | require Logger 15 | alias Honeydew.Job 16 | @behaviour Honeydew.SuccessMode 17 | 18 | @impl true 19 | def validate_args!([]), do: :ok 20 | def validate_args!(args), do: raise ArgumentError, "You provided arguments (#{inspect args}) to the Log success mode, it only accepts an empty list" 21 | 22 | @impl true 23 | def handle_success(%Job{enqueued_at: enqueued_at, started_at: started_at, completed_at: completed_at, result: {:ok, result}} = job, []) do 24 | Logger.info fn -> 25 | queue_time = started_at - enqueued_at 26 | run_time = completed_at - started_at 27 | "Job #{inspect job} completed, sat in queue for #{queue_time}ms, took #{run_time}ms to complete, and returned #{inspect result}." 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/honeydew/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Worker do 2 | @moduledoc """ 3 | A Honeydew Worker, how you specify your jobs. 4 | """ 5 | use GenServer 6 | require Logger 7 | require Honeydew 8 | 9 | alias Honeydew.Crash 10 | alias Honeydew.Job 11 | alias Honeydew.JobMonitor 12 | alias Honeydew.Logger, as: HoneydewLogger 13 | alias Honeydew.Processes 14 | alias Honeydew.Queue 15 | alias Honeydew.JobRunner 16 | alias Honeydew.Workers 17 | alias Honeydew.WorkerSupervisor 18 | 19 | @type private :: term 20 | 21 | @doc """ 22 | Invoked when the worker starts up for the first time. 23 | """ 24 | @callback init(args :: term) :: {:ok, state :: private} 25 | 26 | @doc """ 27 | Invoked when `init/1` returns anything other than `{:ok, state}` or raises an error 28 | """ 29 | @callback init_failed() :: any() 30 | 31 | @optional_callbacks init: 1, init_failed: 0 32 | 33 | defmodule State do 34 | @moduledoc false 35 | 36 | defstruct [:queue, 37 | :queue_pid, 38 | :module, 39 | :has_init_fcn, 40 | :init_args, 41 | :init_retry_secs, 42 | :start_opts, 43 | :job_runner, 44 | :job, 45 | {:ready, false}, 46 | {:private, :no_state}] 47 | end 48 | 49 | @doc false 50 | def child_spec([_supervisor, _queue, %{shutdown: shutdown}, _queue_pid] = opts) do 51 | %{ 52 | id: __MODULE__, 53 | start: {__MODULE__, :start_link, [opts]}, 54 | restart: :transient, 55 | shutdown: shutdown 56 | } 57 | end 58 | 59 | @doc false 60 | def start_link(opts) do 61 | GenServer.start_link(__MODULE__, opts) 62 | end 63 | 64 | @impl true 65 | @doc false 66 | def init([_supervisor, queue, %{ma: {module, init_args}, init_retry_secs: init_retry_secs}, queue_pid] = start_opts) do 67 | Process.flag(:trap_exit, true) 68 | 69 | :ok = Processes.join_group(Workers, queue, self()) 70 | 71 | has_init_fcn = 72 | :functions 73 | |> module.__info__ 74 | |> Enum.member?({:init, 1}) 75 | 76 | {:ok, %State{queue: queue, 77 | queue_pid: queue_pid, 78 | module: module, 79 | init_args: init_args, 80 | init_retry_secs: init_retry_secs, 81 | start_opts: start_opts, 82 | has_init_fcn: has_init_fcn}, {:continue, :module_init}} 83 | end 84 | 85 | # 86 | # Internal API 87 | # 88 | 89 | @doc false 90 | def run(worker, job, job_monitor), do: GenServer.cast(worker, {:run, %{job | job_monitor: job_monitor}}) 91 | @doc false 92 | def module_init(me \\ self()), do: GenServer.cast(me, :module_init) 93 | @doc false 94 | def ready(ready), do: GenServer.cast(self(), {:ready, ready}) 95 | @doc false 96 | def job_finished(worker, job), do: GenServer.call(worker, {:job_finished, job}) 97 | 98 | 99 | defp do_module_init(%State{has_init_fcn: false} = state) do 100 | %{state | ready: true} |> send_ready_or_callback 101 | end 102 | 103 | defp do_module_init(%State{module: module, init_args: init_args} = state) do 104 | try do 105 | case apply(module, :init, [init_args]) do 106 | {:ok, private} -> 107 | %{state | private: {:state, private}, ready: true} 108 | bad -> 109 | HoneydewLogger.worker_init_crashed(module, Crash.new(:bad_return_value, bad)) 110 | %{state | ready: false} 111 | end 112 | rescue e -> 113 | HoneydewLogger.worker_init_crashed(module, Crash.new(:exception, e, __STACKTRACE__)) 114 | %{state | ready: false} 115 | catch 116 | :exit, reason -> 117 | HoneydewLogger.worker_init_crashed(module, Crash.new(:exit, reason)) 118 | %{state | ready: false} 119 | e -> 120 | HoneydewLogger.worker_init_crashed(module, Crash.new(:throw, e, __STACKTRACE__)) 121 | %{state | ready: false} 122 | end 123 | |> send_ready_or_callback 124 | end 125 | 126 | defp send_ready_or_callback(%State{queue_pid: queue_pid, ready: true} = state) do 127 | Honeydew.debug "[Honeydew] Worker #{inspect self()} sending ready" 128 | 129 | Process.link(queue_pid) 130 | Queue.worker_ready(queue_pid) 131 | 132 | state 133 | end 134 | 135 | defp send_ready_or_callback(%State{module: module, init_retry_secs: init_retry_secs} = state) do 136 | :functions 137 | |> module.__info__ 138 | |> Enum.member?({:failed_init, 0}) 139 | |> if do 140 | module.failed_init 141 | else 142 | Logger.info "[Honeydew] Worker #{inspect self()} re-initing in #{init_retry_secs}s" 143 | :timer.apply_after(init_retry_secs * 1_000, __MODULE__, :module_init, [self()]) 144 | end 145 | 146 | state 147 | end 148 | 149 | defp do_run(%Job{job_monitor: job_monitor} = job, %State{module: module, private: private} = state) do 150 | job = %{job | by: node()} 151 | 152 | :ok = JobMonitor.claim(job_monitor, job) 153 | 154 | {:ok, job_runner} = JobRunner.run_link(job, module, private) 155 | 156 | %{state | job_runner: job_runner, job: job} 157 | end 158 | 159 | defp do_job_finished(%Job{result: {:ok, result}, job_monitor: job_monitor}, %State{queue_pid: queue_pid}) do 160 | :ok = JobMonitor.job_succeeded(job_monitor, result) 161 | 162 | Process.delete(:job_monitor) 163 | Queue.worker_ready(queue_pid) 164 | 165 | :ok 166 | end 167 | 168 | defp do_job_finished(%Job{result: {:error, %Crash{} = crash}, job_monitor: job_monitor} = job, _state) do 169 | HoneydewLogger.job_failed(job, crash) 170 | :ok = JobMonitor.job_failed(job_monitor, crash) 171 | Process.delete(:job_monitor) 172 | 173 | :error 174 | end 175 | 176 | @impl true 177 | @doc false 178 | def handle_continue(:module_init, state) do 179 | {:noreply, do_module_init(state)} 180 | end 181 | 182 | @doc false 183 | def handle_continue({:job_finished, job}, state) do 184 | case do_job_finished(job, state) do 185 | :ok -> 186 | {:noreply, state} 187 | :error -> 188 | restart(state) 189 | end 190 | end 191 | 192 | @impl true 193 | @doc false 194 | def handle_cast(:module_init, state), do: {:noreply, do_module_init(state)} 195 | def handle_cast({:run, job}, state), do: {:noreply, do_run(job, state)} 196 | 197 | @impl true 198 | @doc false 199 | def handle_call({:job_finished, job}, {job_runner, _ref}, %State{job_runner: job_runner} = state) do 200 | Process.unlink(job_runner) 201 | {:reply, :ok, %{state | job_runner: nil, job: nil}, {:continue, {:job_finished, job}}} 202 | end 203 | 204 | # 205 | # Our Queue died, our QueueMonitor will stop us soon. 206 | # 207 | @impl true 208 | @doc false 209 | def handle_info({:EXIT, queue_pid, _reason}, %State{queue: queue, queue_pid: queue_pid} = state) do 210 | Logger.warn "[Honeydew] Worker #{inspect queue} (#{inspect self()}) saw its queue die, stopping..." 211 | {:noreply, state} 212 | end 213 | 214 | def handle_info({:EXIT, _pid, :normal}, %State{job_runner: nil} = state), do: {:noreply, state} 215 | def handle_info({:EXIT, _pid, :shutdown}, %State{job_runner: nil} = state), do: {:noreply, state} 216 | def handle_info({:EXIT, _pid, {:shutdown, _}}, %State{job_runner: nil} = state), do: {:noreply, state} 217 | 218 | def handle_info({:EXIT, job_runner, reason}, %State{job_runner: job_runner, queue: queue, job: job} = state) do 219 | Logger.warn "[Honeydew] Worker #{inspect queue} (#{inspect self()}) saw its job runner (#{inspect job_runner}) die during a job, restarting..." 220 | job = %{job | result: {:error, Crash.new(:exit, reason)}} 221 | {:noreply, state, {:continue, {:job_finished, job}}} 222 | end 223 | 224 | def handle_info(msg, %State{queue: queue} = state) do 225 | Logger.warn "[Honeydew] Worker #{inspect queue} (#{inspect self()}) received unexpected message #{inspect msg}, restarting..." 226 | restart(state) 227 | end 228 | 229 | 230 | defp restart(%State{start_opts: start_opts} = state) do 231 | WorkerSupervisor.start_worker(start_opts) 232 | {:stop, :normal, state} 233 | end 234 | end 235 | -------------------------------------------------------------------------------- /lib/honeydew/worker_group_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.WorkerGroupSupervisor do 2 | @moduledoc false 3 | 4 | use DynamicSupervisor 5 | 6 | alias Honeydew.WorkersPerQueueSupervisor 7 | alias Honeydew.Processes 8 | 9 | def start_link([queue, opts]) do 10 | DynamicSupervisor.start_link(__MODULE__, [queue, opts], name: Processes.process(queue, __MODULE__)) 11 | end 12 | 13 | def init(extra_args) do 14 | DynamicSupervisor.init(strategy: :one_for_one, extra_arguments: extra_args) 15 | end 16 | 17 | def start_worker_group(queue, queue_pid) do 18 | queue 19 | |> Processes.process(__MODULE__) 20 | |> DynamicSupervisor.start_child({WorkersPerQueueSupervisor, queue_pid}) 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/honeydew/worker_root_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.WorkerRootSupervisor do 2 | @moduledoc false 3 | 4 | use Supervisor, restart: :transient 5 | alias Honeydew.WorkerGroupSupervisor 6 | alias Honeydew.WorkerStarter 7 | alias Honeydew.NodeMonitorSupervisor 8 | 9 | @type name :: Honeydew.queue_name() 10 | 11 | @spec child_spec([name | any()]) :: Supervisor.child_spec() 12 | def child_spec([name | _] = opts) do 13 | opts 14 | |> super 15 | |> Map.put(:id, name) 16 | end 17 | 18 | def start_link([queue, opts]) do 19 | Supervisor.start_link(__MODULE__, [queue, opts], []) 20 | end 21 | 22 | # if the worker group supervisor shuts down due to too many groups restarting, 23 | # we also want the WorkerStarter to die so that it may restart the necessary 24 | # worker groups when the worker group supervisor comes back up 25 | @impl true 26 | def init([queue, opts]) do 27 | [ 28 | {WorkerGroupSupervisor, [queue, opts]}, 29 | {WorkerStarter, queue} 30 | ] 31 | |> add_node_supervisor(queue, opts) 32 | |> Supervisor.init(strategy: :rest_for_one) 33 | end 34 | 35 | defp add_node_supervisor(children, {:global, _} = queue, %{nodes: nodes}) do 36 | children ++ [{NodeMonitorSupervisor, [queue, nodes]}] 37 | end 38 | defp add_node_supervisor(children, _, _), do: children 39 | end 40 | -------------------------------------------------------------------------------- /lib/honeydew/worker_starter.ex: -------------------------------------------------------------------------------- 1 | # when a queue comes online (or its node connects), it sends a message to this process to start workers. 2 | defmodule Honeydew.WorkerStarter do 3 | @moduledoc false 4 | 5 | use GenServer 6 | 7 | alias Honeydew.WorkerGroupSupervisor 8 | alias Honeydew.Processes 9 | 10 | require Logger 11 | 12 | # called by a queue to tell the workerstarter to start workers 13 | def queue_available(queue, node) do 14 | GenServer.cast({Processes.process(queue, __MODULE__), node}, {:queue_available, self()}) 15 | end 16 | 17 | def start_link(queue) do 18 | GenServer.start_link(__MODULE__, queue, name: Processes.process(queue, __MODULE__)) 19 | end 20 | 21 | def init(queue) do 22 | queue 23 | |> Processes.get_queues() 24 | |> Enum.each(&WorkerGroupSupervisor.start_worker_group(queue, &1)) 25 | 26 | {:ok, queue} 27 | end 28 | 29 | def handle_cast({:queue_available, queue_pid}, queue) do 30 | Logger.info "[Honeydew] Queue process #{inspect queue_pid} from #{inspect queue} on node #{node(queue_pid)} became available, starting workers ..." 31 | 32 | {:ok, _} = WorkerGroupSupervisor.start_worker_group(queue, queue_pid) 33 | 34 | {:noreply, queue} 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/honeydew/worker_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.WorkerSupervisor do 2 | @moduledoc false 3 | 4 | use DynamicSupervisor, restart: :transient 5 | alias Honeydew.Worker 6 | 7 | def start_link([_queue, %{num: num}, _queue_pid] = opts) do 8 | {:ok, supervisor} = DynamicSupervisor.start_link(__MODULE__, [num], []) 9 | 10 | Enum.each(1..num, fn _ -> 11 | start_worker([supervisor | opts]) 12 | end) 13 | 14 | {:ok, supervisor} 15 | end 16 | 17 | @impl true 18 | def init([num]) do 19 | DynamicSupervisor.init(strategy: :one_for_one, max_restarts: num) 20 | end 21 | 22 | def start_worker([supervisor | _rest] = opts) do 23 | spec = Worker.child_spec(opts) 24 | {:ok, _} = DynamicSupervisor.start_child(supervisor, spec) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/honeydew/workers.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Workers do 2 | @moduledoc false 3 | 4 | use Supervisor 5 | 6 | alias Honeydew.WorkerRootSupervisor 7 | alias Honeydew.Processes 8 | 9 | @type name :: Honeydew.queue_name() 10 | @type mod_or_mod_args :: Honeydew.mod_or_mod_args() 11 | @type worker_opts :: Honeydew.worker_opts() 12 | 13 | @spec workers() :: [name] 14 | def workers do 15 | __MODULE__ 16 | |> Supervisor.which_children 17 | |> Enum.map(fn {queue, _, _, _} -> queue end) 18 | |> Enum.sort 19 | end 20 | 21 | @spec stop_workers(name) :: :ok | {:error, :not_running} 22 | def stop_workers(name) do 23 | with :ok <- Supervisor.terminate_child(__MODULE__, name) do 24 | Supervisor.delete_child(__MODULE__, name) 25 | end 26 | end 27 | 28 | @spec start_workers(name, mod_or_mod_args, worker_opts) :: :ok 29 | def start_workers(name, module_and_args, opts \\ []) do 30 | {module, args} = 31 | case module_and_args do 32 | module when is_atom(module) -> {module, []} 33 | {module, args} -> {module, args} 34 | end 35 | 36 | opts = %{ 37 | ma: {module, args}, 38 | num: opts[:num] || 10, 39 | init_retry_secs: opts[:init_retry_secs] || 5, 40 | shutdown: opts[:shutdown] || 10_000, 41 | nodes: opts[:nodes] || [] 42 | } 43 | 44 | unless Code.ensure_loaded?(module) do 45 | raise ArgumentError, invalid_module_error(module) 46 | end 47 | 48 | Processes.start_process_group_scope(name) 49 | 50 | with {:ok, _} <- Supervisor.start_child(__MODULE__, {WorkerRootSupervisor, [name, opts]}) do 51 | :ok 52 | end 53 | end 54 | 55 | def start_link(args) do 56 | Supervisor.start_link(__MODULE__, args, name: __MODULE__) 57 | end 58 | 59 | @impl true 60 | def init(_args) do 61 | Supervisor.init([], strategy: :one_for_one) 62 | end 63 | 64 | def invalid_module_error(module) do 65 | "unable to find module #{inspect module} for workers" 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/honeydew/workers_per_queue_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.WorkersPerQueueSupervisor do 2 | @moduledoc false 3 | 4 | use Supervisor, restart: :transient 5 | alias Honeydew.WorkerSupervisor 6 | alias Honeydew.QueueMonitor 7 | 8 | def start_link(queue, opts, queue_pid) do 9 | Supervisor.start_link(__MODULE__, [queue, opts, queue_pid], []) 10 | end 11 | 12 | @impl true 13 | def init([queue, opts, queue_pid]) do 14 | [ 15 | {WorkerSupervisor, [queue, opts, queue_pid]}, 16 | {QueueMonitor, [self(), queue_pid]} 17 | ] 18 | |> Supervisor.init(strategy: :rest_for_one) 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Mixfile do 2 | use Mix.Project 3 | 4 | @version "1.5.0" 5 | 6 | def project do 7 | [app: :honeydew, 8 | version: @version, 9 | elixir: "~> 1.12.0", 10 | start_permanent: Mix.env() == :prod, 11 | docs: docs(), 12 | deps: deps(), 13 | package: package(), 14 | elixirc_paths: elixirc_paths(Mix.env), 15 | description: "Pluggable local/clusterable job queue focused on safety.", 16 | dialyzer: [ 17 | plt_add_apps: [:mnesia, :ex_unit], 18 | flags: [ 19 | :unmatched_returns, 20 | :error_handling, 21 | :race_conditions, 22 | :no_opaque 23 | ] 24 | ] 25 | ] 26 | end 27 | 28 | defp elixirc_paths(:test), do: ["lib", "test/support"] 29 | defp elixirc_paths(_), do: ["lib"] 30 | 31 | # Configuration for the OTP application 32 | # 33 | # Type `mix help compile.app` for more information 34 | def application do 35 | [extra_applications: [:logger], 36 | included_applications: [:mnesia], 37 | mod: {Honeydew.Application, []}] 38 | end 39 | 40 | defp deps do 41 | [ 42 | {:ecto, "~> 3.0", optional: true, only: [:dev, :prod]}, 43 | {:ex_doc, ">= 0.0.0", only: :dev}, 44 | {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, 45 | # {:eflame, git: "git@github.com:slfritchie/eflame", only: :dev}, 46 | ] 47 | end 48 | 49 | defp package do 50 | [maintainers: ["Michael Shapiro"], 51 | licenses: ["MIT"], 52 | links: %{"GitHub": "https://github.com/koudelka/honeydew"}] 53 | end 54 | 55 | defp docs do 56 | [extras: ["README.md"], 57 | source_url: "https://github.com/koudelka/honeydew", 58 | source_ref: @version, 59 | assets: "assets", 60 | main: "readme"] 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, 3 | "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, 4 | "earmark": {:hex, :earmark, "1.2.6", "b6da42b3831458d3ecc57314dff3051b080b9b2be88c2e5aa41cd642a5b044ed", [:mix], [], "hexpm", "b42a23e9bd92d65d16db2f75553982e58519054095356a418bb8320bbacb58b1"}, 5 | "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, 6 | "ecto": {:hex, :ecto, "3.6.1", "7bb317e3fd0179ad725069fd0fe8a28ebe48fec6282e964ea502e4deccb0bd0f", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cbb3294a990447b19f0725488a749f8cf806374e0d9d0dffc45d61e7aeaf6553"}, 7 | "eflame": {:git, "git@github.com:slfritchie/eflame", "a08518142126f5fc541a3a3c4a04c27f24448bae", []}, 8 | "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, 9 | "ex_doc": {:hex, :ex_doc, "0.24.2", "e4c26603830c1a2286dae45f4412a4d1980e1e89dc779fcd0181ed1d5a05c8d9", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "e134e1d9e821b8d9e4244687fb2ace58d479b67b282de5158333b0d57c6fb7da"}, 10 | "hamcrest": {:hex, :basho_hamcrest, "0.4.1", "fb7b2c92d252a1e9db936750b86089addaebeb8f87967fb4bbdda61e8863338e", [:make, :mix, :rebar3], [], "hexpm"}, 11 | "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, 12 | "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, 13 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, 14 | "meck": {:hex, :meck, "0.8.4", "59ca1cd971372aa223138efcf9b29475bde299e1953046a0c727184790ab1520", [:make, :rebar], [], "hexpm"}, 15 | "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, 16 | "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm"}, 17 | "protobuffs": {:hex, :protobuffs, "0.8.4", "d38ca5f7380d8477c274680273372011890f8d0037c0d7e7db5c0207b89a4e0b", [:make, :rebar], [{:meck, "~> 0.8.4", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, 18 | "riak_pb": {:hex, :riak_pb, "2.3.2", "48ffbf66dbb3f136ab9a7134bac4e496754baa5ef58c4f50a61326736d996390", [:make, :mix, :rebar3], [{:hamcrest, "~> 0.4.1", [hex: :basho_hamcrest, repo: "hexpm", optional: false]}], "hexpm"}, 19 | "riakc": {:hex, :riakc, "2.5.3", "6132d9e687a0dfd314b2b24c4594302ca8b55568a5d733c491d8fb6cd4004763", [:make, :mix, :rebar3], [{:riak_pb, "~> 2.3", [hex: :riak_pb, repo: "hexpm", optional: false]}], "hexpm"}, 20 | "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, 21 | } 22 | -------------------------------------------------------------------------------- /test/hammer/hammer.exs: -------------------------------------------------------------------------------- 1 | defmodule Worker do 2 | def thing do 3 | :hi 4 | end 5 | 6 | # def send(to, msg) do 7 | # send to, msg 8 | # end 9 | end 10 | 11 | defmodule Honeydew.Hammer do 12 | 13 | @num_jobs 1_000_00 14 | 15 | def run(func) do 16 | # :ok = Honeydew.start_queue(:queue) 17 | :ok = Honeydew.start_queue(:queue, queue: {Honeydew.Queue.Mnesia, [ram_copies: [node()]]}) 18 | :ok = Honeydew.start_workers(:queue, Worker, num: 10) 19 | 20 | {microsecs, :ok} = :timer.tc(__MODULE__, func, []) 21 | secs = microsecs/:math.pow(10, 6) 22 | IO.puts("processed #{@num_jobs} in #{secs}s -> #{@num_jobs/secs} per sec") 23 | end 24 | 25 | def reply do 26 | Enum.map(1..@num_jobs, fn _ -> 27 | Task.async(fn -> 28 | {:ok, :hi} = Honeydew.async({:thing, []}, :queue, reply: true) |> Honeydew.yield(20_000) 29 | end) 30 | end) 31 | |> Enum.each(&Task.await(&1, :infinity)) 32 | end 33 | 34 | def no_reply do 35 | Enum.each(1..@num_jobs, fn _ -> 36 | Task.async(fn -> 37 | Honeydew.async(:thing, :queue) 38 | end) 39 | end) 40 | 41 | me = self() 42 | Honeydew.async(fn -> send me, :done end, :queue) 43 | 44 | receive do 45 | :done -> 46 | :ok 47 | end 48 | end 49 | end 50 | 51 | Honeydew.Hammer.run(:no_reply) 52 | -------------------------------------------------------------------------------- /test/hammer/hammer.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mix compile.protocols 3 | elixir -pa _build/$MIX_ENV/consolidated --erl "+P 134217727 +C multi_time_warp" -S mix run `dirname $0`/hammer.exs 4 | -------------------------------------------------------------------------------- /test/honeydew/dispatcher/lru_node_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher.LRUNodeTest do 2 | use ExUnit.Case, async: true 3 | alias Honeydew.Dispatcher.LRUNode 4 | 5 | setup do 6 | {:ok, state} = LRUNode.init 7 | 8 | state = LRUNode.check_in({"a", :a}, state) 9 | state = LRUNode.check_in({"a1", :a}, state) 10 | state = LRUNode.check_in({"b", :b}, state) 11 | state = LRUNode.check_in({"b1", :b}, state) 12 | state = LRUNode.check_in({"c", :c}, state) 13 | state = LRUNode.check_in({"c1", :c}, state) 14 | state = LRUNode.check_in({"d", :d}, state) 15 | 16 | [state: state] 17 | end 18 | 19 | test "cycle through least recently used node", %{state: state} do 20 | assert LRUNode.available?(state) 21 | 22 | {worker, state} = LRUNode.check_out(nil, state) 23 | assert worker == {"a", :a} 24 | {worker, state} = LRUNode.check_out(nil, state) 25 | assert worker == {"b", :b} 26 | {worker, state} = LRUNode.check_out(nil, state) 27 | assert worker == {"c", :c} 28 | {worker, state} = LRUNode.check_out(nil, state) 29 | assert worker == {"d", :d} 30 | 31 | {worker, state} = LRUNode.check_out(nil, state) 32 | assert worker == {"a1", :a} 33 | {worker, state} = LRUNode.check_out(nil, state) 34 | assert worker == {"b1", :b} 35 | {worker, _state} = LRUNode.check_out(nil, state) 36 | assert worker == {"c1", :c} 37 | end 38 | 39 | test "check_out/2 gives nil when none available", %{state: state} do 40 | {_worker, state} = LRUNode.check_out(nil, state) 41 | {_worker, state} = LRUNode.check_out(nil, state) 42 | {_worker, state} = LRUNode.check_out(nil, state) 43 | {_worker, state} = LRUNode.check_out(nil, state) 44 | {_worker, state} = LRUNode.check_out(nil, state) 45 | {_worker, state} = LRUNode.check_out(nil, state) 46 | {_worker, state} = LRUNode.check_out(nil, state) 47 | {worker, state} = LRUNode.check_out(nil, state) 48 | 49 | assert worker == nil 50 | refute LRUNode.available?(state) 51 | end 52 | 53 | test "removes workers", %{state: state} do 54 | state = LRUNode.remove({"b", :b}, state) 55 | state = LRUNode.remove({"d", :d}, state) 56 | 57 | {worker, state} = LRUNode.check_out(nil, state) 58 | assert worker == {"a", :a} 59 | {worker, state} = LRUNode.check_out(nil, state) 60 | assert worker == {"b1", :b} 61 | {worker, state} = LRUNode.check_out(nil, state) 62 | assert worker == {"c", :c} 63 | 64 | {worker, state} = LRUNode.check_out(nil, state) 65 | assert worker == {"a1", :a} 66 | {worker, state} = LRUNode.check_out(nil, state) 67 | assert worker == {"c1", :c} 68 | 69 | refute LRUNode.available?(state) 70 | end 71 | 72 | test "known?/2", %{state: state} do 73 | assert LRUNode.known?({"a", :a}, state) 74 | refute LRUNode.known?({"z", :z}, state) 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /test/honeydew/dispatcher/lru_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher.LRUTest do 2 | use ExUnit.Case, async: true 3 | alias Honeydew.Dispatcher.LRU 4 | 5 | setup do 6 | {:ok, state} = LRU.init 7 | 8 | state = LRU.check_in("a", state) 9 | state = LRU.check_in("b", state) 10 | state = LRU.check_in("c", state) 11 | 12 | [state: state] 13 | end 14 | 15 | test "enqueue/dequeue least recently used", %{state: state} do 16 | assert LRU.available?(state) 17 | 18 | {worker, state} = LRU.check_out(nil, state) 19 | assert worker == "a" 20 | {worker, state} = LRU.check_out(nil, state) 21 | assert worker == "b" 22 | {worker, _state} = LRU.check_out(nil, state) 23 | assert worker == "c" 24 | end 25 | 26 | test "check_out/2 gives nil when none available", %{state: state} do 27 | {_worker, state} = LRU.check_out(nil, state) 28 | {_worker, state} = LRU.check_out(nil, state) 29 | {_worker, state} = LRU.check_out(nil, state) 30 | 31 | {worker, state} = LRU.check_out(nil, state) 32 | assert worker == nil 33 | 34 | refute LRU.available?(state) 35 | end 36 | 37 | test "removes workers", %{state: state} do 38 | state = LRU.remove("b", state) 39 | 40 | {worker, state} = LRU.check_out(nil, state) 41 | assert worker == "a" 42 | {worker, state} = LRU.check_out(nil, state) 43 | assert worker == "c" 44 | 45 | {worker, _state} = LRU.check_out(nil, state) 46 | assert worker == nil 47 | end 48 | 49 | test "known?/2", %{state: state} do 50 | assert LRU.known?("a", state) 51 | refute LRU.known?("z", state) 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/honeydew/dispatcher/mru_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Dispatcher.MRUTest do 2 | use ExUnit.Case, async: true 3 | alias Honeydew.Dispatcher.MRU 4 | 5 | setup do 6 | {:ok, state} = MRU.init 7 | 8 | state = MRU.check_in("a", state) 9 | state = MRU.check_in("b", state) 10 | state = MRU.check_in("c", state) 11 | 12 | [state: state] 13 | end 14 | 15 | test "enqueue/dequeue most recently used", %{state: state} do 16 | assert MRU.available?(state) 17 | 18 | {worker, state} = MRU.check_out(nil, state) 19 | assert worker == "c" 20 | {worker, state} = MRU.check_out(nil, state) 21 | assert worker == "b" 22 | {worker, _state} = MRU.check_out(nil, state) 23 | assert worker == "a" 24 | end 25 | 26 | test "check_out/2 gives nil when none available", %{state: state} do 27 | {_worker, state} = MRU.check_out(nil, state) 28 | {_worker, state} = MRU.check_out(nil, state) 29 | {_worker, state} = MRU.check_out(nil, state) 30 | 31 | {worker, state} = MRU.check_out(nil, state) 32 | assert worker == nil 33 | 34 | refute MRU.available?(state) 35 | end 36 | 37 | test "removes workers", %{state: state} do 38 | state = MRU.remove("b", state) 39 | 40 | {worker, state} = MRU.check_out(nil, state) 41 | assert worker == "c" 42 | {worker, state} = MRU.check_out(nil, state) 43 | assert worker == "a" 44 | 45 | {worker, _state} = MRU.check_out(nil, state) 46 | assert worker == nil 47 | end 48 | 49 | test "known?/2", %{state: state} do 50 | assert MRU.known?("a", state) 51 | refute MRU.known?("z", state) 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/honeydew/ecto_poll_queue_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.EctoPollQueueTest do 2 | use ExUnit.Case, async: true 3 | alias Honeydew.EctoPollQueue 4 | 5 | defmodule PseudoRepo do 6 | def __adapter__ do 7 | Ecto.Adapters.Postgres 8 | end 9 | end 10 | 11 | defmodule UnsupportedRepo do 12 | def __adapter__ do 13 | Ecto.Adapters.ButtDB 14 | end 15 | end 16 | 17 | defmodule PseudoSchema do 18 | end 19 | 20 | describe "rewrite_opts!/1" do 21 | test "should raise when :database isn't supported" do 22 | assert_raise ArgumentError, ~r/repo's ecto adapter/, fn -> 23 | queue = :erlang.unique_integer 24 | EctoPollQueue.rewrite_opts([queue, EctoPollQueue, [repo: UnsupportedRepo]]) 25 | end 26 | end 27 | end 28 | 29 | describe "vaildate_args!/1" do 30 | test "shouldn't raise with valid args" do 31 | :ok = EctoPollQueue.validate_args!([repo: PseudoRepo, schema: PseudoSchema, poll_interval: 1, stale_timeout: 1]) 32 | end 33 | 34 | test "should raise when poll interval isn't an integer" do 35 | assert_raise ArgumentError, ~r/Poll interval must/, fn -> 36 | EctoPollQueue.validate_args!([repo: PseudoRepo, schema: PseudoSchema, poll_interval: 0.5]) 37 | end 38 | end 39 | 40 | test "should raise when stale timeout isn't an integer" do 41 | assert_raise ArgumentError, ~r/Stale timeout must/, fn -> 42 | EctoPollQueue.validate_args!([repo: PseudoRepo, schema: PseudoSchema, stale_timeout: 0.5]) 43 | end 44 | end 45 | 46 | test "should raise when :repo or :schema arguments aren't provided" do 47 | assert_raise ArgumentError, ~r/didn't provide a required argument/, fn -> 48 | EctoPollQueue.validate_args!([repo: PseudoRepo]) 49 | end 50 | 51 | assert_raise ArgumentError, ~r/didn't provide a required argument/, fn -> 52 | EctoPollQueue.validate_args!([schema: PseudoSchema]) 53 | end 54 | end 55 | 56 | test "should raise when :repo or :schema arguments aren't loaded modules" do 57 | assert_raise ArgumentError, ~r/module you provided/, fn -> 58 | EctoPollQueue.validate_args!([repo: :abc, schema: PseudoSchema]) 59 | end 60 | 61 | assert_raise ArgumentError, ~r/module you provided/, fn -> 62 | EctoPollQueue.validate_args!([repo: PseudoRepo, schema: :abc]) 63 | end 64 | end 65 | 66 | end 67 | 68 | end 69 | -------------------------------------------------------------------------------- /test/honeydew/failure_mode/abandon_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.AbandonTest do 2 | use ExUnit.Case, async: true 3 | 4 | @moduletag :capture_log 5 | 6 | setup do 7 | queue = :erlang.unique_integer 8 | :ok = Honeydew.start_queue(queue, failure_mode: Honeydew.FailureMode.Abandon) 9 | :ok = Honeydew.start_workers(queue, Stateless) 10 | 11 | [queue: queue] 12 | end 13 | 14 | test "validate_args!/1" do 15 | import Honeydew.FailureMode.Abandon, only: [validate_args!: 1] 16 | 17 | assert :ok = validate_args!([]) 18 | 19 | assert_raise ArgumentError, fn -> 20 | validate_args!(:abc) 21 | end 22 | end 23 | 24 | test "should remove job from the queue", %{queue: queue} do 25 | {:crash, [self()]} |> Honeydew.async(queue) 26 | assert_receive :job_ran 27 | 28 | Process.sleep(100) # let the failure mode do its thing 29 | 30 | assert Honeydew.status(queue) |> get_in([:queue, :count]) == 0 31 | refute_receive :job_ran 32 | end 33 | 34 | test "should inform the awaiting process of the exception", %{queue: queue} do 35 | {:error, reason} = 36 | {:crash, [self()]} 37 | |> Honeydew.async(queue, reply: true) 38 | |> Honeydew.yield 39 | 40 | assert {%RuntimeError{message: "ignore this crash"}, stacktrace} = reason 41 | assert is_list(stacktrace) 42 | end 43 | 44 | test "should inform the awaiting process of an uncaught throw", %{queue: queue} do 45 | {:error, reason} = 46 | fn -> throw "intentional crash" end 47 | |> Honeydew.async(queue, reply: true) 48 | |> Honeydew.yield 49 | 50 | assert {"intentional crash", stacktrace} = reason 51 | assert is_list(stacktrace) 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/honeydew/failure_mode/exponential_retry_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.ExponentialRetryTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Honeydew.Queue.Mnesia 5 | 6 | @moduletag :capture_log 7 | 8 | setup do 9 | queue = :erlang.unique_integer 10 | failure_queue = "#{queue}_failed" 11 | 12 | :ok = Honeydew.start_queue(queue, queue: {Mnesia, ram_copies: [node()]}, 13 | failure_mode: {Honeydew.FailureMode.ExponentialRetry, 14 | times: 3, 15 | finally: {Honeydew.FailureMode.Move, queue: failure_queue}}) 16 | :ok = Honeydew.start_queue(failure_queue, queue: {Mnesia, [ram_copies: [node()]]}) 17 | :ok = Honeydew.start_workers(queue, Stateless) 18 | 19 | [queue: queue, failure_queue: failure_queue] 20 | end 21 | 22 | test "validate_args!/1" do 23 | import Honeydew.FailureMode.ExponentialRetry, only: [validate_args!: 1] 24 | 25 | assert :ok = validate_args!([times: 2]) 26 | assert :ok = validate_args!([times: 2, finally: {Honeydew.FailureMode.Move, [queue: :abc]}]) 27 | 28 | assert_raise ArgumentError, fn -> 29 | validate_args!([base: -1]) 30 | end 31 | 32 | assert_raise ArgumentError, fn -> 33 | validate_args!(:abc) 34 | end 35 | 36 | assert_raise ArgumentError, fn -> 37 | validate_args!([fun: fn _job, _reason -> :halt end]) 38 | end 39 | 40 | assert_raise ArgumentError, fn -> 41 | validate_args!([times: -1]) 42 | end 43 | 44 | assert_raise ArgumentError, fn -> 45 | validate_args!([times: 2, finally: {Honeydew.FailureMode.Move, [bad: :args]}]) 46 | end 47 | 48 | assert_raise ArgumentError, fn -> 49 | validate_args!([times: 2, finally: {"bad", []}]) 50 | end 51 | end 52 | 53 | test "should retry the job with exponentially increasing delays", %{queue: queue, failure_queue: failure_queue} do 54 | {:crash, [self()]} |> Honeydew.async(queue) 55 | 56 | delays = 57 | Enum.map(0..3, fn _ -> 58 | receive do 59 | :job_ran -> 60 | DateTime.utc_now() 61 | end 62 | end) 63 | |> Enum.chunk_every(2, 1, :discard) 64 | |> Enum.map(fn [a, b] -> DateTime.diff(b, a) end) 65 | 66 | # 2^0 - 1 -> 0 sec delay 67 | # 2^1 - 1 -> 1 sec delay 68 | # 2^2 - 1 -> 3 sec delay 69 | assert_in_delta Enum.at(delays, 0), 0, 1 70 | assert_in_delta Enum.at(delays, 1), 1, 1 71 | assert_in_delta Enum.at(delays, 2), 3, 1 72 | 73 | assert Honeydew.status(queue) |> get_in([:queue, :count]) == 0 74 | refute_receive :job_ran 75 | 76 | assert Honeydew.status(failure_queue) |> get_in([:queue, :count]) == 1 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /test/honeydew/failure_mode/move_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.MoveTest do 2 | use ExUnit.Case, async: true 3 | 4 | @moduletag :capture_log 5 | 6 | setup do 7 | queue = :erlang.unique_integer 8 | failure_queue = "#{queue}_failed" 9 | 10 | :ok = Honeydew.start_queue(queue, failure_mode: {Honeydew.FailureMode.Move, queue: failure_queue}) 11 | :ok = Honeydew.start_queue(failure_queue) 12 | :ok = Honeydew.start_workers(queue, Stateless) 13 | 14 | [queue: queue, failure_queue: failure_queue] 15 | end 16 | 17 | test "validate_args!/1" do 18 | import Honeydew.FailureMode.Move, only: [validate_args!: 1] 19 | 20 | assert :ok = validate_args!(queue: :abc) 21 | assert :ok = validate_args!(queue: {:global, :abc}) 22 | 23 | assert_raise ArgumentError, fn -> 24 | validate_args!(:abc) 25 | end 26 | end 27 | 28 | test "should move the job on the new queue", %{queue: queue, failure_queue: failure_queue} do 29 | {:crash, [self()]} |> Honeydew.async(queue) 30 | assert_receive :job_ran 31 | 32 | Process.sleep(500) # let the failure mode do its thing 33 | 34 | assert Honeydew.status(queue) |> get_in([:queue, :count]) == 0 35 | refute_receive :job_ran 36 | 37 | assert Honeydew.status(failure_queue) |> get_in([:queue, :count]) == 1 38 | end 39 | 40 | test "should inform the awaiting process of the exception", %{queue: queue, failure_queue: failure_queue} do 41 | job = {:crash, [self()]} |> Honeydew.async(queue, reply: true) 42 | 43 | assert {:moved, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 44 | 45 | :ok = Honeydew.start_workers(failure_queue, Stateless) 46 | 47 | # job ran in the failure queue 48 | assert {:error, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 49 | end 50 | 51 | test "should inform the awaiting process of the uncaught throw", %{queue: queue, failure_queue: failure_queue} do 52 | job = fn -> throw "intentional crash" end |> Honeydew.async(queue, reply: true) 53 | 54 | assert {:moved, {"intentional crash", stacktrace}} = Honeydew.yield(job) 55 | assert is_list(stacktrace) 56 | 57 | :ok = Honeydew.start_workers(failure_queue, Stateless) 58 | 59 | # job ran in the failure queue 60 | assert {:error, {"intentional crash", _stacktrace}} = Honeydew.yield(job) 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /test/honeydew/failure_mode/retry_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.FailureMode.RetryTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Honeydew.Queue.Mnesia 5 | 6 | @moduletag :capture_log 7 | 8 | setup do 9 | queue = :erlang.unique_integer 10 | failure_queue = "#{queue}_failed" 11 | 12 | :ok = Honeydew.start_queue(queue, queue: {Mnesia, ram_copies: [node()]}, 13 | failure_mode: {Honeydew.FailureMode.Retry, 14 | times: 3, 15 | finally: {Honeydew.FailureMode.Move, queue: failure_queue}}) 16 | :ok = Honeydew.start_queue(failure_queue, queue: {Mnesia, [ram_copies: [node()]]}) 17 | :ok = Honeydew.start_workers(queue, Stateless) 18 | 19 | [queue: queue, failure_queue: failure_queue] 20 | end 21 | 22 | test "validate_args!/1" do 23 | import Honeydew.FailureMode.Retry, only: [validate_args!: 1] 24 | 25 | assert :ok = validate_args!([fun: fn _job, _reason, _args -> :halt end]) 26 | assert :ok = validate_args!([times: 2]) 27 | assert :ok = validate_args!([times: 2, finally: {Honeydew.FailureMode.Move, [queue: :abc]}]) 28 | 29 | assert_raise ArgumentError, fn -> 30 | validate_args!(:abc) 31 | end 32 | 33 | assert_raise ArgumentError, fn -> 34 | validate_args!([fun: fn _job, _reason -> :halt end]) 35 | end 36 | 37 | assert_raise ArgumentError, fn -> 38 | validate_args!([times: -1]) 39 | end 40 | 41 | assert_raise ArgumentError, fn -> 42 | validate_args!([times: 2, finally: {Honeydew.FailureMode.Move, [bad: :args]}]) 43 | end 44 | 45 | assert_raise ArgumentError, fn -> 46 | validate_args!([times: 2, finally: {"bad", []}]) 47 | end 48 | end 49 | 50 | test "should retry the job", %{queue: queue, failure_queue: failure_queue} do 51 | {:crash, [self()]} |> Honeydew.async(queue) 52 | assert_receive :job_ran 53 | assert_receive :job_ran 54 | assert_receive :job_ran 55 | assert_receive :job_ran 56 | 57 | Process.sleep(500) # let the Move failure mode do its thing 58 | 59 | assert Honeydew.status(queue) |> get_in([:queue, :count]) == 0 60 | refute_receive :job_ran 61 | 62 | assert Honeydew.status(failure_queue) |> get_in([:queue, :count]) == 1 63 | end 64 | 65 | test "should inform the awaiting process of the exception", %{queue: queue, failure_queue: failure_queue} do 66 | job = {:crash, [self()]} |> Honeydew.async(queue, reply: true) 67 | 68 | assert {:retrying, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 69 | assert {:retrying, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 70 | assert {:retrying, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 71 | assert {:moved, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 72 | 73 | :ok = Honeydew.start_workers(failure_queue, Stateless) 74 | 75 | # job ran in the failure queue 76 | assert {:error, {%RuntimeError{message: "ignore this crash"}, _stacktrace}} = Honeydew.yield(job) 77 | end 78 | 79 | test "should inform the awaiting process of an uncaught throw", %{queue: queue, failure_queue: failure_queue} do 80 | job = fn -> throw "intentional crash" end |> Honeydew.async(queue, reply: true) 81 | 82 | assert {:retrying, {"intentional crash", _stacktrace}} = Honeydew.yield(job) 83 | assert {:retrying, {"intentional crash", _stacktrace}} = Honeydew.yield(job) 84 | assert {:retrying, {"intentional crash", _stacktrace}} = Honeydew.yield(job) 85 | assert {:moved, {"intentional crash", _stacktrace}} = Honeydew.yield(job) 86 | 87 | :ok = Honeydew.start_workers(failure_queue, Stateless) 88 | 89 | # job ran in the failure queue 90 | assert {:error, {"intentional crash", _stacktrace}} = Honeydew.yield(job) 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /test/honeydew/global_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.GlobalTest do 2 | use ExUnit.Case, async: true 3 | alias Honeydew.Support.ClusterSetups 4 | 5 | setup [:setup_queue_name] 6 | 7 | describe "simple global queue" do 8 | setup %{queue: queue} do 9 | {:ok, %{node: _queue_node}} = ClusterSetups.start_queue_node(queue) 10 | {:ok, %{node: _worker_node}} = ClusterSetups.start_worker_node(queue) 11 | 12 | :ok 13 | end 14 | 15 | test "hammer async/3", %{queue: queue} do 16 | Enum.each(0..10_000, fn i -> 17 | {:send_msg, [self(), i]} |> Honeydew.async(queue) 18 | assert_receive ^i 19 | end) 20 | end 21 | 22 | test "yield/2", %{queue: queue} do 23 | first_job = {:return, [:hi]} |> Honeydew.async(queue, reply: true) 24 | second_job = {:return, [:there]} |> Honeydew.async(queue, reply: true) 25 | 26 | assert {:ok, :hi} = Honeydew.yield(first_job) 27 | assert {:ok, :there} = Honeydew.yield(second_job) 28 | end 29 | end 30 | 31 | defp setup_queue_name(_), do: {:ok, [queue: generate_queue_name()]} 32 | 33 | defp generate_queue_name do 34 | {:global, :crypto.strong_rand_bytes(10) |> Base.encode16} 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/honeydew/logger/metadata_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Logger.MetadataTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Honeydew.Crash 5 | alias Honeydew.Logger.Metadata 6 | 7 | test "build_crash_reason/1 with an exception crash" do 8 | exception = RuntimeError.exception("foo") 9 | stacktrace = [] 10 | crash = Crash.new(:exception, exception, stacktrace) 11 | 12 | assert {^exception, ^stacktrace} = Metadata.build_crash_reason(crash) 13 | end 14 | 15 | test "build_crash_reason/1 with an uncaught throw crash" do 16 | thrown = :baseball 17 | stacktrace = [] 18 | crash = Crash.new(:throw, thrown, stacktrace) 19 | 20 | assert {{:nocatch, ^thrown}, ^stacktrace} = Metadata.build_crash_reason(crash) 21 | end 22 | 23 | test "build_crash_reason/1 with a bad return value" do 24 | value = :boom 25 | crash = Crash.new(:bad_return_value, value) 26 | 27 | assert {{:bad_return_value, ^value}, []} = Metadata.build_crash_reason(crash) 28 | end 29 | 30 | test "build_crash_reason/1 with an unexpected exit" do 31 | value = :boom 32 | crash = Crash.new(:exit, value) 33 | 34 | assert {{:exit, ^value}, []} = Metadata.build_crash_reason(crash) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/honeydew/logger_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.LoggerTest do 2 | use ExUnit.Case, async: false 3 | 4 | import Honeydew.CrashLoggerHelpers 5 | 6 | alias Honeydew.Crash 7 | alias Honeydew.Job 8 | alias Honeydew.Logger, as: HoneydewLogger 9 | alias Honeydew.Logger.Metadata 10 | 11 | setup [:setup_echoing_error_logger] 12 | 13 | @moduletag :capture_log 14 | 15 | test "worker_init_crashed/1 with an exception crash" do 16 | module = __MODULE__ 17 | error = RuntimeError.exception("foo") 18 | stacktrace = [] 19 | crash = Crash.new(:exception, error, stacktrace) 20 | 21 | :ok = HoneydewLogger.worker_init_crashed(module, crash) 22 | 23 | assert_receive {:honeydew_crash_log, event} 24 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 25 | assert msg =~ ~r/#{inspect(module)}.init\/1 must return \{:ok, .* but raised #{inspect(error)}/ 26 | assert Keyword.fetch!(metadata, :honeydew_crash_reason) == Metadata.build_crash_reason(crash) 27 | end 28 | 29 | test "worker_init_crashed/1 with an uncaught throw crash" do 30 | module = __MODULE__ 31 | thrown = :grenade 32 | stacktrace = [] 33 | crash = Crash.new(:throw, thrown, stacktrace) 34 | 35 | :ok = HoneydewLogger.worker_init_crashed(module, crash) 36 | 37 | assert_receive {:honeydew_crash_log, event} 38 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 39 | assert msg =~ ~r/#{inspect(module)}.init\/1 must return \{:ok, .* but threw #{inspect(thrown)}/ 40 | assert Keyword.fetch!(metadata, :honeydew_crash_reason) == Metadata.build_crash_reason(crash) 41 | end 42 | 43 | test "worker_init_crashed/1 with a bad return value crash" do 44 | module = __MODULE__ 45 | value = "1 million dollars" 46 | crash = Crash.new(:bad_return_value, value) 47 | 48 | :ok = HoneydewLogger.worker_init_crashed(module, crash) 49 | 50 | assert_receive {:honeydew_crash_log, event} 51 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 52 | assert msg =~ ~r/#{inspect(module)}.init\/1 must return \{:ok, .*, got: #{inspect(value)}/ 53 | assert Keyword.fetch!(metadata, :honeydew_crash_reason) == Metadata.build_crash_reason(crash) 54 | end 55 | 56 | test "job_failed/1 with an exception crash" do 57 | job = %Job{} 58 | error = RuntimeError.exception("foo") 59 | stacktrace = [] 60 | crash = Crash.new(:exception, error, stacktrace) 61 | 62 | :ok = HoneydewLogger.job_failed(job, crash) 63 | 64 | assert_receive {:honeydew_crash_log, event} 65 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 66 | assert msg =~ ~r/job failed due to exception/i 67 | assert Keyword.fetch!(metadata, :honeydew_crash_reason) == Metadata.build_crash_reason(crash) 68 | assert %Job{} = Keyword.fetch!(metadata, :honeydew_job) 69 | end 70 | 71 | test "job_failed/1 with an uncaught throw crash" do 72 | job = %Job{} 73 | thrown = :grenade 74 | stacktrace = [] 75 | crash = Crash.new(:throw, thrown, stacktrace) 76 | 77 | :ok = HoneydewLogger.job_failed(job, crash) 78 | 79 | assert_receive {:honeydew_crash_log, event} 80 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 81 | assert msg =~ ~r/job failed due to uncaught throw/i 82 | assert Keyword.fetch!(metadata, :honeydew_crash_reason) == Metadata.build_crash_reason(crash) 83 | assert %Job{} = Keyword.fetch!(metadata, :honeydew_job) 84 | end 85 | 86 | test "job_failed/1 with an unexpecetd exit" do 87 | job = %Job{} 88 | exit_reason = :didnt_want_to_run_anymore 89 | crash = Crash.new(:exit, exit_reason) 90 | 91 | :ok = HoneydewLogger.job_failed(job, crash) 92 | 93 | assert_receive {:honeydew_crash_log, event} 94 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 95 | assert msg =~ ~r/unexpected exit/i 96 | assert Keyword.fetch!(metadata, :honeydew_crash_reason) == Metadata.build_crash_reason(crash) 97 | assert %Job{} = Keyword.fetch!(metadata, :honeydew_job) 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /test/honeydew/queue/ecto_poll_queue_integration_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.EctoPollQueueIntegrationTest do 2 | alias Mix.Shell.IO, as: Output 3 | use ExUnit.Case, async: false 4 | @moduletag timeout: 4 * 60 * 1_000 5 | @examples_root "./examples/ecto_poll_queue" 6 | 7 | # Postgres 8 | test "ecto poll queue external project test: Postgres" do 9 | announce_test("Postgres (unprefixed)") 10 | :ok = mix("deps.get", "postgres", prefixed: false) 11 | :ok = mix("test", "postgres", prefixed: false) 12 | end 13 | 14 | test "ecto poll queue external project test: Postgres (prefixed)" do 15 | announce_test("Postgres (prefixed)") 16 | :ok = mix("deps.get", "postgres", prefixed: true) 17 | :ok = mix("test", "postgres", prefixed: true) 18 | end 19 | 20 | # Cockroach 21 | test "ecto poll queue external project test: Cockroach" do 22 | announce_test("CockroachDB (unprefixed)") 23 | :ok = mix("deps.get", "cockroach", prefixed: false) 24 | :ok = mix("test", "cockroach", prefixed: false) 25 | end 26 | 27 | defp announce_test(message) do 28 | Output.info( 29 | "\n#{IO.ANSI.underline()}[ECTO POLL QUEUE INTEGRATION] #{message}#{IO.ANSI.reset()}" 30 | ) 31 | end 32 | 33 | defp mix(task, database, opts) when is_binary(task), do: mix([task], database, opts) 34 | 35 | defp mix(task, database, prefixed: prefixed_tables) do 36 | environment = [{"MIX_ENV", database}] |> add_prefixed_tables_env(prefixed_tables) 37 | 38 | {_, exit_code} = 39 | System.cmd("mix", task, cd: @examples_root, into: IO.stream(:stdio, 1), env: environment) 40 | 41 | if exit_code == 0 do 42 | :ok 43 | else 44 | {:error, exit_code} 45 | end 46 | end 47 | 48 | defp add_prefixed_tables_env(env, true), do: env ++ [{"prefixed_tables", "true"}] 49 | defp add_prefixed_tables_env(env, false), do: env 50 | end 51 | -------------------------------------------------------------------------------- /test/honeydew/queues_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.QueuesTest do 2 | use ExUnit.Case, async: false # restarts honeydew app 3 | import Helper 4 | alias Honeydew.Queues 5 | alias Honeydew.Queue.State 6 | alias Honeydew.Queue.Mnesia 7 | alias Honeydew.Dispatcher.{LRU, MRU} 8 | alias Honeydew.FailureMode.{Abandon, Retry} 9 | alias Honeydew.SuccessMode.Log 10 | 11 | setup :restart_honeydew 12 | 13 | test "stop_queue/2 removes child spec" do 14 | :ok = Honeydew.start_queue(:abc) 15 | assert [{:abc, _, _, _}] = Supervisor.which_children(Queues) 16 | 17 | :ok = Honeydew.stop_queue(:abc) 18 | assert Enum.empty? Supervisor.which_children(Queues) 19 | end 20 | 21 | describe "start_queue/2" do 22 | test "options" do 23 | nodes = [node()] 24 | 25 | options = [ 26 | queue: {Mnesia, [ram_copies: nodes]}, 27 | dispatcher: {MRU, []}, 28 | failure_mode: {Retry, [times: 5]}, 29 | success_mode: {Log, []}, 30 | suspended: true 31 | ] 32 | 33 | :ok = Honeydew.start_queue({:global, :abc}, options) 34 | assert [{{:global, :abc}, pid, _, _}] = Supervisor.which_children(Queues) 35 | 36 | assert %State{ 37 | dispatcher: {MRU, _dispatcher_state}, 38 | failure_mode: {Retry, [times: 5]}, 39 | module: Mnesia, 40 | queue: {:global, :abc}, 41 | success_mode: {Log, []}, 42 | suspended: true 43 | } = :sys.get_state(pid) 44 | end 45 | 46 | test "raises when failure/success mode args are invalid" do 47 | assert_raise ArgumentError, fn -> 48 | Honeydew.start_queue(:abc, failure_mode: {Abandon, [:abc]}) 49 | end 50 | 51 | assert_raise ArgumentError, fn -> 52 | Honeydew.start_queue(:abc, success_mode: {Log, [:abc]}) 53 | end 54 | end 55 | 56 | test "default options" do 57 | :ok = Honeydew.start_queue(:abc) 58 | assert [{:abc, pid, _, _}] = Supervisor.which_children(Queues) 59 | 60 | assert %State{ 61 | dispatcher: {LRU, _dispatcher_state}, 62 | failure_mode: {Abandon, []}, 63 | module: Mnesia, 64 | queue: :abc, 65 | success_mode: nil, 66 | suspended: false 67 | } = :sys.get_state(pid) 68 | end 69 | 70 | test "starting queue twice will result in error" do 71 | assert :ok = Honeydew.start_queue(:foo) 72 | assert {:error, _} = Honeydew.start_queue(:foo) 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /test/honeydew/worker_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.WorkerTest do 2 | use ExUnit.Case 3 | 4 | alias Honeydew.Processes 5 | 6 | import Honeydew.CrashLoggerHelpers 7 | 8 | defmodule WorkerWithBadInit do 9 | @behaviour Honeydew.Worker 10 | def init(:raise), do: raise "Boom" 11 | def init(:throw), do: throw :boom 12 | def init(:bad), do: :bad 13 | def init(:ok), do: {:ok, %{}} 14 | end 15 | 16 | setup [:setup_queue_name, :setup_queue, :setup_worker_pool] 17 | 18 | @moduletag :capture_log 19 | 20 | @tag :start_workers 21 | test "workers should die when their queue dies", %{queue: queue} do 22 | queue_pid = Processes.get_queue(queue) 23 | %{workers: workers} = Honeydew.status(queue) 24 | 25 | Process.exit(queue_pid, :kill) 26 | 27 | Process.sleep(100) 28 | 29 | workers 30 | |> Map.keys 31 | |> Enum.each(fn w -> assert not Process.alive?(w) end) 32 | end 33 | 34 | describe "logging and exception handling" do 35 | setup [:setup_echoing_error_logger] 36 | 37 | test "when init/1 callback raises an exception", %{queue: queue} do 38 | expected_error = %RuntimeError{message: "Boom"} 39 | Honeydew.start_workers(queue, {WorkerWithBadInit, :raise}, num: 1) 40 | 41 | assert_receive {:honeydew_crash_log, event} 42 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 43 | assert msg =~ ~r/#{inspect(WorkerWithBadInit)}.init\/1 must return \{:ok, .* but raised #{inspect(expected_error)}/ 44 | assert {^expected_error, stacktrace} = Keyword.fetch!(metadata, :honeydew_crash_reason) 45 | assert is_list(stacktrace) 46 | end 47 | 48 | test "when init/1 callback throws an atom", %{queue: queue} do 49 | Honeydew.start_workers(queue, {WorkerWithBadInit, :throw}, num: 1) 50 | 51 | assert_receive {:honeydew_crash_log, event} 52 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 53 | assert msg =~ ~r/#{inspect(WorkerWithBadInit)}.init\/1 must return \{:ok, .* but threw #{inspect(:boom)}/ 54 | assert {{:nocatch, :boom}, stacktrace} = Keyword.fetch!(metadata, :honeydew_crash_reason) 55 | assert is_list(stacktrace) 56 | end 57 | 58 | test "when init/1 callback returns a bad return value", %{queue: queue} do 59 | Honeydew.start_workers(queue, {WorkerWithBadInit, :bad}, num: 1) 60 | 61 | assert_receive {:honeydew_crash_log, event} 62 | assert {:warn, _, {Logger, msg, _timestamp, metadata}} = event 63 | assert msg =~ ~r/#{inspect(WorkerWithBadInit)}.init\/1 must return \{:ok, .*, got: #{inspect(:bad)}/ 64 | assert {{:bad_return_value, :bad}, []} = Keyword.fetch!(metadata, :honeydew_crash_reason) 65 | end 66 | 67 | test "when init/1 callback returns {:ok, state}", %{queue: queue} do 68 | Honeydew.start_workers(queue, {WorkerWithBadInit, :ok}, num: 1) 69 | 70 | refute_receive {:honeydew_crash_log, _event} 71 | end 72 | end 73 | 74 | defp setup_queue_name(%{queue: queue}), do: {:ok, [queue: queue]} 75 | defp setup_queue_name(_), do: {:ok, [queue: generate_queue_name()]} 76 | 77 | defp setup_queue(%{queue: queue}) do 78 | :ok = Honeydew.start_queue(queue) 79 | 80 | on_exit fn -> 81 | Honeydew.stop_queue(queue) 82 | end 83 | end 84 | 85 | defp setup_worker_pool(%{queue: queue, start_workers: true}) do 86 | :ok = Honeydew.start_workers(queue, Stateless, num: 10) 87 | 88 | on_exit fn -> 89 | Honeydew.stop_workers(queue) 90 | end 91 | end 92 | 93 | defp setup_worker_pool(_), do: :ok 94 | 95 | defp generate_queue_name do 96 | :erlang.unique_integer |> to_string 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /test/support/cluster.ex: -------------------------------------------------------------------------------- 1 | # lovingly adapted from https://github.com/phoenixframework/phoenix_pubsub/blob/master/test/support/cluster.ex 2 | 3 | defmodule Honeydew.Support.Cluster do 4 | def init do 5 | Node.start(:"primary@127.0.0.1") 6 | :erl_boot_server.start([:"127.0.0.1"]) 7 | end 8 | 9 | def spawn_nodes(nodes) do 10 | nodes 11 | |> Enum.map(&Task.async(fn -> spawn_node(&1) end)) 12 | |> Enum.map(&Task.await(&1, 30_000)) 13 | end 14 | 15 | def spawn_node(node_name) do 16 | {:ok, node} = :slave.start('127.0.0.1', String.to_atom(node_name), inet_loader_args()) 17 | add_code_paths(node) 18 | transfer_configuration(node) 19 | ensure_applications_started(node) 20 | {:ok, node} 21 | end 22 | 23 | defp rpc(node, module, function, args) do 24 | :rpc.block_call(node, module, function, args) 25 | end 26 | 27 | defp inet_loader_args do 28 | '-loader inet -hosts 127.0.0.1 -setcookie #{:erlang.get_cookie()}' 29 | end 30 | 31 | defp add_code_paths(node) do 32 | rpc(node, :code, :add_paths, [:code.get_path()]) 33 | end 34 | 35 | defp transfer_configuration(node) do 36 | for {app_name, _, _} <- Application.loaded_applications do 37 | for {key, val} <- Application.get_all_env(app_name) do 38 | rpc(node, Application, :put_env, [app_name, key, val]) 39 | end 40 | end 41 | end 42 | 43 | defp ensure_applications_started(node) do 44 | rpc(node, Application, :ensure_all_started, [:mix]) 45 | rpc(node, Mix, :env, [:test]) 46 | for {app_name, _, _} <- Application.loaded_applications do 47 | rpc(node, Application, :ensure_all_started, [app_name]) 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /test/support/cluster_setups.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.Support.ClusterSetups do 2 | alias Honeydew.Support.Cluster 3 | alias Honeydew.Processes 4 | 5 | def start_queue_node(queue) do 6 | fn -> 7 | :ok = Honeydew.start_queue(queue) 8 | end 9 | |> start_node(queue, :queue) 10 | end 11 | 12 | def start_worker_node(queue) do 13 | fn -> 14 | :ok = Honeydew.start_workers(queue, Stateless) 15 | end 16 | |> start_node(queue, :worker) 17 | end 18 | 19 | defp start_node(function, {:global, name} = queue, type) do 20 | {:ok, node} = 21 | [name, type] 22 | |> Enum.join("-") 23 | |> Cluster.spawn_node 24 | 25 | me = self() 26 | 27 | # seems to be necessary to get :pg to sync with the slaves 28 | Processes.start_process_group_scope(queue) 29 | 30 | Node.spawn_link(node, fn -> 31 | function.() 32 | send me, :ready 33 | Process.sleep(:infinity) # sleep so as to not terminate, and cause linked supervisor to crash 34 | end) 35 | 36 | receive do 37 | :ready -> {:ok, %{node: node}} 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/support/crash_logger_helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule Honeydew.CrashLoggerHelpers do 2 | @moduledoc """ 3 | Helpers for testing honeydew crash logs 4 | """ 5 | 6 | import ExUnit.Callbacks 7 | 8 | defmodule EchoingHoneydewCrashLoggerBackend do 9 | @moduledoc false 10 | @behaviour :gen_event 11 | 12 | def init(_) do 13 | {:ok, %{}} 14 | end 15 | 16 | def handle_call({:configure, opts}, state) do 17 | target = Keyword.fetch!(opts, :target) 18 | {:ok, :ok, Map.put(state, :target, target)} 19 | end 20 | 21 | def handle_event({_level, _from, {Logger, _msg, _ts, metadata}} =event, %{target: target} = state) when is_pid(target) do 22 | if Keyword.has_key?(metadata, :honeydew_crash_reason) do 23 | send(target, {:honeydew_crash_log, event}) 24 | end 25 | {:ok, state} 26 | end 27 | 28 | def handle_event(_, state), do: {:ok, state} 29 | 30 | def handle_info(_msg, state) do 31 | {:ok, state} 32 | end 33 | end 34 | 35 | @doc """ 36 | Sets up an error logger backend that sends `{:honeydew_crash_log, event}` 37 | messages on any log statement that has `:honeydew_crash_reason` in its 38 | metadata. 39 | """ 40 | def setup_echoing_error_logger(_) do 41 | test_pid = self() 42 | Logger.add_backend(EchoingHoneydewCrashLoggerBackend, target: test_pid) 43 | Logger.configure_backend(EchoingHoneydewCrashLoggerBackend, target: test_pid) 44 | 45 | on_exit fn -> 46 | Logger.remove_backend(EchoingLoggerBackend) 47 | wait_backend_removal(EchoingLoggerBackend) 48 | end 49 | end 50 | 51 | defp wait_backend_removal(module) do 52 | if module in :gen_event.which_handlers(Logger) do 53 | Process.sleep(20) 54 | wait_backend_removal(module) 55 | else 56 | :ok 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/support/workers/doc_test_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule DocTestWorker do 2 | def ping(_ip) do 3 | Process.sleep(1000) 4 | :pong 5 | end 6 | end 7 | -------------------------------------------------------------------------------- /test/support/workers/fail_init_once_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule FailInitOnceWorker do 2 | def init(test_process) do 3 | send test_process, :init_ran 4 | 5 | if Process.get(:already_failed) do 6 | {:ok, :state} 7 | else 8 | Process.put(:already_failed, true) 9 | throw "intentional init failure" 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /test/support/workers/failed_init_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule FailedInitWorker do 2 | def init(test_process) do 3 | Process.put(:test_process, test_process) 4 | send test_process, {:init, self()} 5 | 6 | receive do 7 | :raise -> 8 | raise "init failed" 9 | :throw -> 10 | throw "init failed" 11 | :exit -> 12 | Process.exit(self(), :eject) 13 | :ok -> 14 | {:ok, nil} 15 | end 16 | end 17 | 18 | def failed_init do 19 | :test_process 20 | |> Process.get 21 | |> send(:failed_init_ran) 22 | 23 | Honeydew.reinitialize_worker() 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /test/support/workers/stateful.ex: -------------------------------------------------------------------------------- 1 | defmodule Stateful do 2 | @behaviour Honeydew.Worker 3 | def init(state) do 4 | {:ok, state} 5 | end 6 | 7 | def send_msg(to, msg, state) do 8 | send(to, {msg, state}) 9 | end 10 | 11 | def return(term, state) do 12 | {term, state} 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/support/workers/stateless.ex: -------------------------------------------------------------------------------- 1 | defmodule Stateless do 2 | @behaviour Honeydew.Worker 3 | import Honeydew.Progress 4 | 5 | def send_msg(to, msg) do 6 | send(to, msg) 7 | end 8 | 9 | def return(term) do 10 | term 11 | end 12 | 13 | def sleep(time) do 14 | Process.sleep(time) 15 | end 16 | 17 | def crash(pid) do 18 | send pid, :job_ran 19 | raise "ignore this crash" 20 | end 21 | 22 | def emit_progress(update) do 23 | progress(update) 24 | Process.sleep(500) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | defmodule TestSuccessMode do 2 | @behaviour Honeydew.SuccessMode 3 | 4 | @impl true 5 | def validate_args!(to: to) when is_pid(to), do: :ok 6 | 7 | @impl true 8 | def handle_success(job, [to: to]) do 9 | send to, job 10 | end 11 | end 12 | 13 | defmodule Helper do 14 | def restart_honeydew(_context) do 15 | :ok = Application.stop(:honeydew) 16 | :ok = Application.start(:honeydew) 17 | end 18 | end 19 | 20 | defmodule Tree do 21 | def tree(supervisor) do 22 | supervisor 23 | |> do_tree(0) 24 | |> List.flatten 25 | |> Enum.join("\n") 26 | |> IO.puts 27 | end 28 | 29 | defp do_tree(supervisor, depth) do 30 | supervisor 31 | |> Supervisor.which_children 32 | |> Enum.map(fn 33 | {id, pid, :supervisor, module} -> 34 | str = String.duplicate(" ", depth) <> "|-" <> "#{inspect pid} #{inspect module} #{inspect id}" 35 | [str | do_tree(pid, depth+1)] 36 | {id, pid, :worker, module} -> 37 | String.duplicate(" ", depth) <> "|-" <> "#{inspect pid} #{inspect module} #{inspect id}" 38 | end) 39 | end 40 | end 41 | 42 | # defmodule BadInit do 43 | # def init(_) do 44 | # :bad 45 | # end 46 | # end 47 | 48 | # defmodule RaiseOnInit do 49 | # def init(_) do 50 | # raise "bad" 51 | # end 52 | # end 53 | 54 | # defmodule LinkDiesOnInit do 55 | # def init(_) do 56 | # spawn_link fn -> :born_to_die end 57 | # end 58 | # end 59 | 60 | Honeydew.Support.Cluster.init() 61 | 62 | ExUnit.start() 63 | --------------------------------------------------------------------------------