├── .formatter.exs ├── .github └── workflows │ └── main.yml ├── .gitignore ├── .gitlab-ci.yml ├── .tool-versions ├── CHANGELOG.md ├── README.md ├── config ├── config.exs ├── dev.exs └── test.exs ├── docker-compose.yml ├── lib ├── flume.ex ├── flume │ ├── bulk_event.ex │ ├── config.ex │ ├── default_logger.ex │ ├── errors.ex │ ├── event.ex │ ├── instrumentation.ex │ ├── instrumentation │ │ ├── default_event_handler.ex │ │ └── event_handler.ex │ ├── logger.ex │ ├── mock.ex │ ├── pipeline.ex │ ├── pipeline │ │ ├── api.ex │ │ ├── bulk_event │ │ │ └── worker.ex │ │ ├── context.ex │ │ ├── control │ │ │ └── options.ex │ │ ├── default_api.ex │ │ ├── event.ex │ │ ├── event │ │ │ ├── consumer.ex │ │ │ ├── producer.ex │ │ │ ├── producer_consumer.ex │ │ │ └── worker.ex │ │ ├── mock_api.ex │ │ ├── system_event.ex │ │ └── system_event │ │ │ ├── consumer.ex │ │ │ ├── producer.ex │ │ │ ├── supervisor.ex │ │ │ └── worker.ex │ ├── queue │ │ ├── api.ex │ │ ├── backoff.ex │ │ ├── default_api.ex │ │ ├── manager.ex │ │ ├── mock_api.ex │ │ ├── processing_scheduler.ex │ │ └── scheduler.ex │ ├── redis │ │ ├── bulk_dequeue.ex │ │ ├── client.ex │ │ ├── command.ex │ │ ├── job.ex │ │ ├── lock.ex │ │ ├── script.ex │ │ ├── sorted_set.ex │ │ └── supervisor.ex │ ├── supervisor.ex │ ├── support │ │ ├── pipelines.ex │ │ └── time.ex │ ├── utils.ex │ └── utils │ │ └── integer_extension.ex └── mix │ └── tasks │ └── redis_benchmark.ex ├── mix.exs ├── mix.lock ├── priv └── scripts │ ├── enqueue_processing_jobs.lua │ └── release_lock.lua └── test ├── factories └── job_factory.ex ├── flume ├── pipeline │ ├── control │ │ └── options_test.exs │ ├── event │ │ ├── consumer_test.exs │ │ ├── producer_consumer_test.exs │ │ ├── producer_test.exs │ │ └── worker_test.exs │ └── event_test.exs ├── pipeline_test.exs ├── queue │ └── manager_test.exs ├── redis │ ├── client_test.exs │ ├── command_test.exs │ ├── job_test.exs │ └── lock_test.exs └── utils_test.exs ├── flume_test.exs ├── support ├── echo_consumer.ex ├── echo_consumer_with_timestamp.ex ├── echo_context_worker.ex ├── echo_worker.ex ├── test_producer.ex └── test_with_redis.ex └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | inputs: [ 3 | "mix.exs", "{config,lib,test}/**/*.{ex,exs}" 4 | ], 5 | 6 | locals_without_parens: [ 7 | # Formatter tests 8 | assert_format: 2, 9 | assert_format: 3, 10 | assert_same: 1, 11 | assert_same: 2, 12 | 13 | # Errors tests 14 | assert_eval_raise: 3 15 | ] 16 | ] 17 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Elixir CI 2 | 3 | # This workflow runs on pushes to the main branch 4 | on: 5 | push: 6 | branches: [ master ] 7 | 8 | jobs: 9 | build: 10 | runs-on: elixir:1.8.2-otp-22 11 | steps: 12 | - uses: actions/checkout@v4 13 | 14 | - name: Install dependencies 15 | run: mix deps.get 16 | 17 | - name: Lint 18 | run: mix format --check-formatted 19 | 20 | - name: Start Redis container 21 | uses: supercharge/redis-github-action@1.7.0 22 | with: 23 | redis-version: '5' 24 | 25 | - name: Run tests 26 | run: | 27 | REDIS_HOST=localhost REDIS_PORT=6379 mix test 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | /log/ 23 | 24 | *.swp 25 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: elixir:1.6.6-alpine 2 | cache: 3 | key: flume 4 | paths: 5 | - _build 6 | - deps 7 | 8 | services: 9 | - redis:alpine 10 | 11 | .shared-runner: &shared-runner 12 | tags: 13 | - shared 14 | - runner 15 | 16 | stages: 17 | - lint 18 | - test 19 | 20 | before_script: 21 | - mix local.rebar --force 22 | - mix local.hex --force 23 | - mix deps.get --only test 24 | 25 | lint: 26 | <<: *shared-runner 27 | stage: lint 28 | script: 29 | - mix format --check-formatted 30 | 31 | test: 32 | <<: *shared-runner 33 | stage: test 34 | variables: 35 | FLUME_REDIS_HOST: localhost 36 | FLUME_REDIS_PORT: 6379 37 | MIX_ENV: test 38 | script: 39 | - mix coveralls.html --preload-modules --cover --color 40 | coverage: '/\[TOTAL\]\s+(\d+\.\d+)%/' 41 | artifacts: 42 | paths: 43 | - cover/ 44 | expire_in: 1 week 45 | tags: 46 | - shared 47 | - runner 48 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | elixir 1.8.2 2 | erlang 21.3 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.2.0 4 | * Add `Flume.API.job_counts/1` to get count of jobs in the pipeline which are yet to be processed. 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flume 2 | 3 | ![Test](https://github.com/scripbox/flume/workflows/CI/badge.svg?branch=master&event=push) 4 | 5 | Flume is a job processing system backed by [GenStage](https://github.com/elixir-lang/gen_stage) & [Redis](https://redis.io/) 6 | 7 | ## Table of Contents 8 | 9 | - [Features](#features) 10 | - [Requirements](#requirements) 11 | - [Installation](#installation) 12 | - [Usage](#usage) 13 | - [Pipelines](#pipelines) 14 | - [Enqueuing Jobs](#enqueuing-jobs) 15 | - [Creating Workers](#creating-workers) 16 | - [Scheduled Jobs](#scheduled-jobs) 17 | - [Rate Limiting](#rate-limiting) 18 | - [Batch Processing](#batch-processing) 19 | - [Pipeline Control](#pipeline-control) 20 | - [Instrumentation](#instrumentation) 21 | - [Writing Tests](#writing-tests) 22 | - [Roadmap](#roadmap) 23 | - [References](#references) 24 | - [Contributing](#contributing) 25 | 26 | ## Features 27 | 28 | - **Durability** - Jobs are backed up before processing. Incase of crashes, these 29 | jobs are restored. 30 | - **Back-pressure** - Uses [gen_stage](https://github.com/elixir-lang/gen_stage) to support this. 31 | - **Scheduled Jobs** - Jobs can be scheduled to run at any point in future. 32 | - **Rate Limiting** - Uses redis to maintain rate-limit on pipelines. 33 | - **Batch Processing** - Jobs are grouped based on size. 34 | - **Logging** - Provides a behaviour `Flume.Logger` to define your own logger module. 35 | - **Pipeline Control** - Queues can be pause/resume at runtime. 36 | - **Instrumentation** - Metrics like worker duration and latency to fetch jobs from redis are emitted via [telemetry](https://github.com/beam-telemetry/telemetry). 37 | - **Exponential Back-off** - On failure, jobs are retried with exponential back-off. Minimum and maximum can be set via configuration. 38 | 39 | ## Requirements 40 | 41 | - Elixir 1.6.6+ 42 | - Erlang/OTP 21.1+ 43 | - Redis 4.0+ 44 | 45 | ## Installation 46 | 47 | Add Flume to your list of dependencies in `mix.exs`: 48 | 49 | ```elixir 50 | def deps do 51 | [ 52 | {:flume, github: "scripbox/flume"} 53 | ] 54 | end 55 | ``` 56 | 57 | Then run `mix deps.get` to install Flume and its dependencies. 58 | 59 | ## Usage 60 | 61 | Add Flume supervisor to your application's supervision tree: 62 | 63 | ```elixir 64 | defmodule MyApplication.Application do 65 | use Application 66 | 67 | import Supervisor.Spec 68 | 69 | def start(_type, _args) do 70 | children = [ 71 | # Start Flume supervisor 72 | supervisor(Flume, []) 73 | ] 74 | 75 | opts = [strategy: :one_for_one, name: MyApplication.Supervisor] 76 | Supervisor.start_link(children, opts) 77 | end 78 | end 79 | ``` 80 | 81 | Add `config/flume.exs`: 82 | 83 | ```elixir 84 | config :flume, 85 | name: Flume, 86 | # Redis host 87 | host: "127.0.0.1", 88 | # Redis port 89 | port: "6379", 90 | # Redis keys namespace 91 | namespace: "my-app", 92 | # Redis database 93 | database: 0, 94 | # Redis pool size 95 | redis_pool_size: 10, 96 | # Redis connection timeout in ms (Default 5000 ms) 97 | redis_timeout: 10_000, 98 | # Retry backoff intial in ms (Default 500 ms) 99 | backoff_initial: 30_000, 100 | # Retry backoff maximum in ms (Default 10_000 ms) 101 | backoff_max: 36_00_000, 102 | # Maximum number of retries (Default 5) 103 | max_retries: 15, 104 | # Scheduled jobs poll interval in ms (Default 10_000 ms) 105 | scheduler_poll_interval: 10_000, 106 | # Time to move jobs from processing queue to retry queue in seconds (Default 600 sec) 107 | visibility_timeout: 600, 108 | # ttl of the acquired lock to fetch jobs for bulk pipelines in ms (Default 30_000 ms) 109 | dequeue_lock_ttl: 30_000, 110 | # process timeout to fetch jobs for bulk pipelines in ms (Default 10_000 ms) 111 | dequeue_process_timeout: 10_000, 112 | # time to poll the queue again if it was locked by another process in ms (Default 500 ms) 113 | dequeue_lock_poll_interval: 500 114 | ``` 115 | 116 | Import flume config in `config/config.exs` as given below: 117 | 118 | ```elixir 119 | ... 120 | import_config "#{Mix.env()}.exs" 121 | +import_config "flume.exs" 122 | ``` 123 | 124 | ### Pipelines 125 | 126 | Each pipeline is a GenStage pipeline having these parameters - 127 | 128 | * `name` - Name of the pipeline 129 | * `queue` - Name of the Redis queue to pull jobs from 130 | * `max_demand` - Maximum number of jobs to pull from the queue 131 | 132 | **Configuration** 133 | 134 | ```elixir 135 | config :flume, 136 | pipelines: [ 137 | %{name: "default_pipeline", queue: "default", max_demand: 1000}, 138 | ] 139 | ``` 140 | 141 | Flume supervisor will start these processes: 142 | 143 | ```asciidoc 144 | [Flume.Supervisor] <- (Supervisor) 145 | | 146 | | 147 | | 148 | [default_pipeline_producer] <- (Producer) 149 | | 150 | | 151 | | 152 | [default_pipeline_producer_consumer] <- (ProducerConsumer) 153 | | 154 | | 155 | | 156 | [default_pipeline_consumer_supervisor] <- (ConsumerSupervisor) 157 | / \ 158 | / \ 159 | / \ 160 | [worker_1] [worker_2] <- (Worker Processes) 161 | ``` 162 | 163 | ### Enqueuing Jobs 164 | 165 | Enqueuing jobs into flume requires these things - 166 | 167 | * Specify a `queue-name` (like `priority`) 168 | * Specify the worker module (`MyApp.FancyWorker`) 169 | * Specify the worker module's function name (default `:perform`) 170 | * Specify the arguments as per the worker module's function arity 171 | 172 | **With default function** 173 | 174 | ```elixir 175 | Flume.enqueue(:queue_name, MyApp.FancyWorker, [arg_1, arg_2]) 176 | ``` 177 | 178 | **With custom function** 179 | 180 | ```elixir 181 | Flume.enqueue(:queue_name, MyApp.FancyWorker, :myfunc, [arg_1, arg_2]) 182 | ``` 183 | 184 | ### Creating Workers 185 | 186 | Worker modules are responsible for processing a job. 187 | A worker module should define the `function-name` with the exact arity used while queuing the job. 188 | 189 | ```elixir 190 | defmodule MyApp.FancyWorker do 191 | def perform(arg_1, arg_2) do 192 | # your job processing logic 193 | end 194 | end 195 | ``` 196 | 197 | ### Scheduled Jobs 198 | 199 | **With default function** 200 | 201 | ```elixir 202 | # 10 seconds 203 | schedule_time = 10_000 204 | 205 | Flume.enqueue_in(:queue_name, schedule_time, MyApp.FancyWorker, [arg_1, arg_2]) 206 | ``` 207 | 208 | **With custom function** 209 | 210 | ```elixir 211 | # 10 seconds 212 | schedule_time = 10_000 213 | 214 | Flume.enqueue_in(:queue_name, schedule_time, MyApp.FancyWorker, :myfunc, [arg_1, arg_2]) 215 | ``` 216 | 217 | ### Rate Limiting 218 | 219 | Flume supports rate-limiting for each configured pipeline. 220 | 221 | Rate-Limiting has two key parameters - 222 | 223 | - `rate_limit_scale` - Time scale in `milliseconds` for the pipeline 224 | - `rate_limit_count` - Total number of jobs to be processed within the time scale 225 | - `rate_limit_key`(optional) - Using this option, rate limit can be set across pipelines. 226 | 227 |        **Note**: When this option is not set, rate limit will be maintained for a pipeline. 228 | 229 | ```elixir 230 | rate_limit_count = 1000 231 | rate_limit_scale = 6 * 1000 232 | 233 | config :flume, 234 | pipelines: [ 235 | # This pipeline will process 1000 jobs every 6 seconds 236 | %{ 237 | name: "promotional_email_pipeline", 238 | queue: "promotional_email", 239 | rate_limit_count: rate_limit_count, 240 | rate_limit_scale: rate_limit_scale, 241 | rate_limit_key: "email" 242 | }, 243 | %{ 244 | name: "transactional_email_pipeline", 245 | queue: "transactional_email", 246 | rate_limit_count: rate_limit_count, 247 | rate_limit_scale: rate_limit_scale, 248 | rate_limit_key: "email" 249 | } 250 | ] 251 | 252 | OR 253 | 254 | config :flume 255 | pipelines: [ 256 | %{ 257 | name: "webhooks_pipeline", 258 | queue: "webhooks", 259 | rate_limit_count: 1000, 260 | rate_limit_scale: 5000 261 | } 262 | ] 263 | ``` 264 | 265 | Flume will process the configured number of jobs (`rate_limit_count`) for each rate-limited pipeline, 266 | even if we are running multiple instances of our application. 267 | 268 | ### Batch Processing 269 | 270 | Flume supports batch-processing for each configured pipeline. 271 | It groups individual jobs by the configured `batch_size` option and 272 | each worker process will receive a group of jobs. 273 | 274 | ```elixir 275 | config :flume, 276 | pipelines: [ 277 | # This pipeline will pull (100 * 10) jobs from the queue 278 | # and group them in batches of 10. 279 | %{ 280 | name: "batch_pipeline", 281 | queue: "batch-queue", 282 | max_demand: 100, 283 | batch_size: 10 284 | } 285 | ] 286 | ``` 287 | 288 | ```elixir 289 | defmodule MyApp.BatchWorker do 290 | def perform(args) do 291 | # args will be a list of arguments 292 | # E.g - [[job_1_args], [job_2_args], ...] 293 | # your job processing logic 294 | end 295 | end 296 | ``` 297 | 298 | ### Pipeline Control 299 | 300 | Flume has support to pause/resume each pipeline. 301 | Once a pipeline is paused, the producer process will stop pulling jobs from the queue. 302 | It will process the jobs which are already pulled from the queue. 303 | 304 | Refer to "Options" section for supported options and default values. 305 | 306 | **Pause all pipelines** 307 | ```elixir 308 | # Pause all pipelines permanently (in Redis) and asynchronously 309 | Flume.pause_all(temporary: false, async: true) 310 | ``` 311 | 312 | **Pause a pipeline** 313 | 314 | ```elixir 315 | # Pause a pipeline temporarily (in current node) and asynchronously 316 | Flume.pause(:default_pipeline, temporary: true, async: true) 317 | ``` 318 | 319 | **Resume all pipelines** 320 | ```elixir 321 | # Resume all pipelines temporarily (in current node) and synchronously with infinite timeout 322 | Flume.resume_all(temporary: true, async: false, timeout: :infinity) 323 | ``` 324 | 325 | **Resume a pipeline** 326 | 327 | ```elixir 328 | # Resume a pipeline permanently (in Redis) and synchronously with a 10000 milli-second timeout 329 | Flume.resume(:default_pipeline, temporary: false, async: false, timeout: 10000) 330 | ``` 331 | 332 | #### Options 333 | The following options can be used to pause/resume a pipeline 334 | * `:async` - (boolean) Defaults to `false`. 335 | * `true` - The caller will not wait for the operation to complete. 336 | * `false` - The caller will wait for the operation to complete, this can lead to timeout if the operation takes too long to succeed. See https://hexdocs.pm/elixir/GenServer.html#call/3 for more details. 337 | * `:temporary` - (boolean) Defaults to `true`. 338 | * `true` - The pause/resume operation will be applied only on the current node. 339 | * `false` - Will update the value in persistent-store (Redis) and will apply the operation on all nodes. 340 | * `:timeout` - (timeout) Defaults to `5000`. Timeout(in milliseconds) for synchronous pause/resume calls. See https://hexdocs.pm/elixir/GenServer.html#call/3-timeouts for more details. 341 | 342 | ### Instrumentation 343 | 344 | We use [telemetry](https://github.com/beam-telemetry/telemetry) to emit metrics. 345 | Following metrics are emitted: 346 | 347 | - duration of a job/worker 348 | - count, latency and payload_size of dequeued jobs 349 | 350 | ## Writing Tests 351 | 352 | **To enable mock in the test environment** 353 | 354 | **config/test.exs** 355 | 356 | ```elixir 357 | config :flume, mock: true 358 | ``` 359 | 360 | **To mock individual test** 361 | 362 | ```elixir 363 | import Flume.Mock 364 | ... 365 | describe "enqueue/4" do 366 | test "mock works" do 367 | with_flume_mock do 368 | Flume.enqueue(:test, List, :last, [[1]]) 369 | 370 | assert_receive %{ 371 | queue: :test, 372 | worker: List, 373 | function_name: :last, 374 | args: [[1]] 375 | } 376 | end 377 | end 378 | end 379 | ``` 380 | 381 | **To enable mock for all tests in a module** 382 | 383 | ```elixir 384 | defmodule ListTest do 385 | use ExUnit.Case, async: true 386 | use Flume.Mock 387 | 388 | describe "enqueue/4" do 389 | test "mock works" do 390 | Flume.enqueue(:test, List, :last, [[1]]) 391 | 392 | assert_receive %{ 393 | queue: :test, 394 | worker: List, 395 | function_name: :last, 396 | args: [[1]] 397 | } 398 | end 399 | end 400 | end 401 | ``` 402 | 403 | ## Roadmap 404 | 405 | * Support multiple queue backends (right now only Redis is supported) 406 | 407 | ## References 408 | 409 | * Background Processing in Elixir with GenStage (https://medium.com/@scripbox_tech/background-processing-in-elixir-with-genstage-efb6cb8ca94a) 410 | 411 | ## Contributing 412 | 413 | * Check formatting (`mix format --check-formatted`) 414 | * Run all tests (`mix test`) 415 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | config :flume, 6 | name: Flume, 7 | host: {:system, "FLUME_REDIS_HOST", "127.0.0.1"}, 8 | port: {:system, "FLUME_REDIS_PORT", 6379}, 9 | namespace: "flume", 10 | database: 0, 11 | redis_timeout: 5000, 12 | redis_pool_size: 10, 13 | instrumentation: [ 14 | handler_module: Flume.Instrumentation.DefaultEventHandler, 15 | handler_function: :handle, 16 | metadata: [app_name: :flume] 17 | ], 18 | pipelines: [], 19 | backoff_initial: 500, 20 | backoff_max: 10_000, 21 | scheduler_poll_interval: 10_000, 22 | max_retries: 10 23 | 24 | import_config "#{Mix.env()}.exs" 25 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :flume, 4 | name: Flume, 5 | host: {:system, "FLUME_REDIS_HOST", "127.0.0.1"}, 6 | port: {:system, "FLUME_REDIS_PORT", "6379"}, 7 | namespace: "flume_dev", 8 | database: 0, 9 | redis_timeout: 5000, 10 | redis_pool_size: 5, 11 | pipelines: [ 12 | %{ 13 | name: "default-pipeline", 14 | queue: "default", 15 | max_demand: 5 16 | }, 17 | %{ 18 | name: "low-pipeline", 19 | queue: "low", 20 | max_demand: 2 21 | }, 22 | %{ 23 | name: "priority-pipeline", 24 | queue: "priority", 25 | max_demand: 10 26 | }, 27 | %{ 28 | name: "limited-pipeline", 29 | queue: "limited", 30 | max_demand: 5, 31 | rate_limit_count: 10, 32 | rate_limit_scale: 10_000, 33 | rate_limit_key: "limited", 34 | instrument: true 35 | }, 36 | %{ 37 | name: "batch-pipeline", 38 | queue: "batch", 39 | max_demand: 5, 40 | batch_size: 2, 41 | instrument: true 42 | } 43 | ], 44 | backoff_initial: 500, 45 | backoff_max: 10_000, 46 | scheduler_poll_interval: 10_000, 47 | max_retries: 10 48 | 49 | config :logger, backends: [{LoggerFileBackend, :error_log}] 50 | 51 | config :logger, :error_log, 52 | format: "$date $time $metadata [$level] $message\n", 53 | path: "log/dev.log", 54 | level: :debug 55 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :flume, 4 | name: Flume, 5 | host: {:system, "FLUME_REDIS_HOST", "127.0.0.1"}, 6 | port: {:system, "FLUME_REDIS_PORT", "6379"}, 7 | namespace: "flume_test", 8 | database: 0, 9 | redis_timeout: 5000, 10 | redis_pool_size: 1, 11 | pipelines: [%{name: "default_pipeline", queue: "default", max_demand: 1000}], 12 | backoff_initial: 1, 13 | backoff_max: 2, 14 | scheduler_poll_interval: 10_000, 15 | max_retries: 5 16 | 17 | config :logger, 18 | format: "[$level] $message\n", 19 | backends: [{LoggerFileBackend, :error_log}], 20 | level: :warn 21 | 22 | config :logger, :error_log, 23 | path: "log/test.log", 24 | level: :warn 25 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | redis: 2 | image: redis 3 | ports: 4 | - '6379:6379' 5 | 6 | flume: 7 | image: bitwalker/alpine-elixir:1.6.6 8 | command: elixir --sname flume --cookie flume -S mix run --no-halt 9 | working_dir: /opt/flume 10 | environment: 11 | - FLUME_REDIS_HOST=redis 12 | volumes: 13 | - '.:/opt/flume' 14 | links: 15 | - redis 16 | -------------------------------------------------------------------------------- /lib/flume.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume do 2 | alias Flume.{Config, Pipeline} 3 | alias Flume.Queue.Manager 4 | 5 | @deprecated "Flume.start/2 instead" 6 | defdelegate start(type, args), to: Flume.Supervisor 7 | 8 | @deprecated "Flume.start_link/0 instead" 9 | defdelegate start_link, to: Flume.Supervisor 10 | 11 | def bulk_enqueue(queue, jobs) do 12 | apply(Config.queue_api_module(), :bulk_enqueue, [queue, jobs]) 13 | end 14 | 15 | def bulk_enqueue(queue, jobs, opts) do 16 | apply(Config.queue_api_module(), :bulk_enqueue, [queue, jobs, opts]) 17 | end 18 | 19 | def enqueue(queue, worker, args) do 20 | apply(Config.queue_api_module(), :enqueue, [queue, worker, args]) 21 | end 22 | 23 | def enqueue(queue, worker, function_name, args) do 24 | apply(Config.queue_api_module(), :enqueue, [queue, worker, function_name, args]) 25 | end 26 | 27 | def enqueue(queue, worker, function_name, args, opts) do 28 | apply(Config.queue_api_module(), :enqueue, [ 29 | queue, 30 | worker, 31 | function_name, 32 | args, 33 | opts 34 | ]) 35 | end 36 | 37 | def enqueue_in(queue, time_in_seconds, worker, args) do 38 | apply(Config.queue_api_module(), :enqueue_in, [queue, time_in_seconds, worker, args]) 39 | end 40 | 41 | def enqueue_in(queue, time_in_seconds, worker, function_name, args) do 42 | apply(Config.queue_api_module(), :enqueue_in, [ 43 | queue, 44 | time_in_seconds, 45 | worker, 46 | function_name, 47 | args 48 | ]) 49 | end 50 | 51 | def enqueue_in(queue, time_in_seconds, worker, function_name, args, opts) do 52 | apply(Config.queue_api_module(), :enqueue_in, [ 53 | queue, 54 | time_in_seconds, 55 | worker, 56 | function_name, 57 | args, 58 | opts 59 | ]) 60 | end 61 | 62 | @spec pause(Flume.Pipeline.Control.Options.option_spec()) :: list(:ok) 63 | def pause_all(options \\ []) do 64 | Config.pipeline_names() |> Enum.map(&pause(&1, options)) 65 | end 66 | 67 | @spec pause(Flume.Pipeline.Control.Options.option_spec()) :: list(:ok) 68 | def resume_all(options \\ []) do 69 | Config.pipeline_names() |> Enum.map(&resume(&1, options)) 70 | end 71 | 72 | @spec pause(String.t() | atom(), Flume.Pipeline.Control.Options.option_spec()) :: :ok 73 | defdelegate pause(pipeline_name, options \\ []), to: Pipeline 74 | 75 | @spec resume(String.t() | atom(), Flume.Pipeline.Control.Options.option_spec()) :: :ok 76 | defdelegate resume(pipeline_name, options \\ []), to: Pipeline 77 | 78 | defdelegate pipelines, to: Config 79 | 80 | def pending_jobs_count(pipeline_names \\ Config.pipeline_names()) do 81 | Pipeline.Event.pending_workers_count(pipeline_names) + 82 | Pipeline.SystemEvent.pending_workers_count() 83 | end 84 | 85 | def worker_context, do: Pipeline.Context.get() 86 | 87 | @doc """ 88 | Returns count of jobs in the pipeline which are yet to be processed. 89 | 90 | ## Examples 91 | ``` 92 | Flume.API.job_counts(["queue-1", "queue-2"]) 93 | {:ok, [2, 3]} 94 | 95 | Flume.API.job_counts(["queue-1", "not-a-queue-name"]) 96 | {:ok, [2, 0]} 97 | ``` 98 | """ 99 | @spec job_counts(nonempty_list(binary)) :: 100 | {:ok, nonempty_list(Redix.Protocol.redis_value())} 101 | | {:error, atom | Redix.Error.t() | Redix.ConnectionError.t()} 102 | def job_counts(queues), do: Manager.job_counts(namespace(), queues) 103 | 104 | defp namespace, do: Config.namespace() 105 | end 106 | -------------------------------------------------------------------------------- /lib/flume/bulk_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.BulkEvent do 2 | @moduledoc """ 3 | This module is responsible for creating the bulk event 4 | received by the consumer stages. 5 | """ 6 | # Sample Event Schema 7 | # { 8 | # "class": "Elixir.Worker", 9 | # "function": "perform", 10 | # "queue": "test", 11 | # "args": [[["arg_1"], ["arg_2"]], 12 | # "events": [] 13 | # } 14 | 15 | alias Flume.Event 16 | 17 | @keys [ 18 | class: nil, 19 | function: nil, 20 | queue: nil, 21 | args: [], 22 | events: [] 23 | ] 24 | 25 | @type t :: %__MODULE__{ 26 | class: String.t() | atom, 27 | function: String.t(), 28 | queue: String.t(), 29 | args: List.t(), 30 | events: List.t() 31 | } 32 | 33 | defstruct @keys 34 | 35 | # @doc false 36 | def new(%Event{} = event) do 37 | struct(__MODULE__, %{ 38 | class: event.class, 39 | function: event.function, 40 | queue: event.queue, 41 | args: [[event.args]], 42 | events: [event] 43 | }) 44 | end 45 | 46 | def new([]), do: struct(__MODULE__, %{}) 47 | 48 | def new([first_event | other_events]) do 49 | new(first_event) 50 | |> append(other_events) 51 | end 52 | 53 | def new(_) do 54 | struct(__MODULE__, %{}) 55 | end 56 | 57 | defp append(%__MODULE__{args: [bulk_args]} = bulk_event, %Event{} = event) do 58 | bulk_event 59 | |> Map.merge(%{ 60 | args: [bulk_args ++ [event.args]], 61 | events: bulk_event.events ++ [event] 62 | }) 63 | end 64 | 65 | defp append(%__MODULE__{} = bulk_event, []) do 66 | bulk_event 67 | end 68 | 69 | defp append(%__MODULE__{} = bulk_event, [event | other_events]) do 70 | bulk_event 71 | |> append(event) 72 | |> append(other_events) 73 | end 74 | 75 | defp append(%__MODULE__{} = bulk_event, _) do 76 | bulk_event 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/flume/config.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Config do 2 | @defaults %{ 3 | backoff_initial: 500, 4 | backoff_max: 10_000, 5 | database: 0, 6 | host: "127.0.0.1", 7 | logger: Flume.DefaultLogger, 8 | mock: false, 9 | max_retries: 5, 10 | name: Flume, 11 | namespace: "flume", 12 | password: nil, 13 | pipelines: [], 14 | port: 6379, 15 | redis_pool_size: 10, 16 | redis_timeout: 5000, 17 | scheduler_poll_interval: 10_000, 18 | dequeue_lock_ttl: 30_000, 19 | dequeue_process_timeout: 10_000, 20 | dequeue_lock_poll_interval: 500, 21 | debug_log: false, 22 | # In seconds 23 | visibility_timeout: 600, 24 | instrumentation: [ 25 | handler_module: Flume.Instrumentation.DefaultEventHandler, 26 | handler_function: :handle, 27 | config: [app_name: :flume] 28 | ] 29 | } 30 | 31 | @integer_keys [ 32 | :port, 33 | :database, 34 | :redis_timeout, 35 | :scheduler_poll_interval, 36 | :backoff_initial, 37 | :backoff_max 38 | ] 39 | 40 | alias Flume.Utils.IntegerExtension 41 | 42 | Map.keys(@defaults) 43 | |> Enum.each(fn key -> 44 | def unquote(key)(), do: get(unquote(key)) 45 | end) 46 | 47 | def get(key), do: get(key, default(key)) 48 | 49 | def get(key, fallback) do 50 | value = 51 | case Application.get_env(:flume, key, fallback) do 52 | {:system, varname} -> System.get_env(varname) 53 | {:system, varname, default} -> System.get_env(varname) || default 54 | value -> value 55 | end 56 | 57 | parse(key, value) 58 | end 59 | 60 | defp default(key), do: Map.get(@defaults, key) 61 | 62 | defp parse(key, value) when key in @integer_keys do 63 | case IntegerExtension.parse(value) do 64 | :error -> 65 | raise Flume.Errors.InvalidConfiguration, key 66 | 67 | parsed_value -> 68 | parsed_value 69 | end 70 | end 71 | 72 | defp parse(_key, value), do: value 73 | 74 | def redis_opts(opts) do 75 | [ 76 | host: Keyword.get(opts, :host, host()), 77 | port: Keyword.get(opts, :port, port()), 78 | database: Keyword.get(opts, :database, database()), 79 | password: Keyword.get(opts, :password, password()) 80 | ] 81 | end 82 | 83 | def connection_opts(opts) do 84 | [timeout: Keyword.get(opts, :timeout, redis_timeout())] 85 | end 86 | 87 | def scheduler_opts do 88 | [ 89 | namespace: namespace(), 90 | scheduler_poll_interval: scheduler_poll_interval() 91 | ] 92 | end 93 | 94 | def queues, do: Enum.map(pipelines(), & &1.queue) 95 | 96 | def pipeline_names, do: Enum.map(pipelines(), & &1.name) 97 | 98 | def queue_api_module do 99 | case mock() do 100 | false -> 101 | Flume.Queue.DefaultAPI 102 | 103 | true -> 104 | Flume.Queue.MockAPI 105 | end 106 | end 107 | 108 | def pipeline_api_module do 109 | case mock() do 110 | false -> 111 | Flume.Pipeline.DefaultAPI 112 | 113 | true -> 114 | Flume.Pipeline.MockAPI 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /lib/flume/default_logger.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.DefaultLogger do 2 | @behaviour Flume.Logger 3 | 4 | require Logger 5 | 6 | def debug(message, %{}) do 7 | if Flume.Config.debug_log(), do: Logger.debug(message) 8 | end 9 | 10 | def debug(message, opts) do 11 | if Flume.Config.debug_log(), do: Logger.debug("#{message} - #{inspect(opts)}") 12 | end 13 | 14 | def error(message, %{}), do: Logger.error(message) 15 | def error(message, opts), do: Logger.error("#{message} - #{inspect(opts)}") 16 | 17 | def info(message, %{}), do: Logger.info(message) 18 | def info(message, opts), do: Logger.info("#{message} - #{inspect(opts)}") 19 | 20 | def warn(message, %{}), do: Logger.warn(message) 21 | def warn(message, opts), do: Logger.warn("#{message} - #{inspect(opts)}") 22 | end 23 | -------------------------------------------------------------------------------- /lib/flume/errors.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Errors do 2 | defmodule InvalidConfiguration do 3 | defexception [:message] 4 | 5 | def exception(key) do 6 | msg = "Invalid configuration for #{key}" 7 | %__MODULE__{message: msg} 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/flume/event.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Event do 2 | @moduledoc """ 3 | This module is responsible for decoding the 4 | serialized event received by the consumer stages. 5 | """ 6 | 7 | @default_function_name "perform" 8 | 9 | # Sample Event Schema 10 | # { 11 | # "class": "Elixir.Worker", 12 | # "function": "perform", 13 | # "queue": "test", 14 | # "jid": "1082fd87-2508-4eb4-8fba-2958584a60e3", 15 | # "args": [1], 16 | # "retry_count": 1, 17 | # "enqueued_at": 1514367662, 18 | # "finished_at": 1514367664, 19 | # "failed_at": null, 20 | # "retried_at": null, 21 | # "error_message": "", 22 | # "error_backtrace": "error backtrace" 23 | # } 24 | @keys [ 25 | class: nil, 26 | function: @default_function_name, 27 | queue: nil, 28 | jid: nil, 29 | args: [], 30 | retry_count: 0, 31 | enqueued_at: nil, 32 | finished_at: nil, 33 | failed_at: nil, 34 | retried_at: nil, 35 | error_message: nil, 36 | error_backtrace: nil, 37 | context: nil 38 | ] 39 | 40 | @type t :: %__MODULE__{ 41 | class: String.t() | atom, 42 | function: String.t(), 43 | queue: String.t(), 44 | jid: String.t(), 45 | args: List.t(), 46 | retry_count: non_neg_integer, 47 | enqueued_at: DateTime.t(), 48 | finished_at: DateTime.t(), 49 | failed_at: DateTime.t(), 50 | retried_at: DateTime.t(), 51 | error_message: String.t(), 52 | error_backtrace: String.t(), 53 | context: map() 54 | } 55 | 56 | @derive {Jason.Encoder, only: Keyword.keys(@keys)} 57 | defstruct [:original_json | @keys] 58 | 59 | @doc false 60 | def new(attributes) when is_map(attributes) do 61 | struct(__MODULE__, %{ 62 | class: attributes["class"], 63 | function: attributes["function"], 64 | queue: attributes["queue"], 65 | jid: attributes["jid"], 66 | args: attributes["args"], 67 | retry_count: attributes["retry_count"], 68 | enqueued_at: attributes["enqueued_at"], 69 | finished_at: attributes["finished_at"], 70 | failed_at: attributes["failed_at"], 71 | retried_at: attributes["retried_at"], 72 | error_message: attributes["error_message"], 73 | error_backtrace: attributes["error_backtrace"], 74 | context: attributes["context"] 75 | }) 76 | end 77 | 78 | def new(_) do 79 | struct(__MODULE__, %{}) 80 | end 81 | 82 | @doc """ 83 | Decode the JSON payload storing the original json as part of the struct. 84 | """ 85 | @spec decode(binary) :: {:ok, %__MODULE__{}} | {:error, Jason.DecodeError.t()} 86 | def decode(payload) do 87 | case Jason.decode(payload) do 88 | {:ok, %{"args" => %{}} = response} -> 89 | {:ok, %__MODULE__{__MODULE__.new(response) | original_json: payload, args: []}} 90 | 91 | {:ok, response} -> 92 | %__MODULE__{__MODULE__.new(response) | original_json: payload} 93 | 94 | {:error, error} -> 95 | {:error, error} 96 | 97 | {:error, :invalid, pos} -> 98 | {:error, "Invalid json at position: #{pos}"} 99 | end 100 | end 101 | 102 | @doc """ 103 | Decode the JSON payload storing the original json as part of the struct, raising if there is an error 104 | """ 105 | @spec decode!(binary) :: %__MODULE__{} 106 | def decode!(payload) do 107 | case Jason.decode!(payload) do 108 | %{"args" => %{}} = response -> 109 | %__MODULE__{__MODULE__.new(response) | original_json: payload, args: []} 110 | 111 | response -> 112 | %__MODULE__{__MODULE__.new(response) | original_json: payload} 113 | end 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /lib/flume/instrumentation.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Instrumentation do 2 | defdelegate attach(handler_id, event_name, function, config), to: :telemetry 3 | 4 | defdelegate attach_many(handler_id, event_names, function, config), to: :telemetry 5 | 6 | def execute(event_name, event_value, true = _instrument) do 7 | :telemetry.execute(event_name, event_value) 8 | end 9 | 10 | # Skip invoking telemetry when instrument is not true 11 | def execute(_event_name, _event_value, _instrument), do: nil 12 | 13 | def execute(event_name, event_value, metadata, true = _instrument) do 14 | :telemetry.execute(event_name, event_value, metadata) 15 | end 16 | 17 | # Skip invoking telemetry when instrument is not true 18 | def execute(_event_name, _event_value, _metadata, _instrument), do: nil 19 | 20 | defmacro measure(do: yield) do 21 | quote do 22 | start_time = Flume.Instrumentation.unix_seconds() 23 | result = unquote(yield) 24 | duration = Flume.Instrumentation.unix_seconds() - start_time 25 | {duration, result} 26 | end 27 | end 28 | 29 | def unix_seconds(unit \\ :millisecond), do: DateTime.to_unix(DateTime.utc_now(), unit) 30 | 31 | def format_module(module_name) do 32 | module_name |> to_string() |> String.replace(".", "_") |> String.downcase() 33 | end 34 | 35 | def format_event_name([]), do: "" 36 | 37 | def format_event_name([name | other_names]) do 38 | Path.join("#{name}", format_event_name(other_names)) 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/flume/instrumentation/default_event_handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Instrumentation.DefaultEventHandler do 2 | @behaviour Flume.Instrumentation.EventHandler 3 | 4 | require Flume.Logger 5 | 6 | alias Flume.{Instrumentation, Logger} 7 | 8 | def handle( 9 | event_name, 10 | %{value: value}, 11 | %{module: module}, 12 | nil 13 | ) do 14 | Logger.info("#{metric_path(event_name, module)} - #{value}") 15 | end 16 | 17 | def handle( 18 | event_name, 19 | %{value: value}, 20 | %{module: module}, 21 | app_name: app_name 22 | ) do 23 | Logger.info("#{app_name}/#{metric_path(event_name, module)} - #{value}") 24 | end 25 | 26 | def handle(event_name, %{value: value}, _metadata, nil) do 27 | Logger.info("#{Instrumentation.format_event_name(event_name)} - #{value}") 28 | end 29 | 30 | def handle(event_name, %{value: value}, _metadata, app_name: app_name) do 31 | Logger.info("#{app_name}/#{Instrumentation.format_event_name(event_name)} - #{value}") 32 | end 33 | 34 | def handle(_, _, _, _) do 35 | :ok 36 | end 37 | 38 | defp metric_path(event_name, nil), do: Instrumentation.format_event_name(event_name) 39 | 40 | defp metric_path(event_name, module) do 41 | formatted_event_name = Instrumentation.format_event_name(event_name) 42 | "#{formatted_event_name}/#{module}" 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/flume/instrumentation/event_handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Instrumentation.EventHandler do 2 | @moduledoc """ 3 | This module acts as an interface for dispatching telemetry events. 4 | """ 5 | @callback handle(atom(), integer(), any(), any()) :: any() 6 | end 7 | -------------------------------------------------------------------------------- /lib/flume/logger.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Logger do 2 | @moduledoc """ 3 | Behaviour module for logging to ensure that 4 | the following callbacks are implemented 5 | """ 6 | 7 | @callback debug(String.t(), map()) :: :ok | :error 8 | @callback error(String.t(), map()) :: :ok | :error 9 | @callback info(String.t(), map()) :: :ok | :error 10 | @callback warn(String.t(), map()) :: :ok | :error 11 | 12 | defmacro debug(message) do 13 | quote location: :keep do 14 | apply(Flume.Config.logger(), :debug, [unquote(message), %{}]) 15 | end 16 | end 17 | 18 | defmacro error(message) do 19 | quote location: :keep do 20 | apply(Flume.Config.logger(), :error, [unquote(message), %{}]) 21 | end 22 | end 23 | 24 | defmacro info(message) do 25 | quote location: :keep do 26 | apply(Flume.Config.logger(), :info, [unquote(message), %{}]) 27 | end 28 | end 29 | 30 | defmacro warn(message) do 31 | quote location: :keep do 32 | apply(Flume.Config.logger(), :warn, [unquote(message), %{}]) 33 | end 34 | end 35 | 36 | defmacro debug(message, opts) do 37 | quote location: :keep do 38 | apply(Flume.Config.logger(), :debug, [unquote(message), unquote(opts)]) 39 | end 40 | end 41 | 42 | defmacro error(message, opts) do 43 | quote location: :keep do 44 | apply(Flume.Config.logger(), :error, [unquote(message), unquote(opts)]) 45 | end 46 | end 47 | 48 | defmacro info(message, opts) do 49 | quote location: :keep do 50 | apply(Flume.Config.logger(), :info, [unquote(message), unquote(opts)]) 51 | end 52 | end 53 | 54 | defmacro warn(message, opts) do 55 | quote location: :keep do 56 | apply(Flume.Config.logger(), :warn, [unquote(message), unquote(opts)]) 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/flume/mock.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Mock do 2 | defmacro __using__(_) do 3 | quote do 4 | setup _mock do 5 | Application.put_env(:flume, :mock, true) 6 | 7 | on_exit(fn -> 8 | Application.put_env(:flume, :mock, false) 9 | end) 10 | 11 | :ok 12 | end 13 | end 14 | end 15 | 16 | defmacro with_flume_mock(do: yield) do 17 | quote do 18 | Application.put_env(:flume, :mock, true) 19 | 20 | on_exit(fn -> 21 | Application.put_env(:flume, :mock, false) 22 | end) 23 | 24 | unquote(yield) 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/flume/pipeline.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline do 2 | alias Flume.Pipeline.Control 3 | alias Flume.Utils.IntegerExtension 4 | 5 | @default_max_demand 500 6 | 7 | defstruct [ 8 | :name, 9 | :queue, 10 | :rate_limit_count, 11 | :rate_limit_scale, 12 | :rate_limit_key, 13 | :max_demand, 14 | :batch_size, 15 | :paused, 16 | :producer, 17 | :instrument 18 | ] 19 | 20 | def new(%{name: name, queue: queue} = opts) do 21 | batch_size = IntegerExtension.parse(opts[:batch_size], nil) 22 | rate_limit_count = IntegerExtension.parse(opts[:rate_limit_count], nil) 23 | rate_limit_scale = IntegerExtension.parse(opts[:rate_limit_scale], nil) 24 | max_demand = IntegerExtension.parse(opts[:max_demand], @default_max_demand) 25 | 26 | %__MODULE__{ 27 | name: name, 28 | queue: queue, 29 | rate_limit_count: rate_limit_count, 30 | rate_limit_scale: rate_limit_scale, 31 | rate_limit_key: opts[:rate_limit_key], 32 | max_demand: max_demand, 33 | batch_size: batch_size, 34 | instrument: opts[:instrument] 35 | } 36 | end 37 | 38 | def pause(pipeline_name, options \\ []) do 39 | with {:ok, pipeline_name} <- validate_pipeline_name(pipeline_name), 40 | {:ok, options} <- Control.Options.sanitized_options(options) do 41 | apply(Flume.Config.pipeline_api_module(), :pause, [pipeline_name, options]) 42 | end 43 | end 44 | 45 | def resume(pipeline_name, options \\ []) do 46 | with {:ok, pipeline_name} <- validate_pipeline_name(pipeline_name), 47 | {:ok, options} <- Control.Options.sanitized_options(options) do 48 | apply(Flume.Config.pipeline_api_module(), :resume, [pipeline_name, options]) 49 | end 50 | end 51 | 52 | defp validate_pipeline_name(pipeline_name) when is_atom(pipeline_name), 53 | do: validate_pipeline_name(to_string(pipeline_name)) 54 | 55 | defp validate_pipeline_name(pipeline_name) when is_binary(pipeline_name) do 56 | if Enum.any?(Flume.Config.pipeline_names(), &(&1 == pipeline_name)) do 57 | {:ok, pipeline_name} 58 | else 59 | {:error, "pipeline #{pipeline_name} has not been configured"} 60 | end 61 | end 62 | 63 | defp validate_pipeline_name(_) do 64 | {:error, "invalid value for a pipeline name"} 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/flume/pipeline/api.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.API do 2 | @callback pause(String.t(), keyword()) :: :ok | {:error, String.t()} 3 | @callback resume(String.t(), keyword()) :: :ok | {:error, String.t()} 4 | end 5 | -------------------------------------------------------------------------------- /lib/flume/pipeline/bulk_event/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.BulkEvent.Worker do 2 | require Flume.{Instrumentation, Logger} 3 | 4 | alias Flume.{BulkEvent, Instrumentation, Logger} 5 | alias Flume.Pipeline.SystemEvent, as: SystemEventPipeline 6 | alias Flume.Pipeline.Context, as: WorkerContext 7 | 8 | def process(%{name: pipeline_name} = pipeline, %BulkEvent{class: class} = bulk_event) do 9 | {duration, _} = 10 | Instrumentation.measure do 11 | Logger.debug("#{pipeline_name} [Consumer] received bulk event - #{inspect(bulk_event)}") 12 | 13 | set_context(bulk_event.events) 14 | 15 | do_process_event(pipeline, bulk_event) 16 | end 17 | 18 | Instrumentation.execute( 19 | [String.to_atom(pipeline_name), :worker], 20 | %{duration: duration}, 21 | %{module: Instrumentation.format_module(class)}, 22 | pipeline[:instrument] 23 | ) 24 | rescue 25 | e in [ArgumentError] -> 26 | Logger.error( 27 | "#{pipeline_name} [Consumer] error while processing event: #{Kernel.inspect(e)}" 28 | ) 29 | end 30 | 31 | defp set_context(events), do: extract_contexts(events) |> WorkerContext.put() 32 | 33 | defp do_process_event(%{name: pipeline_name} = pipeline, %BulkEvent{ 34 | class: class, 35 | function: function, 36 | args: args, 37 | events: events 38 | }) do 39 | {duration, _} = 40 | Instrumentation.measure do 41 | apply_function(%{class: class, function_name: function}, args) 42 | end 43 | 44 | Instrumentation.execute( 45 | [String.to_atom(pipeline_name), :worker, :job], 46 | %{duration: duration}, 47 | %{module: Instrumentation.format_module(class)}, 48 | pipeline[:instrument] 49 | ) 50 | 51 | Logger.debug("#{pipeline_name} [Consumer] processed bulk event: #{class}") 52 | 53 | # Mark all events as success 54 | events 55 | |> Enum.map(fn event -> {:success, event} end) 56 | |> SystemEventPipeline.enqueue() 57 | rescue 58 | e in _ -> 59 | error_message = Kernel.inspect(e) 60 | handle_failure(pipeline_name, class, events, error_message) 61 | catch 62 | :exit, {:timeout, message} -> 63 | handle_failure(pipeline_name, class, events, inspect(message)) 64 | end 65 | 66 | defp apply_function(%{class: class, function_name: function_name}, args) do 67 | function_name = String.to_atom(function_name) 68 | 69 | [class] 70 | |> Module.safe_concat() 71 | |> apply(function_name, args) 72 | end 73 | 74 | defp extract_contexts(events) do 75 | events 76 | |> Enum.map(& &1.context) 77 | |> Enum.reject(&is_nil/1) 78 | end 79 | 80 | defp handle_failure(pipeline_name, class, events, error_message) do 81 | Logger.error("#{pipeline_name} [Consumer] failed with error: #{error_message}", %{ 82 | worker_name: class 83 | }) 84 | 85 | # Mark all events as failed 86 | events 87 | |> Enum.map(fn event -> {:failed, event, error_message} end) 88 | |> SystemEventPipeline.enqueue() 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/flume/pipeline/context.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Context do 2 | @namespace "flume_worker_context" 3 | 4 | def put(nil), do: nil 5 | 6 | def put(context) do 7 | Process.put(@namespace, context) 8 | end 9 | 10 | def get do 11 | Process.get(@namespace) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/flume/pipeline/control/options.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Control.Options do 2 | @option_specs [ 3 | async: [type: :boolean, default: false], 4 | timeout: [type: :timeout, default: 5000], 5 | temporary: [type: :boolean, default: true] 6 | ] 7 | 8 | @doc """ 9 | Returns options for pausing/resuming a pipeline 10 | ### Examples 11 | iex> Flume.Pipeline.Control.Options.sanitized_options 12 | {:ok, [temporary: true, timeout: 5000, async: false]} 13 | iex> Flume.Pipeline.Control.Options.sanitized_options([unwanted: "option", async: false, timeout: 1000]) 14 | {:ok, [temporary: true, timeout: 1000, async: false]} 15 | iex> Flume.Pipeline.Control.Options.sanitized_options([temporary: true, async: true, timeout: :infinity, extra: "value"]) 16 | {:ok, [temporary: true, timeout: :infinity, async: true]} 17 | iex> Flume.Pipeline.Control.Options.sanitized_options([temporary: 1, async: false]) 18 | {:error, "expected :temporary to be a boolean, got: 1"} 19 | iex> Flume.Pipeline.Control.Options.sanitized_options([temporary: false, async: 0]) 20 | {:error, "expected :async to be a boolean, got: 0"} 21 | iex> Flume.Pipeline.Control.Options.sanitized_options([temporary: false, async: true, timeout: -1]) 22 | {:error, "expected :timeout to be a non-negative integer or :infinity, got: -1"} 23 | iex> Flume.Pipeline.Control.Options.sanitized_options([temporary: false, async: true, timeout: :inf]) 24 | {:error, "expected :timeout to be a non-negative integer or :infinity, got: :inf"} 25 | """ 26 | @type option_spec :: [ 27 | async: boolean(), 28 | temporary: boolean(), 29 | timeout: timeout() 30 | ] 31 | 32 | @spec sanitized_options(option_spec()) :: {:ok, option_spec()} | {:error, String.t()} 33 | def sanitized_options(options \\ []) do 34 | Enum.reduce_while(@option_specs, {:ok, []}, fn {key, _} = ele, {:ok, result} -> 35 | case validate_option(ele, options) do 36 | {:ok, value} -> 37 | {:cont, {:ok, Keyword.put(result, key, value)}} 38 | 39 | {:error, _} = result -> 40 | {:halt, result} 41 | end 42 | end) 43 | end 44 | 45 | defp validate_option({key, spec}, options) do 46 | value = Keyword.get(options, key, spec[:default]) 47 | validate_type(spec[:type], key, value) 48 | end 49 | 50 | defp validate_type(:boolean, key, value) when not is_boolean(value), 51 | do: {:error, "expected #{inspect(key)} to be a boolean, got: #{inspect(value)}"} 52 | 53 | defp validate_type(:timeout, key, value) 54 | when (not is_integer(value) or value < 0) and value != :infinity, 55 | do: 56 | {:error, 57 | "expected #{inspect(key)} to be a non-negative integer or :infinity, got: #{ 58 | inspect(value) 59 | }"} 60 | 61 | defp validate_type(_type, _key, value), do: {:ok, value} 62 | end 63 | -------------------------------------------------------------------------------- /lib/flume/pipeline/default_api.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.DefaultAPI do 2 | @behaviour Flume.Pipeline.API 3 | 4 | alias Flume.Pipeline.Event 5 | 6 | @impl Flume.Pipeline.API 7 | def pause(pipeline_name, opts), do: Event.pause(pipeline_name, opts) 8 | 9 | @impl Flume.Pipeline.API 10 | def resume(pipeline_name, opts), do: Event.resume(pipeline_name, opts) 11 | end 12 | -------------------------------------------------------------------------------- /lib/flume/pipeline/event.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event do 2 | alias Flume.{Config, Instrumentation, Pipeline} 3 | alias Flume.Pipeline.Event, as: EventPipeline 4 | alias Flume.Redis.Client, as: RedisClient 5 | 6 | def attach_instrumentation(%Pipeline{name: name, queue: queue, instrument: true} = _pipeline) do 7 | instrumentation = Config.instrumentation() 8 | name_atom = String.to_atom(name) 9 | queue_atom = String.to_atom(queue) 10 | 11 | Instrumentation.attach_many( 12 | name_atom, 13 | [ 14 | [name_atom, :worker], 15 | [name_atom, :worker, :job], 16 | [queue_atom, :enqueue], 17 | [queue_atom, :dequeue] 18 | ], 19 | fn event_name, event_value, metadata, config -> 20 | apply( 21 | instrumentation[:handler_module], 22 | instrumentation[:handler_function], 23 | [event_name, event_value, metadata, config] 24 | ) 25 | end, 26 | instrumentation[:metadata] 27 | ) 28 | end 29 | 30 | def attach_instrumentation(_), do: nil 31 | 32 | def paused_state(pipeline_name) do 33 | paused_redis_key(pipeline_name) 34 | |> RedisClient.get!() 35 | |> case do 36 | nil -> false 37 | value -> String.to_existing_atom(value) 38 | end 39 | end 40 | 41 | def pause(pipeline_name, opts) do 42 | unless opts[:temporary] do 43 | RedisClient.set(paused_redis_key(pipeline_name), true) 44 | end 45 | 46 | EventPipeline.Producer.pause(pipeline_name, opts[:async], opts[:timeout]) 47 | end 48 | 49 | def resume(pipeline_name, opts) do 50 | unless opts[:temporary] do 51 | RedisClient.del(paused_redis_key(pipeline_name)) 52 | end 53 | 54 | EventPipeline.Producer.resume(pipeline_name, opts[:async], opts[:timeout]) 55 | end 56 | 57 | def pending_workers_count(pipeline_names \\ Flume.Config.pipeline_names()) do 58 | pipeline_names 59 | |> Enum.map(fn pipeline_name -> 60 | consumer_supervisor_process_name(pipeline_name) 61 | |> EventPipeline.Consumer.workers_count() 62 | end) 63 | |> Enum.sum() 64 | end 65 | 66 | defp consumer_supervisor_process_name(pipeline_name), 67 | do: :"#{pipeline_name}_consumer_supervisor" 68 | 69 | defp paused_redis_key(pipeline_name), 70 | do: "#{Config.namespace()}:pipeline:#{pipeline_name}:paused" 71 | end 72 | -------------------------------------------------------------------------------- /lib/flume/pipeline/event/consumer.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.Consumer do 2 | @moduledoc """ 3 | A Consumer will be a consumer supervisor that 4 | will spawn event processor for each event. 5 | """ 6 | 7 | use ConsumerSupervisor 8 | 9 | alias Flume.Pipeline.Event.Worker 10 | 11 | # Client API 12 | def start_link(state \\ %{}, worker \\ Worker) do 13 | ConsumerSupervisor.start_link( 14 | __MODULE__, 15 | {state, worker}, 16 | name: process_name(state.name) 17 | ) 18 | end 19 | 20 | # Server callbacks 21 | def init({state, worker}) do 22 | instrument = Map.get(state, :instrument, false) 23 | 24 | children = [ 25 | worker(worker, [%{name: state.name, instrument: instrument}], restart: :temporary) 26 | ] 27 | 28 | upstream = upstream_process_name(state.name) 29 | 30 | { 31 | :ok, 32 | children, 33 | strategy: :one_for_one, 34 | max_restarts: 20, 35 | max_seconds: 10, 36 | subscribe_to: [{upstream, [min_demand: 0, max_demand: state.max_demand]}] 37 | } 38 | end 39 | 40 | def workers_count(process_name) do 41 | %{workers: workers_count} = ConsumerSupervisor.count_children(process_name) 42 | workers_count 43 | end 44 | 45 | defp process_name(pipeline_name) do 46 | :"#{pipeline_name}_consumer_supervisor" 47 | end 48 | 49 | defp upstream_process_name(pipeline_name) do 50 | :"#{pipeline_name}_producer_consumer" 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/flume/pipeline/event/producer.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.Producer do 2 | @moduledoc """ 3 | Polls for a batch of events from the source (Redis queue). 4 | This stage acts as a Producer in the GenStage pipeline. 5 | 6 | [**Producer**] <- ProducerConsumer <- Consumer 7 | """ 8 | use GenStage 9 | 10 | require Flume.{Instrumentation, Logger} 11 | 12 | alias Flume.{Logger, Instrumentation, Utils, Config} 13 | alias Flume.Queue.Manager, as: QueueManager 14 | alias Flume.Pipeline.Event, as: EventPipeline 15 | 16 | # 2 seconds 17 | @default_interval 2000 18 | @lock_poll_interval Config.dequeue_lock_poll_interval() 19 | 20 | # Client API 21 | def start_link(%{name: pipeline_name, queue: _queue} = state) do 22 | GenStage.start_link(__MODULE__, state, name: process_name(pipeline_name)) 23 | end 24 | 25 | def pause(pipeline_name, async, timeout \\ 5000) 26 | 27 | def pause(pipeline_name, true = _async, _timeout) do 28 | GenStage.cast(process_name(pipeline_name), :pause) 29 | end 30 | 31 | def pause(pipeline_name, false = _async, timeout) do 32 | GenStage.call(process_name(pipeline_name), :pause, timeout) 33 | end 34 | 35 | def resume(pipeline_name, async, timeout \\ 5000) 36 | 37 | def resume(pipeline_name, true = _async, _timeout) do 38 | GenStage.cast(process_name(pipeline_name), :resume) 39 | end 40 | 41 | def resume(pipeline_name, false = _async, timeout) do 42 | GenStage.call(process_name(pipeline_name), :resume, timeout) 43 | end 44 | 45 | # Server callbacks 46 | def init(%{name: name} = state) do 47 | state = 48 | state 49 | |> Map.put(:paused, EventPipeline.paused_state(name)) 50 | |> Map.put(:fetch_events_scheduled, false) 51 | |> Map.put(:demand, 0) 52 | 53 | {:producer, state} 54 | end 55 | 56 | def handle_demand(demand, state) when demand > 0 do 57 | Logger.debug("#{state.name} [Producer] handling demand of #{demand}") 58 | 59 | new_demand = state.demand + demand 60 | state = Map.put(state, :demand, new_demand) 61 | 62 | dispatch_events(state) 63 | end 64 | 65 | def handle_info(:fetch_events, state) do 66 | # This callback is invoked by the Process.send_after/3 message below. 67 | state = Map.put(state, :fetch_events_scheduled, false) 68 | dispatch_events(state) 69 | end 70 | 71 | def handle_call(:pause, _from, %{paused: true} = state) do 72 | {:reply, :ok, [], state} 73 | end 74 | 75 | def handle_call(:pause, _from, state) do 76 | state = Map.put(state, :paused, true) 77 | 78 | {:reply, :ok, [], state} 79 | end 80 | 81 | def handle_call(:resume, _from, %{paused: true} = state) do 82 | state = Map.put(state, :paused, false) 83 | 84 | {:reply, :ok, [], state} 85 | end 86 | 87 | def handle_call(:resume, _from, state) do 88 | {:reply, :ok, [], state} 89 | end 90 | 91 | def handle_cast(:pause, %{paused: true} = state) do 92 | {:noreply, [], state} 93 | end 94 | 95 | def handle_cast(:pause, state) do 96 | state = Map.put(state, :paused, true) 97 | 98 | {:noreply, [], state} 99 | end 100 | 101 | def handle_cast(:resume, %{paused: true} = state) do 102 | state = Map.put(state, :paused, false) 103 | 104 | {:noreply, [], state} 105 | end 106 | 107 | def handle_cast(:resume, state) do 108 | {:noreply, [], state} 109 | end 110 | 111 | defp dispatch_events(%{paused: true} = state) do 112 | state = schedule_fetch_events(state) 113 | 114 | {:noreply, [], state} 115 | end 116 | 117 | defp dispatch_events(%{demand: demand, batch_size: nil} = state) when demand > 0 do 118 | Logger.debug("#{state.name} [Producer] pulling #{demand} events") 119 | 120 | {duration, _} = 121 | Instrumentation.measure do 122 | fetch_result = take(demand, state) 123 | end 124 | 125 | case fetch_result do 126 | {:ok, events} -> handle_successful_fetch(events, state, duration) 127 | {:error, :locked} -> handle_locked_fetch(state) 128 | end 129 | end 130 | 131 | defp dispatch_events(%{demand: demand, batch_size: batch_size} = state) 132 | when demand > 0 and batch_size > 0 do 133 | events_to_ask = demand * batch_size 134 | 135 | Logger.debug("#{state.name} [Producer] pulling #{events_to_ask} events") 136 | 137 | {duration, _} = 138 | Instrumentation.measure do 139 | fetch_result = take(events_to_ask, state) 140 | end 141 | 142 | case fetch_result do 143 | {:ok, events} -> handle_successful_fetch(events, state, duration, batch_size) 144 | {:error, :locked} -> handle_locked_fetch(state) 145 | end 146 | end 147 | 148 | defp dispatch_events(state) do 149 | state = schedule_fetch_events(state) 150 | 151 | {:noreply, [], state} 152 | end 153 | 154 | defp handle_successful_fetch(events, state, fetch_duration, batch_size \\ 1) do 155 | count = length(events) 156 | Logger.debug("#{state.name} [Producer] pulled #{count} events from source") 157 | 158 | queue_atom = String.to_atom(state.queue) 159 | 160 | Instrumentation.execute( 161 | [queue_atom, :dequeue], 162 | %{count: count, latency: fetch_duration, payload_size: Utils.payload_size(events)}, 163 | state.instrument 164 | ) 165 | 166 | new_demand = state.demand - round(count / batch_size) 167 | state = Map.put(state, :demand, new_demand) 168 | 169 | state = schedule_fetch_events(state) 170 | 171 | {:noreply, events, state} 172 | end 173 | 174 | defp handle_locked_fetch(state) do 175 | state = schedule_fetch_events(state, @lock_poll_interval) 176 | 177 | {:noreply, [], state} 178 | end 179 | 180 | defp schedule_fetch_events(state), do: schedule_fetch_events(state, @default_interval) 181 | 182 | defp schedule_fetch_events(%{fetch_events_scheduled: true} = state, _), do: state 183 | 184 | defp schedule_fetch_events(%{demand: demand} = state, interval) 185 | when demand > 0 do 186 | # Schedule the next request 187 | Process.send_after(self(), :fetch_events, interval) 188 | Map.put(state, :fetch_events_scheduled, true) 189 | end 190 | 191 | defp schedule_fetch_events(state, _interval), do: state 192 | 193 | # For regular pipelines 194 | defp take(demand, %{rate_limit_count: nil, rate_limit_scale: nil} = state) do 195 | QueueManager.fetch_jobs(Config.namespace(), state.queue, demand) 196 | end 197 | 198 | # For rate-limited pipelines 199 | defp take( 200 | demand, 201 | %{rate_limit_count: rate_limit_count, rate_limit_scale: rate_limit_scale} = state 202 | ) do 203 | QueueManager.fetch_jobs( 204 | Config.namespace(), 205 | state.queue, 206 | demand, 207 | %{ 208 | rate_limit_count: rate_limit_count, 209 | rate_limit_scale: rate_limit_scale, 210 | rate_limit_key: Map.get(state, :rate_limit_key) 211 | } 212 | ) 213 | end 214 | 215 | def process_name(pipeline_name), do: :"#{pipeline_name}_producer" 216 | end 217 | -------------------------------------------------------------------------------- /lib/flume/pipeline/event/producer_consumer.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.ProducerConsumer do 2 | @moduledoc """ 3 | Takes a batch of events periodically to be sent to the consumers. 4 | This stage acts as a Producer-Consumer in the GenStage pipeline. 5 | 6 | Producer <- [**ProducerConsumer**] <- Consumer 7 | """ 8 | use GenStage 9 | 10 | require Flume.Logger 11 | 12 | alias Flume.{BulkEvent, Event, Logger} 13 | 14 | # Client API 15 | def start_link(%{} = pipeline) do 16 | GenStage.start_link(__MODULE__, pipeline, name: process_name(pipeline.name)) 17 | end 18 | 19 | # Server callbacks 20 | def init(state) do 21 | upstream = upstream_process_name(state.name) 22 | 23 | {:producer_consumer, state, 24 | subscribe_to: [{upstream, min_demand: 0, max_demand: state.max_demand}]} 25 | end 26 | 27 | # Process events one-by-one when batch_size is not set 28 | def handle_events(events, _from, %{batch_size: nil} = state) do 29 | Logger.debug("#{state.name} [ProducerConsumer] received #{length(events)} events") 30 | 31 | {:noreply, events, state} 32 | end 33 | 34 | # Group the events by the specified :batch_size 35 | # The consumer will receive each group as a single event 36 | # and process the group together 37 | def handle_events(events, _from, state) do 38 | Logger.debug("#{state.name} [ProducerConsumer] received #{length(events)} events") 39 | 40 | grouped_events = group_similar_events(events, state.batch_size) 41 | 42 | {:noreply, grouped_events, state} 43 | end 44 | 45 | defp process_name(pipeline_name) do 46 | :"#{pipeline_name}_producer_consumer" 47 | end 48 | 49 | defp upstream_process_name(pipeline_name), do: :"#{pipeline_name}_producer" 50 | 51 | defp group_similar_events(events, batch_size) do 52 | events 53 | |> Enum.map(&Event.decode!/1) 54 | |> Enum.group_by(& &1.class) 55 | |> Map.values() 56 | |> Enum.flat_map(fn event_group -> 57 | event_group 58 | |> Enum.chunk_every(batch_size) 59 | |> Enum.map(&BulkEvent.new/1) 60 | end) 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/flume/pipeline/event/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.Worker do 2 | @moduledoc """ 3 | Processes each event dispatched from the previous pipeline stage. 4 | This stage acts as a Consumer in the GenStage pipeline. 5 | 6 | Producer <- ProducerConsumer <- ConsumerSupervisor <- [**Consumer**] 7 | """ 8 | 9 | require Flume.{Instrumentation, Logger} 10 | 11 | alias Flume.{BulkEvent, Event, Instrumentation, Logger} 12 | alias Flume.Pipeline.BulkEvent, as: BulkEventPipeline 13 | alias Flume.Pipeline.SystemEvent, as: SystemEventPipeline 14 | alias Flume.Pipeline.Context, as: WorkerContext 15 | 16 | # Client API 17 | def start_link(pipeline, %BulkEvent{} = bulk_event) do 18 | Task.start_link(BulkEventPipeline.Worker, :process, [pipeline, bulk_event]) 19 | end 20 | 21 | def start_link(pipeline, event) do 22 | Task.start_link(__MODULE__, :process, [pipeline, event]) 23 | end 24 | 25 | def process(%{name: pipeline_name} = pipeline, event) do 26 | {duration, %Event{class: class}} = 27 | Instrumentation.measure do 28 | Logger.debug("#{pipeline_name} [Consumer] received event - #{inspect(event)}") 29 | 30 | event = Event.decode!(event) 31 | WorkerContext.put(event.context) 32 | 33 | do_process_event(pipeline, event) 34 | event 35 | end 36 | 37 | Instrumentation.execute( 38 | [String.to_atom(pipeline_name), :worker], 39 | %{duration: duration}, 40 | %{module: Instrumentation.format_module(class)}, 41 | pipeline[:instrument] 42 | ) 43 | rescue 44 | e in [Jason.DecodeError, ArgumentError] -> 45 | Logger.error("#{pipeline.name} [Consumer] failed while parsing event: #{Kernel.inspect(e)}") 46 | end 47 | 48 | defp do_process_event( 49 | %{name: pipeline_name} = pipeline, 50 | %Event{ 51 | function: function, 52 | class: class, 53 | args: args, 54 | jid: jid 55 | } = event 56 | ) do 57 | {duration, _} = 58 | Instrumentation.measure do 59 | apply_function(%{class: class, function_name: function}, args) 60 | end 61 | 62 | Instrumentation.execute( 63 | [String.to_atom(pipeline_name), :worker, :job], 64 | %{duration: duration}, 65 | %{module: Instrumentation.format_module(class)}, 66 | pipeline[:instrument] 67 | ) 68 | 69 | Logger.debug("#{pipeline_name} [Consumer] processed event: #{class} - #{jid}") 70 | 71 | SystemEventPipeline.enqueue({:success, event}) 72 | rescue 73 | e in _ -> 74 | error_message = Kernel.inspect(e) 75 | handle_failure(pipeline_name, event, error_message) 76 | catch 77 | :exit, {:timeout, message} -> 78 | handle_failure(pipeline_name, event, inspect(message)) 79 | end 80 | 81 | defp apply_function(%{class: class, function_name: function_name}, args) do 82 | function_name = String.to_atom(function_name) 83 | 84 | [class] 85 | |> Module.safe_concat() 86 | |> apply(function_name, args) 87 | end 88 | 89 | defp handle_failure( 90 | pipeline_name, 91 | %Event{class: class, function: function, args: args, retry_count: retry_count} = event, 92 | error_message 93 | ) do 94 | Logger.error("#{pipeline_name} [Consumer] failed with error: #{error_message}", %{ 95 | worker_name: class, 96 | function: function, 97 | args: args, 98 | retry_count: retry_count 99 | }) 100 | 101 | SystemEventPipeline.enqueue({:failed, event, error_message}) 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /lib/flume/pipeline/mock_api.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.MockAPI do 2 | @behaviour Flume.Pipeline.API 3 | 4 | @impl Flume.Pipeline.API 5 | def pause(pipeline_name, options) do 6 | send(self(), %{pipeline_name: pipeline_name, action: :pause, options: options}) 7 | end 8 | 9 | @impl Flume.Pipeline.API 10 | def resume(pipeline_name, options) do 11 | send(self(), %{pipeline_name: pipeline_name, action: :resume, options: options}) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/flume/pipeline/system_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.SystemEvent do 2 | alias Flume.Pipeline.SystemEvent 3 | 4 | defdelegate enqueue(event_or_events), to: SystemEvent.Producer 5 | 6 | def pending_workers_count do 7 | Supervisor.which_children(SystemEvent.Supervisor) 8 | |> Enum.filter(fn {module, _, _, _} -> module == SystemEvent.Consumer end) 9 | |> Enum.reduce(0, fn {_, pid, _, _}, acc -> 10 | acc + SystemEvent.Consumer.workers_count(pid) 11 | end) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/flume/pipeline/system_event/consumer.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.SystemEvent.Consumer do 2 | @moduledoc """ 3 | A consumer will be consumer supervisor that will 4 | spawn Worker tasks for each event. 5 | """ 6 | 7 | use ConsumerSupervisor 8 | 9 | alias Flume.Pipeline.SystemEvent.{Producer, Worker} 10 | 11 | def start_link do 12 | ConsumerSupervisor.start_link(__MODULE__, :ok) 13 | end 14 | 15 | # Callbacks 16 | 17 | def init(:ok) do 18 | children = [ 19 | worker(Worker, [], restart: :temporary) 20 | ] 21 | 22 | { 23 | :ok, 24 | children, 25 | strategy: :one_for_one, subscribe_to: [{Producer, max_demand: 1000}] 26 | } 27 | end 28 | 29 | def workers_count(process_name) do 30 | %{workers: workers_count} = ConsumerSupervisor.count_children(process_name) 31 | workers_count 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/flume/pipeline/system_event/producer.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.SystemEvent.Producer do 2 | @moduledoc """ 3 | A producer will accept new events and these events will be 4 | dispatched to consumers. 5 | """ 6 | 7 | use GenStage 8 | 9 | def start_link(initial) do 10 | GenStage.start_link(__MODULE__, initial, name: __MODULE__) 11 | end 12 | 13 | def enqueue(events) when is_list(events) do 14 | GenStage.cast(__MODULE__, {:enqueue, events}) 15 | end 16 | 17 | def enqueue(event), do: enqueue([event]) 18 | 19 | def init(initial) do 20 | {:producer, initial} 21 | end 22 | 23 | def handle_demand(demand, counter) when demand > 0 do 24 | {:noreply, [], counter} 25 | end 26 | 27 | def handle_cast({:enqueue, events}, counter) do 28 | events_count = length(events) 29 | 30 | {:noreply, events, counter + events_count} 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/flume/pipeline/system_event/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.SystemEvent.Supervisor do 2 | use Supervisor 3 | 4 | alias Flume.Pipeline.SystemEvent 5 | 6 | def start_link do 7 | children = [ 8 | worker(SystemEvent.Producer, [0]), 9 | worker(SystemEvent.Consumer, []) 10 | ] 11 | 12 | opts = [ 13 | strategy: :one_for_one, 14 | max_restarts: 20, 15 | max_seconds: 10, 16 | name: __MODULE__ 17 | ] 18 | 19 | Supervisor.start_link(children, opts) 20 | end 21 | 22 | def init(opts) do 23 | {:ok, opts} 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/flume/pipeline/system_event/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.SystemEvent.Worker do 2 | @moduledoc """ 3 | A worker will spawn a task for each event. 4 | """ 5 | use Retry 6 | 7 | alias Flume.{Config, Event} 8 | alias Flume.Queue.Manager, as: QueueManager 9 | alias Flume.Pipeline.SystemEvent 10 | 11 | # In milliseconds 12 | @retry_expiry_timeout 10_000 13 | 14 | def start_link({:success, %Event{} = event}) do 15 | Task.start_link(__MODULE__, :success, [event]) 16 | end 17 | 18 | def start_link({:failed, %Event{} = event, exception_message}) do 19 | Task.start_link(__MODULE__, :fail, [event, exception_message]) 20 | end 21 | 22 | def success(event) do 23 | retry with: exponential_backoff() |> randomize() |> expiry(@retry_expiry_timeout) do 24 | QueueManager.remove_processing(Config.namespace(), event.queue, event.original_json) 25 | after 26 | result -> result 27 | else 28 | _ -> SystemEvent.Producer.enqueue({:success, event}) 29 | end 30 | end 31 | 32 | def fail(event, error_message) do 33 | retry with: exponential_backoff() |> randomize() |> expiry(@retry_expiry_timeout) do 34 | QueueManager.retry_or_fail_job( 35 | Config.namespace(), 36 | event.queue, 37 | event.original_json, 38 | error_message 39 | ) 40 | after 41 | result -> result 42 | else 43 | _ -> SystemEvent.Producer.enqueue({:failed, event, error_message}) 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/flume/queue/api.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.API do 2 | @callback bulk_enqueue(String.t(), [any()], [any()]) :: {:ok, term} | {:error, String.t()} 3 | 4 | @callback enqueue(String.t(), Atom.t(), Atom.t(), [any()], [any()]) :: 5 | {:ok, term} | {:error, String.t()} 6 | 7 | @callback enqueue_in(String.t(), integer, Atom.t(), Atom.t(), [any()], [any()]) :: 8 | {:ok, term} | {:error, String.t()} 9 | end 10 | -------------------------------------------------------------------------------- /lib/flume/queue/backoff.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.Backoff do 2 | @backoff_exponent 1.5 3 | 4 | alias Flume.Config 5 | 6 | def calc_next_backoff(count) do 7 | backoff_current = Config.backoff_initial() * count 8 | backoff_max = Config.backoff_max() 9 | next_exponential_backoff = round(backoff_current * @backoff_exponent) 10 | 11 | if backoff_max == :infinity do 12 | next_exponential_backoff 13 | else 14 | min(next_exponential_backoff, backoff_max) 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/flume/queue/default_api.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.DefaultAPI do 2 | @behaviour Flume.Queue.API 3 | 4 | alias Flume.Config 5 | alias Flume.Queue.Manager 6 | 7 | def bulk_enqueue(queue, jobs, opts \\ []) do 8 | Manager.bulk_enqueue(namespace(), queue, jobs, opts) 9 | end 10 | 11 | def enqueue( 12 | queue, 13 | worker, 14 | function_name \\ :perform, 15 | args, 16 | opts \\ [] 17 | ) do 18 | Manager.enqueue( 19 | namespace(), 20 | queue, 21 | worker, 22 | function_name, 23 | args, 24 | opts 25 | ) 26 | end 27 | 28 | def enqueue_in( 29 | queue, 30 | time_in_seconds, 31 | worker, 32 | function_name \\ :perform, 33 | args, 34 | opts \\ [] 35 | ) do 36 | Manager.enqueue_in( 37 | namespace(), 38 | queue, 39 | time_in_seconds, 40 | worker, 41 | function_name, 42 | args, 43 | opts 44 | ) 45 | end 46 | 47 | defp namespace, do: Config.namespace() 48 | end 49 | -------------------------------------------------------------------------------- /lib/flume/queue/manager.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.Manager do 2 | require Flume.Logger 3 | 4 | alias Flume.{Config, Event, Logger, Instrumentation, Utils, Redis} 5 | alias Flume.Redis.Job 6 | alias Flume.Queue.Backoff 7 | alias Flume.Support.Time, as: TimeExtension 8 | 9 | @external_resource "priv/scripts/enqueue_processing_jobs.lua" 10 | @external_resource "priv/scripts/release_lock.lua" 11 | 12 | def enqueue( 13 | namespace, 14 | queue, 15 | worker, 16 | function_name, 17 | args, 18 | opts \\ [] 19 | ) do 20 | job = serialized_job(queue, worker, function_name, args, opts[:context]) 21 | queue_atom = if is_atom(queue), do: queue, else: String.to_atom(queue) 22 | 23 | Instrumentation.execute( 24 | [queue_atom, :enqueue], 25 | %{payload_size: byte_size(job)}, 26 | true 27 | ) 28 | 29 | Job.enqueue(queue_key(namespace, queue), job) 30 | end 31 | 32 | def bulk_enqueue(namespace, queue, jobs, opts \\ []) do 33 | jobs = 34 | jobs 35 | |> Enum.map(fn 36 | [worker, function_name, args] -> 37 | serialized_job(queue, worker, function_name, args, opts[:context]) 38 | 39 | [worker, args] -> 40 | serialized_job(queue, worker, :perform, args, opts[:context]) 41 | end) 42 | 43 | queue_atom = if is_atom(queue), do: queue, else: String.to_atom(queue) 44 | 45 | Instrumentation.execute( 46 | [queue_atom, :enqueue], 47 | %{payload_size: Utils.payload_size(jobs)}, 48 | true 49 | ) 50 | 51 | Job.bulk_enqueue(queue_key(namespace, queue), jobs) 52 | end 53 | 54 | def enqueue_in( 55 | namespace, 56 | queue, 57 | time_in_seconds, 58 | worker, 59 | function_name, 60 | args, 61 | opts \\ [] 62 | ) do 63 | queue_name = scheduled_key(namespace) 64 | job = serialized_job(queue, worker, function_name, args, opts[:context]) 65 | 66 | schedule_job_at(queue_name, time_in_seconds, job) 67 | end 68 | 69 | def job_counts(namespace, [_queue | _] = queues) do 70 | queues 71 | |> Enum.map(&(fully_qualified_queue_name(namespace, &1) |> Redis.Command.llen())) 72 | |> Redis.Client.pipeline() 73 | |> case do 74 | {:ok, counts} -> 75 | {:ok, Enum.zip(queues, counts)} 76 | 77 | {:error, reason} -> 78 | Logger.error("Error in getting job counts #{inspect(reason)}") 79 | {:error, reason} 80 | end 81 | end 82 | 83 | def fetch_jobs( 84 | namespace, 85 | queue, 86 | count, 87 | %{rate_limit_count: rate_limit_count, rate_limit_scale: rate_limit_scale} = 88 | rate_limit_opts 89 | ) do 90 | {current_score, previous_score} = current_and_previous_score(rate_limit_scale) 91 | 92 | Job.bulk_dequeue( 93 | queue_key(namespace, queue), 94 | processing_key(namespace, queue), 95 | rate_limit_key(namespace, queue, rate_limit_opts[:rate_limit_key]), 96 | count, 97 | rate_limit_count, 98 | previous_score, 99 | current_score 100 | ) 101 | end 102 | 103 | def fetch_jobs(namespace, queue, count) do 104 | Job.bulk_dequeue( 105 | queue_key(namespace, queue), 106 | processing_key(namespace, queue), 107 | count, 108 | TimeExtension.time_to_score() 109 | ) 110 | end 111 | 112 | def enqueue_processing_jobs(namespace, utc_time, queue, limit) do 113 | Job.enqueue_processing_jobs( 114 | processing_key(namespace, queue), 115 | queue_key(namespace, queue), 116 | TimeExtension.time_to_score(utc_time), 117 | limit 118 | ) 119 | end 120 | 121 | def retry_or_fail_job(namespace, queue, serialized_job, error) do 122 | deserialized_job = Event.decode!(serialized_job) 123 | retry_count = deserialized_job.retry_count || 0 124 | 125 | response = 126 | if retry_count < Config.max_retries() do 127 | retry_job(namespace, deserialized_job, error, retry_count + 1) 128 | else 129 | Logger.info("Max retires on job #{deserialized_job.jid} exceeded") 130 | fail_job(namespace, deserialized_job, error) 131 | end 132 | 133 | case response do 134 | {:ok, _} -> 135 | remove_retry(namespace, deserialized_job.original_json) 136 | remove_processing(namespace, queue, deserialized_job.original_json) 137 | 138 | {:error, _} -> 139 | Logger.info("Failed to move job to a retry or dead queue.") 140 | end 141 | end 142 | 143 | def retry_job(namespace, deserialized_job, error, count) do 144 | job = %{ 145 | deserialized_job 146 | | retry_count: count, 147 | failed_at: TimeExtension.unix_seconds(), 148 | error_message: error 149 | } 150 | 151 | retry_at = next_time_to_retry(count) 152 | schedule_job_at(retry_key(namespace), retry_at, Jason.encode!(job)) 153 | end 154 | 155 | def fail_job(namespace, job, error) do 156 | job = %{ 157 | job 158 | | retry_count: job.retry_count || 0, 159 | failed_at: TimeExtension.unix_seconds(), 160 | error_message: error 161 | } 162 | 163 | Job.fail_job!(dead_key(namespace), Jason.encode!(job)) 164 | {:ok, nil} 165 | rescue 166 | e in [Redix.Error, Redix.ConnectionError] -> 167 | Logger.error("[#{dead_key(namespace)}] Job: #{job} failed with error: #{Kernel.inspect(e)}") 168 | {:error, e.reason} 169 | end 170 | 171 | def remove_retry(namespace, job) do 172 | queue_key = retry_key(namespace) 173 | count = Job.remove_scheduled_job!(queue_key, job) 174 | {:ok, count} 175 | rescue 176 | e in [Redix.Error, Redix.ConnectionError] -> 177 | Logger.error( 178 | "[#{retry_key(namespace)}] Job: #{job} failed with error: #{Kernel.inspect(e)}" 179 | ) 180 | 181 | {:error, e.message} 182 | end 183 | 184 | def remove_processing(namespace, queue, job) do 185 | processing_queue_key = processing_key(namespace, queue) 186 | count = Job.remove_processing!(processing_queue_key, job) 187 | {:ok, count} 188 | rescue 189 | e in [Redix.Error, Redix.ConnectionError] -> 190 | Logger.error("[#{queue}] Job: #{job} failed with error: #{Kernel.inspect(e)}") 191 | 192 | {:error, e.reason} 193 | end 194 | 195 | @doc """ 196 | Retrieves all the scheduled and retry jobs from the redis sorted set 197 | based on the queue name and max score and enqueues them into the main 198 | queue which will be processed. 199 | ## Examples 200 | iex> Flume.Queue.Manager.remove_and_enqueue_scheduled_jobs('flume_test', "1515224298.912696") 201 | {:ok, 0} 202 | """ 203 | # TODO: Refactor this! 204 | # This function and other functions related to this are complex because 205 | # we are trying to mix the processing of `scheduled` and `retry` jobs together. 206 | # This can be simplified by having two scheduler processes for each. 207 | def remove_and_enqueue_scheduled_jobs(namespace, max_score) do 208 | scheduled_keys(namespace) 209 | |> Job.scheduled_jobs(max_score) 210 | |> case do 211 | {:error, error_message} -> 212 | {:error, error_message} 213 | 214 | {:ok, scheduled_queues_and_jobs} -> 215 | if Enum.all?(scheduled_queues_and_jobs, fn {_, jobs} -> Enum.empty?(jobs) end) do 216 | {:ok, 0} 217 | else 218 | enqueued_jobs = enqueue_scheduled_jobs(namespace, scheduled_queues_and_jobs) 219 | count = Job.bulk_remove_scheduled!(enqueued_jobs) |> Enum.count() 220 | {:ok, count} 221 | end 222 | end 223 | end 224 | 225 | def enqueue_scheduled_jobs(namespace, scheduled_queues_and_jobs) do 226 | queues_and_jobs = 227 | scheduled_queues_and_jobs 228 | |> Enum.flat_map(fn {scheduled_queue, jobs} -> 229 | Enum.map(jobs, fn job -> 230 | deserialized_job = Event.decode!(job) 231 | {scheduled_queue, queue_key(namespace, deserialized_job.queue), job} 232 | end) 233 | end) 234 | 235 | Job.bulk_enqueue_scheduled!(queues_and_jobs) 236 | end 237 | 238 | defp schedule_job_at(queue, retry_at, job) do 239 | Job.schedule_job(queue, retry_at, job) 240 | end 241 | 242 | defp serialized_job(queue, worker, function_name, args, context) do 243 | %Event{ 244 | queue: queue, 245 | class: worker, 246 | function: function_name, 247 | jid: UUID.uuid4(), 248 | args: args, 249 | enqueued_at: TimeExtension.unix_seconds(), 250 | retry_count: 0 251 | } 252 | |> add_context_to_event(context) 253 | |> Jason.encode!() 254 | end 255 | 256 | defp add_context_to_event(event, nil), do: event 257 | defp add_context_to_event(event, context) when context == %{}, do: event 258 | defp add_context_to_event(event, context), do: Map.put(event, :context, context) 259 | 260 | defp next_time_to_retry(retry_count) do 261 | retry_count 262 | |> Backoff.calc_next_backoff() 263 | |> TimeExtension.offset_from_now() 264 | |> TimeExtension.unix_seconds() 265 | end 266 | 267 | defp full_key(namespace, key), do: "#{namespace}:#{key}" 268 | 269 | defp queue_key(namespace, queue), do: full_key(namespace, "queue:#{queue}") 270 | 271 | defp retry_key(namespace), do: full_key(namespace, "retry") 272 | 273 | defp dead_key(namespace), do: full_key(namespace, "dead") 274 | 275 | defp scheduled_key(namespace), do: full_key(namespace, "scheduled") 276 | 277 | defp scheduled_keys(namespace) do 278 | [scheduled_key(namespace), retry_key(namespace)] 279 | end 280 | 281 | defp fully_qualified_queue_name(namespace, queue_name), do: "#{namespace}:queue:#{queue_name}" 282 | 283 | def processing_key(namespace, queue), do: full_key(namespace, "queue:processing:#{queue}") 284 | 285 | def rate_limit_key(namespace, queue, nil), do: full_key(namespace, "queue:limit:#{queue}") 286 | 287 | def rate_limit_key(namespace, _queue, key), do: full_key(namespace, "limit:#{key}") 288 | 289 | defp current_and_previous_score(offset_in_ms) do 290 | current_time = DateTime.utc_now() 291 | current_score = TimeExtension.unix_seconds(current_time) 292 | 293 | previous_score = 294 | TimeExtension.offset_before(offset_in_ms, current_time) 295 | |> TimeExtension.time_to_score() 296 | 297 | {current_score, previous_score} 298 | end 299 | end 300 | -------------------------------------------------------------------------------- /lib/flume/queue/mock_api.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.MockAPI do 2 | @behaviour Flume.Queue.API 3 | 4 | def bulk_enqueue(queue, jobs, opts \\ []) 5 | 6 | def bulk_enqueue(queue, jobs, []) do 7 | message = %{queue: queue, jobs: jobs} 8 | 9 | send(self(), message) 10 | 11 | {:ok, message} 12 | end 13 | 14 | def bulk_enqueue(queue, jobs, opts) do 15 | send(self(), %{queue: queue, jobs: jobs, options: opts}) 16 | end 17 | 18 | def enqueue( 19 | queue, 20 | worker, 21 | function_name \\ :perform, 22 | args, 23 | opts \\ [] 24 | ) 25 | 26 | def enqueue( 27 | queue, 28 | worker, 29 | function_name, 30 | args, 31 | [] 32 | ) do 33 | message = %{queue: queue, worker: worker, function_name: function_name, args: args} 34 | 35 | send(self(), message) 36 | 37 | {:ok, message} 38 | end 39 | 40 | def enqueue( 41 | queue, 42 | worker, 43 | function_name, 44 | args, 45 | opts 46 | ) do 47 | message = %{ 48 | queue: queue, 49 | worker: worker, 50 | function_name: function_name, 51 | args: args, 52 | options: opts 53 | } 54 | 55 | send(self(), message) 56 | 57 | {:ok, message} 58 | end 59 | 60 | def enqueue_in( 61 | queue, 62 | time_in_seconds, 63 | worker, 64 | function_name \\ :perform, 65 | args, 66 | opts \\ [] 67 | ) 68 | 69 | def enqueue_in( 70 | queue, 71 | time_in_seconds, 72 | worker, 73 | function_name, 74 | args, 75 | [] 76 | ) do 77 | message = %{ 78 | schedule_in: time_in_seconds, 79 | queue: queue, 80 | worker: worker, 81 | function_name: function_name, 82 | args: args 83 | } 84 | 85 | send(self(), message) 86 | 87 | {:ok, message} 88 | end 89 | 90 | def enqueue_in( 91 | queue, 92 | time_in_seconds, 93 | worker, 94 | function_name, 95 | args, 96 | opts 97 | ) do 98 | message = %{ 99 | schedule_in: time_in_seconds, 100 | queue: queue, 101 | worker: worker, 102 | function_name: function_name, 103 | args: args, 104 | options: opts 105 | } 106 | 107 | send(self(), message) 108 | 109 | {:ok, message} 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /lib/flume/queue/processing_scheduler.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.ProcessingScheduler do 2 | require Flume.Logger 3 | 4 | use GenServer 5 | 6 | alias Flume.{Config, Logger} 7 | alias Flume.Queue.Manager 8 | 9 | @max_limit 1000 10 | 11 | defmodule State do 12 | defstruct namespace: nil, scheduler_poll_interval: nil, queue: nil 13 | end 14 | 15 | def start_link(opts \\ []) do 16 | GenServer.start_link(__MODULE__, opts) 17 | end 18 | 19 | def init(opts \\ []) do 20 | state = struct(State, opts) 21 | schedule_work(state.scheduler_poll_interval) 22 | 23 | {:ok, state} 24 | end 25 | 26 | def handle_info(:schedule_work, state) do 27 | work(state) 28 | schedule_work(state.scheduler_poll_interval) 29 | 30 | {:noreply, state} 31 | end 32 | 33 | def handle_info(msg, state) do 34 | Logger.warn("#{__MODULE__}: Unknown message - #{inspect(msg)}") 35 | 36 | {:noreply, state} 37 | end 38 | 39 | defp work(state) do 40 | Manager.enqueue_processing_jobs( 41 | state.namespace, 42 | time_before_visibility_timeout(), 43 | state.queue, 44 | @max_limit 45 | ) 46 | |> case do 47 | {:ok, 0} -> 48 | Logger.debug("#{__MODULE__}: No processing jobs to enqueue") 49 | 50 | {:ok, count} -> 51 | Logger.debug("#{__MODULE__}: Enqueue #{count} jobs from processing queue") 52 | 53 | {:error, error_message} -> 54 | Logger.error("#{__MODULE__}: Failed to enqueue jobs - #{Kernel.inspect(error_message)}") 55 | end 56 | end 57 | 58 | defp schedule_work(scheduler_poll_interval) do 59 | Process.send_after(self(), :schedule_work, scheduler_poll_interval) 60 | end 61 | 62 | defp time_before_visibility_timeout do 63 | DateTime.utc_now() 64 | |> DateTime.to_unix() 65 | |> Kernel.-(Config.visibility_timeout()) 66 | |> DateTime.from_unix!() 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /lib/flume/queue/scheduler.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.Scheduler do 2 | require Flume.Logger 3 | 4 | use GenServer 5 | 6 | alias Flume.Logger 7 | alias Flume.Queue.Manager 8 | alias Flume.Support.Time 9 | 10 | defmodule State do 11 | defstruct namespace: nil, scheduler_poll_interval: nil 12 | end 13 | 14 | def start_link(opts \\ []) do 15 | GenServer.start_link(__MODULE__, opts) 16 | end 17 | 18 | def init(opts \\ []) do 19 | state = struct(State, opts) 20 | schedule_work(state) 21 | 22 | {:ok, state} 23 | end 24 | 25 | def handle_info(:schedule_work, state) do 26 | work(state) 27 | schedule_work(state) 28 | 29 | {:noreply, state} 30 | end 31 | 32 | def handle_info(msg, state) do 33 | Logger.warn("#{__MODULE__}: Unknown message - #{inspect(msg)}") 34 | 35 | {:noreply, state} 36 | end 37 | 38 | defp work(state) do 39 | Manager.remove_and_enqueue_scheduled_jobs( 40 | state.namespace, 41 | Time.time_to_score() 42 | ) 43 | |> case do 44 | {:ok, 0} -> 45 | Logger.debug("#{__MODULE__}: Waiting for new jobs") 46 | 47 | {:ok, count} -> 48 | Logger.debug("#{__MODULE__}: Processed #{count} jobs") 49 | 50 | {:error, error_message} -> 51 | Logger.error("#{__MODULE__}: Failed to fetch jobs - #{Kernel.inspect(error_message)}") 52 | end 53 | end 54 | 55 | defp schedule_work(state) do 56 | Process.send_after(self(), :schedule_work, state.scheduler_poll_interval) 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /lib/flume/redis/bulk_dequeue.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.BulkDequeue do 2 | require Flume.Logger 3 | 4 | alias Flume.Redis.{Client, Lock} 5 | alias Flume.{Config, Utils, Logger} 6 | 7 | @dequeue_lock_suffix "bulk_dequeue_lock" 8 | @dequeue_lock_ttl Config.dequeue_lock_ttl() 9 | # This should always be much lower than dequeue_lock_ttl 10 | @dequeue_process_timeout Config.dequeue_process_timeout() 11 | 12 | def exec( 13 | dequeue_key, 14 | processing_sorted_set_key, 15 | count, 16 | current_score 17 | ) do 18 | lock_key = bulk_dequeue_lock_key(dequeue_key) 19 | 20 | case acquire_lock(lock_key) do 21 | {:ok, lock_token} -> 22 | exec( 23 | dequeue_key, 24 | processing_sorted_set_key, 25 | count, 26 | current_score, 27 | lock_token 28 | ) 29 | 30 | {:error, error} -> 31 | Logger.debug("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(error)}") 32 | {:ok, []} 33 | end 34 | end 35 | 36 | def exec( 37 | dequeue_key, 38 | processing_sorted_set_key, 39 | count, 40 | current_score, 41 | lock_token 42 | ) do 43 | result = 44 | Utils.safe_apply( 45 | fn -> 46 | lrange(count, dequeue_key) 47 | |> dequeue( 48 | dequeue_key, 49 | processing_sorted_set_key, 50 | current_score 51 | ) 52 | end, 53 | @dequeue_process_timeout 54 | ) 55 | 56 | case bulk_dequeue_lock_key(dequeue_key) 57 | |> release_lock(lock_token) do 58 | {:error, reason} -> 59 | Logger.error("[Lock release] #{dequeue_key}: error: #{inspect(reason)}") 60 | 61 | :ok -> 62 | :ok 63 | end 64 | 65 | case result do 66 | {:ok, result} -> 67 | result 68 | 69 | {:exit, {error, _stack}} -> 70 | Logger.error("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(error)}") 71 | {:ok, []} 72 | 73 | {:timeout, reason} -> 74 | Logger.error("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(reason)}") 75 | {:ok, []} 76 | end 77 | end 78 | 79 | def exec_rate_limited( 80 | dequeue_key, 81 | processing_sorted_set_key, 82 | limit_sorted_set_key, 83 | count, 84 | max_count, 85 | previous_score, 86 | current_score 87 | ) do 88 | lock_key = bulk_dequeue_lock_key(dequeue_key) 89 | 90 | case acquire_lock(lock_key) do 91 | {:ok, lock_token} -> 92 | exec_rate_limited( 93 | dequeue_key, 94 | processing_sorted_set_key, 95 | limit_sorted_set_key, 96 | count, 97 | max_count, 98 | previous_score, 99 | current_score, 100 | lock_token 101 | ) 102 | 103 | {:error, error} -> 104 | Logger.debug("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(error)}") 105 | {:ok, []} 106 | end 107 | end 108 | 109 | def exec_rate_limited( 110 | dequeue_key, 111 | processing_sorted_set_key, 112 | limit_sorted_set_key, 113 | count, 114 | max_count, 115 | previous_score, 116 | current_score, 117 | lock_token 118 | ) do 119 | result = 120 | Utils.safe_apply( 121 | fn -> 122 | clean_up_rate_limiting(limit_sorted_set_key, previous_score) 123 | 124 | lrange( 125 | dequeue_key, 126 | count, 127 | max_count, 128 | limit_sorted_set_key, 129 | previous_score, 130 | current_score 131 | ) 132 | |> dequeue( 133 | dequeue_key, 134 | processing_sorted_set_key, 135 | limit_sorted_set_key, 136 | current_score 137 | ) 138 | end, 139 | @dequeue_process_timeout 140 | ) 141 | 142 | case bulk_dequeue_lock_key(dequeue_key) 143 | |> release_lock(lock_token) do 144 | {:error, reason} -> 145 | Logger.error("[Lock release] #{dequeue_key}: error: #{inspect(reason)}") 146 | 147 | :ok -> 148 | :ok 149 | end 150 | 151 | case result do 152 | {:ok, result} -> 153 | result 154 | 155 | {:exit, reason} -> 156 | Logger.error("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(reason)}") 157 | {:ok, []} 158 | 159 | {:timeout, reason} -> 160 | Logger.error("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(reason)}") 161 | {:ok, []} 162 | end 163 | end 164 | 165 | defp lrange( 166 | dequeue_key, 167 | count, 168 | max_count, 169 | limit_sorted_set_key, 170 | previous_score, 171 | current_score 172 | ) do 173 | fetch_count( 174 | count, 175 | max_count, 176 | limit_sorted_set_key, 177 | previous_score, 178 | current_score 179 | ) 180 | |> lrange(dequeue_key) 181 | end 182 | 183 | defp lrange(count, dequeue_key) when count > 0 do 184 | case Client.lrange(dequeue_key, 0, count - 1) do 185 | {:ok, res} -> 186 | res 187 | 188 | {:error, reason} -> 189 | Logger.error("[Dequeue] #{dequeue_key}:#{count} error: #{inspect(reason)}") 190 | [] 191 | end 192 | end 193 | 194 | defp lrange(_count, _dequeue_key), do: [] 195 | 196 | defp fetch_count( 197 | count, 198 | max_count, 199 | limit_sorted_set_key, 200 | previous_score, 201 | _current_score 202 | ) do 203 | processed_count = 204 | case Client.zcount(limit_sorted_set_key, previous_score) do 205 | {:ok, count} -> 206 | count 207 | 208 | {:error, reason} -> 209 | Logger.error("[Dequeue] #{limit_sorted_set_key}:#{count} error: #{inspect(reason)}") 210 | :infinity 211 | end 212 | 213 | if processed_count < max_count do 214 | remaining_count = max_count - processed_count 215 | adjust_fetch_count(count, remaining_count) 216 | else 217 | 0 218 | end 219 | end 220 | 221 | defp adjust_fetch_count(count, remaining_count) when remaining_count < count, 222 | do: remaining_count 223 | 224 | defp adjust_fetch_count(count, _remaining_count), do: count 225 | 226 | defp dequeue( 227 | [], 228 | _dequeue_key, 229 | _processing_sorted_set_key, 230 | _limit_sorted_set_key, 231 | _current_score 232 | ), 233 | do: {:ok, []} 234 | 235 | defp dequeue( 236 | jobs, 237 | dequeue_key, 238 | processing_sorted_set_key, 239 | limit_sorted_set_key, 240 | current_score 241 | ) do 242 | jobs_with_score = Enum.flat_map(jobs, fn job -> [current_score, job] end) 243 | trimmed_jobs_with_score = Enum.flat_map(jobs, fn job -> [current_score, checksum(job)] end) 244 | 245 | ltrim_command = Client.ltrim_command(dequeue_key, length(jobs), -1) 246 | zadd_processing_command = Client.bulk_zadd_command(processing_sorted_set_key, jobs_with_score) 247 | zadd_limit_command = Client.bulk_zadd_command(limit_sorted_set_key, trimmed_jobs_with_score) 248 | 249 | job_length = length(jobs) 250 | expected_response = {:ok, [job_length, "OK", job_length]} 251 | 252 | case Client.transaction_pipeline([ 253 | zadd_processing_command, 254 | ltrim_command, 255 | zadd_limit_command 256 | ]) do 257 | ^expected_response -> 258 | {:ok, jobs} 259 | 260 | error_res -> 261 | Logger.info( 262 | "[Dequeue]: #{dequeue_key} error: Expected: #{inspect(expected_response)}, Got: #{ 263 | inspect(error_res) 264 | }, queue_name: #{dequeue_key}" 265 | ) 266 | 267 | {:ok, []} 268 | end 269 | end 270 | 271 | defp checksum(job), do: :crypto.hash(:md5, job) |> Base.encode16() 272 | 273 | defp dequeue([], _dequeue_key, _processing_sorted_set_key, _current_score), do: {:ok, []} 274 | 275 | defp dequeue(jobs, dequeue_key, processing_sorted_set_key, current_score) do 276 | jobs_with_score = Enum.flat_map(jobs, fn job -> [current_score, job] end) 277 | 278 | ltrim_command = Client.ltrim_command(dequeue_key, length(jobs), -1) 279 | zadd_processing_command = Client.bulk_zadd_command(processing_sorted_set_key, jobs_with_score) 280 | 281 | expected_response = {:ok, [length(jobs), "OK"]} 282 | 283 | case Client.transaction_pipeline([ 284 | zadd_processing_command, 285 | ltrim_command 286 | ]) do 287 | ^expected_response -> 288 | {:ok, jobs} 289 | 290 | error_res -> 291 | Logger.info( 292 | "[Dequeue]: error: Expected: #{inspect(expected_response)}, Got: #{inspect(error_res)}, queue_name: #{ 293 | dequeue_key 294 | }" 295 | ) 296 | 297 | {:ok, []} 298 | end 299 | end 300 | 301 | defp clean_up_rate_limiting(key, previous_score) do 302 | Client.zremrangebyscore(key, "-inf", previous_score) 303 | end 304 | 305 | defp acquire_lock(lock_key), 306 | do: Lock.acquire(lock_key, @dequeue_lock_ttl) 307 | 308 | defdelegate release_lock(lock_key, token), to: Lock, as: :release 309 | 310 | defp bulk_dequeue_lock_key(dequeue_key) do 311 | "#{dequeue_key}:#{@dequeue_lock_suffix}" 312 | end 313 | end 314 | -------------------------------------------------------------------------------- /lib/flume/redis/client.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.Client do 2 | require Flume.Logger 3 | 4 | alias Flume.{Config, Logger} 5 | alias Flume.Redis.Command 6 | 7 | # Redis commands 8 | @decr "DECR" 9 | @decrby "DECRBY" 10 | @del "DEL" 11 | @evalsha "EVALSHA" 12 | @eval "EVAL" 13 | @get "GET" 14 | @hgetall "HGETALL" 15 | @incr "INCR" 16 | @incrby "INCRBY" 17 | @keys "KEYS" 18 | @load "LOAD" 19 | @lpush "LPUSH" 20 | @ltrim "LTRIM" 21 | @rpush "RPUSH" 22 | @lrange "LRANGE" 23 | @lrem "LREM" 24 | @rpoplpush "RPOPLPUSH" 25 | @sadd "SADD" 26 | @script "SCRIPT" 27 | @set "SET" 28 | @smembers "SMEMBERS" 29 | @zadd "ZADD" 30 | @zcount "ZCOUNT" 31 | @zrem "ZREM" 32 | @zrange "ZRANGE" 33 | @zremrangebyscore "ZREMRANGEBYSCORE" 34 | 35 | @doc """ 36 | Get all keys by key pattern. 37 | """ 38 | def keys(pattern) do 39 | query([@keys, pattern]) 40 | end 41 | 42 | @doc """ 43 | Get all keys by key pattern. 44 | """ 45 | def keys!(pattern) do 46 | query!([@keys, pattern]) 47 | end 48 | 49 | @doc """ 50 | Get all members of the set by key. 51 | """ 52 | def smembers(key) do 53 | query([@smembers, key]) 54 | end 55 | 56 | @doc """ 57 | Get value of a key. 58 | """ 59 | def get!(key) do 60 | query!([@get, key]) 61 | end 62 | 63 | @doc """ 64 | Delete the key. 65 | """ 66 | def del(key) do 67 | query([@del, key]) 68 | end 69 | 70 | @doc """ 71 | Get all values of the hash by key. 72 | """ 73 | def hgetall(key) do 74 | query([@hgetall, key]) 75 | end 76 | 77 | @doc """ 78 | Get all values of the hash by key. 79 | """ 80 | def hgetall!(key) do 81 | query!([@hgetall, key]) 82 | end 83 | 84 | @doc """ 85 | Increments value of a key by 1. 86 | 87 | ### Examples 88 | iex> Flume.Redis.Client.incr("flume_test:test:incr") 89 | {:ok, 1} 90 | """ 91 | def incr(key) do 92 | query([@incr, key]) 93 | end 94 | 95 | @doc """ 96 | Increments value of a key by given number. 97 | 98 | ### Examples 99 | iex> Flume.Redis.Client.incrby("flume_test:test:incrby", 10) 100 | {:ok, 10} 101 | """ 102 | def incrby(key, count) do 103 | query([@incrby, key, count]) 104 | end 105 | 106 | @doc """ 107 | Decrements value of a key by 1. 108 | 109 | ### Examples 110 | iex> Flume.Redis.Client.decr("flume_test:test:decr") 111 | {:ok, -1} 112 | """ 113 | def decr(key) do 114 | query([@decr, key]) 115 | end 116 | 117 | @doc """ 118 | Decrements value of a key by given number. 119 | 120 | ### Examples 121 | iex> Flume.Redis.Client.decrby("flume_test:test:decrby", 10) 122 | {:ok, -10} 123 | """ 124 | def decrby(key, count) do 125 | query([@decrby, key, count]) 126 | end 127 | 128 | @doc """ 129 | Add the member to the set stored at key. 130 | """ 131 | def sadd(key, value) do 132 | query([@sadd, key, value]) 133 | end 134 | 135 | @doc """ 136 | Add the member to the set stored at key. 137 | """ 138 | def sadd!(key, value) do 139 | query!([@sadd, key, value]) 140 | end 141 | 142 | @doc """ 143 | Sets the value for a key 144 | """ 145 | def set(key, value) do 146 | query([@set, key, value]) 147 | end 148 | 149 | def set_nx(key, value, timeout) do 150 | query([@set, key, value, "NX", "PX", timeout]) 151 | end 152 | 153 | @doc """ 154 | Pushes an element at the start of a list. 155 | 156 | ## Examples 157 | iex> Flume.Redis.Client.lpush("flume_test:test:lpush", 1) 158 | {:ok, 1} 159 | """ 160 | def lpush(list_name, value) do 161 | query([@lpush, list_name, value]) 162 | end 163 | 164 | def lpush_command(list_name, value) do 165 | [@lpush, list_name, value] 166 | end 167 | 168 | def ltrim_command(list_name, start, finish) do 169 | [@ltrim, list_name, start, finish] 170 | end 171 | 172 | @doc """ 173 | Pushes an element at the end of a list. 174 | 175 | ## Examples 176 | iex> Flume.Redis.Client.rpush("flume_test:test:rpush", 1) 177 | {:ok, 1} 178 | """ 179 | def rpush(list_name, value) do 180 | query([@rpush, list_name, value]) 181 | end 182 | 183 | def bulk_rpush(list_name, values) when is_list(values) do 184 | query([@rpush, list_name] ++ values) 185 | end 186 | 187 | def rpush_command(list_name, value) do 188 | [@rpush, list_name, value] 189 | end 190 | 191 | @doc """ 192 | Returns length of the list. 193 | 194 | ## Examples 195 | iex> Flume.Redis.Client.llen!("flume_test:test:stack") 196 | 0 197 | """ 198 | def llen!(list_name), do: Command.llen(list_name) |> query!() 199 | 200 | def llen(list_name), do: Command.llen(list_name) |> query() 201 | 202 | @doc """ 203 | Removes given values from the list. 204 | 205 | ## Examples 206 | iex> Flume.Redis.Client.lpush("flume_test:test:lb:stack", 1) 207 | {:ok, 1} 208 | iex> Flume.Redis.Client.lpush("flume_test:test:lb:stack", 2) 209 | {:ok, 2} 210 | iex> Flume.Redis.Client.lrem_batch("flume_test:test:lb:stack", [1, 2]) 211 | {:ok, [1, 1]} 212 | """ 213 | def lrem_batch(list_name, values) do 214 | commands = 215 | Enum.map(values, fn value -> 216 | [@lrem, list_name, 1, value] 217 | end) 218 | 219 | case pipeline(commands) do 220 | {:error, reason} -> 221 | {:error, reason} 222 | 223 | {:ok, responses} -> 224 | success_responses = 225 | responses 226 | |> Enum.map(fn response -> 227 | case response do 228 | value when value in [:undefined, nil] -> 229 | nil 230 | 231 | error when error in [%Redix.Error{}, %Redix.ConnectionError{}] -> 232 | Logger.error("#{__MODULE__} - Error running command - #{Kernel.inspect(error)}") 233 | 234 | nil 235 | 236 | value -> 237 | value 238 | end 239 | end) 240 | |> Enum.reject(&is_nil/1) 241 | 242 | {:ok, success_responses} 243 | end 244 | end 245 | 246 | @doc """ 247 | From the given count, pops those many elements from the list and 248 | pushes it to different list atomically. 249 | 250 | ## Examples 251 | iex> Flume.Redis.Client.lpush("flume_test:test:rlb:stack", 1) 252 | {:ok, 1} 253 | iex> Flume.Redis.Client.lpush("flume_test:test:rlb:stack", 2) 254 | {:ok, 2} 255 | iex> Flume.Redis.Client.rpop_lpush_batch("flume_test:test:rlb:stack", "flume_test:test:rlb:new_stack", 2) 256 | {:ok, ["1", "2"]} 257 | """ 258 | def rpop_lpush_batch(from, to, count) do 259 | commands = 260 | Enum.map(1..count, fn _ -> 261 | [@rpoplpush, from, to] 262 | end) 263 | 264 | case pipeline(commands) do 265 | {:error, reason} -> 266 | {:error, reason} 267 | 268 | {:ok, responses} -> 269 | success_responses = 270 | responses 271 | |> Enum.map(fn response -> 272 | case response do 273 | value when value in [:undefined, nil] -> 274 | nil 275 | 276 | error when error in [%Redix.Error{}, %Redix.ConnectionError{}] -> 277 | Logger.error("#{__MODULE__} - Error running command - #{Kernel.inspect(error)}") 278 | nil 279 | 280 | value -> 281 | value 282 | end 283 | end) 284 | |> Enum.reject(&is_nil/1) 285 | 286 | {:ok, success_responses} 287 | end 288 | end 289 | 290 | def lrem!(key, value, count \\ 1) do 291 | query!([@lrem, key, count, value]) 292 | end 293 | 294 | def lrange!(key, range_start \\ 0, range_end \\ -1) do 295 | query!([@lrange, key, range_start, range_end]) 296 | end 297 | 298 | def lrange(key, range_start \\ 0, range_end \\ -1) do 299 | query([@lrange, key, range_start, range_end]) 300 | end 301 | 302 | def lrange_command(key, range_start \\ 0, range_end \\ -1) do 303 | [@lrange, key, range_start, range_end] 304 | end 305 | 306 | def zadd(key, score, value) do 307 | query([@zadd, key, score, value]) 308 | end 309 | 310 | def zadd!(key, score, value) do 311 | query!([@zadd, key, score, value]) 312 | end 313 | 314 | def bulk_zadd_command(key, scores_with_value) do 315 | [@zadd, key] ++ scores_with_value 316 | end 317 | 318 | def zrem!(set, member) do 319 | query!([@zrem, set, member]) 320 | end 321 | 322 | def zrem_command(set, member) do 323 | [@zrem, set, member] 324 | end 325 | 326 | def zrange!(key, range_start \\ 0, range_end \\ -1) do 327 | query!([@zrange, key, range_start, range_end]) 328 | end 329 | 330 | def zcount!(key, range_start \\ "-inf", range_end \\ "+inf") do 331 | query!([@zcount, key, range_start, range_end]) 332 | end 333 | 334 | def zcount(key, range_start \\ "-inf", range_end \\ "+inf") do 335 | query([@zcount, key, range_start, range_end]) 336 | end 337 | 338 | def zcount_command(key, range_start \\ "-inf", range_end \\ "+inf") do 339 | [@zcount, key, range_start, range_end] 340 | end 341 | 342 | def zremrangebyscore(key, range_start \\ "-inf", range_end \\ "+inf") do 343 | query([@zremrangebyscore, key, range_start, range_end]) 344 | end 345 | 346 | def del!(key) do 347 | query!([@del, key]) 348 | end 349 | 350 | def hset(hash, key, value), do: Command.hset(hash, key, value) |> query() 351 | 352 | def hscan(attr_list), do: Command.hscan(attr_list) |> pipeline() 353 | 354 | def hincrby(hash, key, increment \\ 1), do: Command.hincrby(hash, key, increment) |> query() 355 | 356 | def hdel(hash_key_list) when hash_key_list |> is_list(), 357 | do: hash_key_list |> Command.hdel() |> pipeline() 358 | 359 | def load_script!(script) do 360 | query!([@script, @load, script]) 361 | end 362 | 363 | def evalsha_command(args) do 364 | [@evalsha] ++ args 365 | end 366 | 367 | def eval_command(args) do 368 | [@eval] ++ args 369 | end 370 | 371 | def hmget(hash_key_list) when hash_key_list |> is_list(), 372 | do: hash_key_list |> Command.hmget() |> pipeline() 373 | 374 | def hmget(hash, key), do: Command.hmget(hash, key) |> query() 375 | 376 | def query(command) do 377 | Redix.command(redix_worker_name(), command, timeout: Config.redis_timeout()) 378 | end 379 | 380 | def query!(command) do 381 | Redix.command!(redix_worker_name(), command, timeout: Config.redis_timeout()) 382 | end 383 | 384 | def pipeline([]), do: [] 385 | 386 | def pipeline(commands) when is_list(commands) do 387 | Redix.pipeline(redix_worker_name(), commands, timeout: Config.redis_timeout()) 388 | end 389 | 390 | def transaction_pipeline([]), do: [] 391 | 392 | def transaction_pipeline(commands) when is_list(commands) do 393 | Redix.transaction_pipeline(redix_worker_name(), commands, timeout: Config.redis_timeout()) 394 | end 395 | 396 | # Private API 397 | defp random_index() do 398 | rem(System.unique_integer([:positive]), Config.redis_pool_size()) 399 | end 400 | 401 | defp redix_worker_name do 402 | :"#{Flume.Redis.Supervisor.redix_worker_prefix()}_#{random_index()}" 403 | end 404 | end 405 | -------------------------------------------------------------------------------- /lib/flume/redis/command.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.Command do 2 | @match "MATCH" 3 | 4 | @hdel "HDEL" 5 | @hincrby "HINCRBY" 6 | @hmget "HMGET" 7 | @hscan "HSCAN" 8 | @hset "HSET" 9 | @llen "LLEN" 10 | 11 | @doc """ 12 | Prepares HDEL commands for list of {hash, key} pairs 13 | ## Examples 14 | iex> Flume.Redis.Preparer.hdel([{"hash_1", "key_1"}, {"hash_1", "key_2"}, {"hash_2", "key_1"}]) 15 | [["HDEL", "hash_1", "key_1", "key_2"], ["HDEL", "hash_2", "key_1"]] 16 | """ 17 | def hdel(hash_key_list) when hash_key_list |> is_list() do 18 | hash_key_list 19 | |> Enum.group_by(&(&1 |> elem(0))) 20 | |> Enum.map(fn {hash, hash_key_list} -> 21 | keys = 22 | hash_key_list 23 | |> Enum.map(&(&1 |> elem(1))) 24 | 25 | hdel(hash, keys) 26 | end) 27 | end 28 | 29 | def hdel(hash, keys) when keys |> is_list() do 30 | [@hdel, hash] ++ keys 31 | end 32 | 33 | def hdel(hash, key) do 34 | [@hdel, hash, key] 35 | end 36 | 37 | def hincrby(hash, key, increment \\ 1), do: [@hincrby, hash, key, increment] 38 | 39 | def hmget(hash_key_list) when hash_key_list |> is_list() do 40 | hash_key_list |> Enum.map(fn {hash, keys} -> hmget(hash, keys) end) 41 | end 42 | 43 | def hmget(hash, keys) when keys |> is_list() do 44 | [@hmget, hash] ++ keys 45 | end 46 | 47 | def hmget(hash, key) do 48 | [@hmget, hash, key] 49 | end 50 | 51 | def hscan(attr_list) when attr_list |> is_list() do 52 | attr_list 53 | |> Enum.map(fn 54 | [_, _] = attrs -> apply(&hscan/2, attrs) 55 | [_, _, _] = attrs -> apply(&hscan/3, attrs) 56 | end) 57 | end 58 | 59 | def hscan(hash, cursor), do: [@hscan, hash, cursor] 60 | 61 | def hscan(hash, cursor, pattern), do: [@hscan, hash, cursor, @match, pattern] 62 | 63 | def hset(hash, key, value), do: [@hset, hash, key, value] 64 | 65 | @doc """ 66 | Returns command for getting the length of a list. 67 | 68 | ## Examples 69 | iex> Flume.Redis.Command.llen("flume:test:stack") 70 | ["LLEN", "flume:test:stack"] 71 | """ 72 | def llen(key), do: [@llen, key] 73 | end 74 | -------------------------------------------------------------------------------- /lib/flume/redis/job.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.Job do 2 | require Flume.Logger 3 | 4 | alias Flume.Logger 5 | alias Flume.Support.Time 6 | alias Flume.Redis.{Client, Script, SortedSet, BulkDequeue} 7 | 8 | @enqueue_processing_jobs_script Script.compile(:enqueue_processing_jobs) 9 | 10 | def enqueue(queue_key, job) do 11 | try do 12 | response = Client.rpush(queue_key, job) 13 | 14 | case response do 15 | {:ok, [%Redix.Error{}, %Redix.Error{}]} = error -> error 16 | {:ok, [%Redix.Error{}, _]} = error -> error 17 | {:ok, [_, %Redix.Error{}]} = error -> error 18 | {:ok, [_, _]} -> :ok 19 | other -> other 20 | end 21 | catch 22 | :exit, e -> 23 | Logger.info("Error enqueueing - #{Kernel.inspect(e)}") 24 | {:error, :timeout} 25 | end 26 | end 27 | 28 | def bulk_enqueue(queue_key, jobs) do 29 | Client.bulk_rpush(queue_key, jobs) 30 | end 31 | 32 | defdelegate bulk_dequeue( 33 | dequeue_key, 34 | processing_sorted_set_key, 35 | count, 36 | current_score 37 | ), 38 | to: BulkDequeue, 39 | as: :exec 40 | 41 | defdelegate bulk_dequeue( 42 | dequeue_key, 43 | processing_sorted_set_key, 44 | limit_sorted_set_key, 45 | count, 46 | max_count, 47 | previous_score, 48 | current_score 49 | ), 50 | to: BulkDequeue, 51 | as: :exec_rate_limited 52 | 53 | def enqueue_processing_jobs(sorted_set_key, queue_key, current_score, limit) do 54 | Script.eval(@enqueue_processing_jobs_script, [ 55 | _num_of_keys = 2, 56 | sorted_set_key, 57 | queue_key, 58 | current_score, 59 | limit 60 | ]) 61 | |> case do 62 | {:error, reason} -> 63 | {:error, reason} 64 | 65 | {:ok, response} -> 66 | {:ok, response} 67 | end 68 | end 69 | 70 | def bulk_enqueue_scheduled!(queues_and_jobs) do 71 | group_by_queue(queues_and_jobs) 72 | |> Enum.map(fn {queue, scheduled_queues_and_jobs} -> 73 | jobs = Enum.map(scheduled_queues_and_jobs, fn {_, job} -> job end) 74 | response = bulk_enqueue(queue, jobs) 75 | {scheduled_queues_and_jobs, response} 76 | end) 77 | |> Enum.flat_map(fn {scheduled_queues_and_jobs, response} -> 78 | case response do 79 | {:error, error} -> 80 | Logger.error("Error running command - #{Kernel.inspect(error)}") 81 | [] 82 | 83 | {:ok, _count} -> 84 | scheduled_queues_and_jobs 85 | end 86 | end) 87 | end 88 | 89 | def bulk_remove_scheduled!(scheduled_queues_and_jobs) do 90 | bulk_remove_scheduled_commands(scheduled_queues_and_jobs) 91 | |> Client.pipeline() 92 | |> case do 93 | {:error, reason} -> 94 | [{:error, reason}] 95 | 96 | {:ok, responses} -> 97 | scheduled_queues_and_jobs 98 | |> Enum.zip(responses) 99 | |> Enum.map(fn {{_, job}, response} -> 100 | case response do 101 | value when value in [:undefined, nil] -> 102 | nil 103 | 104 | error when error in [%Redix.Error{}, %Redix.ConnectionError{}] -> 105 | Logger.error("Error running command - #{Kernel.inspect(error)}") 106 | nil 107 | 108 | _value -> 109 | job 110 | end 111 | end) 112 | |> Enum.reject(&is_nil/1) 113 | end 114 | end 115 | 116 | def schedule_job(queue_key, schedule_at, job) do 117 | score = 118 | if is_float(schedule_at) do 119 | schedule_at |> Float.to_string() 120 | else 121 | schedule_at |> Integer.to_string() 122 | end 123 | 124 | try do 125 | case SortedSet.add(queue_key, score, job) do 126 | {:ok, jid} -> {:ok, jid} 127 | other -> other 128 | end 129 | catch 130 | :exit, e -> 131 | Logger.info("Error enqueueing - #{Kernel.inspect(e)}") 132 | {:error, :timeout} 133 | end 134 | end 135 | 136 | def remove_job!(queue_key, job) do 137 | Client.lrem!(queue_key, job) 138 | end 139 | 140 | def remove_processing!(processing_sorted_set_key, job) do 141 | Client.zrem!(processing_sorted_set_key, job) 142 | end 143 | 144 | def remove_scheduled_job!(queue_key, job) do 145 | SortedSet.remove!(queue_key, job) 146 | end 147 | 148 | def fail_job!(queue_key, job) do 149 | SortedSet.add!(queue_key, Time.time_to_score(), job) 150 | end 151 | 152 | def fetch_all!(queue_key) do 153 | Client.lrange!(queue_key) 154 | end 155 | 156 | def fetch_all!(:retry, queue_key) do 157 | SortedSet.fetch_by_range!(queue_key) 158 | end 159 | 160 | def scheduled_jobs(queues, score) do 161 | Enum.map(queues, &["ZRANGEBYSCORE", &1, 0, score]) 162 | |> Client.pipeline() 163 | |> case do 164 | {:error, reason} -> 165 | {:error, reason} 166 | 167 | {:ok, response} -> 168 | # TODO: Handle error response in response array 169 | updated_jobs = 170 | response 171 | |> Enum.map(fn jobs -> 172 | Enum.map(jobs, fn job -> 173 | case job do 174 | value when value in [:undefined, nil] -> 175 | nil 176 | 177 | error when error in [%Redix.Error{}, %Redix.ConnectionError{}] -> 178 | Logger.error("Error running command - #{Kernel.inspect(error)}") 179 | nil 180 | 181 | value -> 182 | value 183 | end 184 | end) 185 | |> Enum.reject(&is_nil/1) 186 | end) 187 | 188 | {:ok, Enum.zip(queues, updated_jobs)} 189 | end 190 | end 191 | 192 | defp group_by_queue(queues_and_jobs) do 193 | Enum.group_by( 194 | queues_and_jobs, 195 | fn {_scheduled_queue, queue_name, _job} -> queue_name end, 196 | fn {scheduled_queue, _queue_name, job} -> {scheduled_queue, job} end 197 | ) 198 | end 199 | 200 | defp bulk_remove_scheduled_commands([]), do: [] 201 | 202 | defp bulk_remove_scheduled_commands([{set_name, job} | scheduled_queues_and_jobs]) do 203 | cmd = Client.zrem_command(set_name, job) 204 | [cmd | bulk_remove_scheduled_commands(scheduled_queues_and_jobs)] 205 | end 206 | end 207 | -------------------------------------------------------------------------------- /lib/flume/redis/lock.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.Lock do 2 | require Flume.Logger 3 | 4 | alias Flume.Redis.{Client, Script} 5 | 6 | @release_lock_script Script.compile(:release_lock) 7 | 8 | def acquire( 9 | lock_key, 10 | ttl 11 | ) do 12 | token = UUID.uuid4() 13 | 14 | case Client.set_nx(lock_key, token, ttl) do 15 | {:ok, "OK"} -> 16 | {:ok, token} 17 | 18 | {:ok, nil} -> 19 | {:error, :locked} 20 | 21 | {:error, reason} -> 22 | {:error, reason} 23 | end 24 | end 25 | 26 | def release(lock_key, token) do 27 | response = 28 | Script.eval(@release_lock_script, [ 29 | _num_of_keys = 1, 30 | lock_key, 31 | token 32 | ]) 33 | 34 | case response do 35 | {:ok, _count} -> 36 | :ok 37 | 38 | {:error, val} -> 39 | {:error, val} 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/flume/redis/script.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.Script do 2 | @moduledoc """ 3 | Provides helpers to load lua scripts into redis and calculate sha1 4 | """ 5 | 6 | alias Flume.Redis.Client 7 | 8 | @spec load() :: :ok 9 | def load do 10 | dir = scripts_dir() 11 | 12 | for file <- File.ls!(dir), path = Path.join(dir, file), script = File.read!(path) do 13 | Client.load_script!(script) 14 | end 15 | 16 | :ok 17 | end 18 | 19 | @spec compile(binary) :: {binary, binary} 20 | def compile(script_name) do 21 | script = 22 | Path.join(scripts_dir(), "#{script_name}.lua") 23 | |> File.read!() 24 | 25 | hash_sha = :crypto.hash(:sha, script) 26 | {script, Base.encode16(hash_sha, case: :lower)} 27 | end 28 | 29 | @spec eval({binary, binary}, List.t()) :: {:ok, term} | {:error, term} 30 | def eval({script, sha}, arguments) do 31 | result = 32 | Client.evalsha_command([sha | arguments]) 33 | |> Client.query() 34 | 35 | case result do 36 | {:error, %Redix.Error{message: "NOSCRIPT" <> _}} -> 37 | Client.eval_command([script | arguments]) 38 | |> Client.query() 39 | 40 | result -> 41 | result 42 | end 43 | end 44 | 45 | @spec scripts_dir() :: binary 46 | defp scripts_dir, do: :code.priv_dir(:flume) |> Path.join("scripts") 47 | end 48 | -------------------------------------------------------------------------------- /lib/flume/redis/sorted_set.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.SortedSet do 2 | alias Flume.Redis.Client 3 | 4 | defdelegate add(key, score, value), to: Client, as: :zadd 5 | 6 | defdelegate add!(key, score, value), to: Client, as: :zadd! 7 | 8 | defdelegate remove!(key, value), to: Client, as: :zrem! 9 | 10 | defdelegate fetch_by_range!(key, start_range \\ 0, end_range \\ 0), to: Client, as: :zrange! 11 | end 12 | -------------------------------------------------------------------------------- /lib/flume/redis/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.Supervisor do 2 | @doc false 3 | use Application 4 | 5 | alias Flume.Config 6 | 7 | @redix_worker_prefix "flume_redix" 8 | 9 | def start(_type, _args) do 10 | Supervisor.start_link([], strategy: :one_for_one) 11 | end 12 | 13 | def start_link(opts \\ []) do 14 | sup_opts = [ 15 | strategy: :one_for_one, 16 | max_restarts: 20, 17 | max_seconds: 5, 18 | name: __MODULE__ 19 | ] 20 | 21 | {:ok, pid} = Supervisor.start_link(redix_worker_spec(opts), sup_opts) 22 | 23 | # Load redis lua scripts 24 | Flume.Redis.Script.load() 25 | |> case do 26 | :ok -> 27 | {:ok, pid} 28 | 29 | error -> 30 | {:shutdown, error} 31 | end 32 | end 33 | 34 | def child_spec(opts) do 35 | %{ 36 | id: __MODULE__, 37 | start: {__MODULE__, :start_link, [opts]}, 38 | type: :supervisor, 39 | shutdown: 500 40 | } 41 | end 42 | 43 | def redix_worker_prefix do 44 | @redix_worker_prefix 45 | end 46 | 47 | # Private API 48 | 49 | defp redix_worker_spec(options) do 50 | pool_size = Config.redis_pool_size() 51 | 52 | # Create the redix children list of workers: 53 | for i <- 0..(pool_size - 1) do 54 | connection_opts = 55 | Keyword.put(Config.connection_opts(options), :name, :"#{redix_worker_prefix()}_#{i}") 56 | 57 | args = Keyword.merge(Config.redis_opts(options), connection_opts) 58 | Supervisor.child_spec({Redix, args}, id: {Redix, i}) 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/flume/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Supervisor do 2 | @moduledoc """ 3 | Flume is a job processing system backed by Redis & GenStage. 4 | Each pipeline processes jobs from a specific Redis queue. 5 | Flume has a retry mechanism that keeps retrying the jobs with an exponential backoff. 6 | """ 7 | use Application 8 | 9 | import Supervisor.Spec 10 | 11 | alias Flume.Config 12 | 13 | def start(_type, _args) do 14 | Supervisor.start_link([], strategy: :one_for_one) 15 | end 16 | 17 | def start_link do 18 | children = 19 | if Config.mock() do 20 | [] 21 | else 22 | # This order matters, first we need to start all redix worker processes 23 | # then all other processes. 24 | [ 25 | supervisor(Flume.Redis.Supervisor, []), 26 | worker(Flume.Queue.Scheduler, [Config.scheduler_opts()]), 27 | supervisor(Flume.Pipeline.SystemEvent.Supervisor, []), 28 | supervisor(Task.Supervisor, [[name: Flume.SafeApplySupervisor]]) 29 | ] ++ Flume.Support.Pipelines.list() 30 | end 31 | 32 | opts = [ 33 | strategy: :one_for_one, 34 | max_restarts: 20, 35 | max_seconds: 5, 36 | name: Flume.Supervisor 37 | ] 38 | 39 | {:ok, _pid} = Supervisor.start_link(children, opts) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/flume/support/pipelines.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Support.Pipelines do 2 | @moduledoc """ 3 | This module returns the pipelines and its children 4 | based on the configuration 5 | """ 6 | 7 | alias Flume.Pipeline.Event, as: EventPipeline 8 | alias Flume.Config, as: FlumeConfig 9 | 10 | @doc false 11 | def list do 12 | import Supervisor.Spec 13 | 14 | Flume.Config.pipelines() 15 | |> Enum.flat_map(fn pipeline -> 16 | pipeline_struct = Flume.Pipeline.new(pipeline) 17 | scheduler_options = FlumeConfig.scheduler_opts() ++ [queue: pipeline.queue] 18 | EventPipeline.attach_instrumentation(pipeline_struct) 19 | 20 | [ 21 | worker(EventPipeline.Producer, [pipeline_struct], id: generate_id()), 22 | worker(EventPipeline.ProducerConsumer, [pipeline_struct], id: generate_id()), 23 | worker(EventPipeline.Consumer, [pipeline_struct], id: generate_id()), 24 | worker(Flume.Queue.ProcessingScheduler, [scheduler_options], id: generate_id()) 25 | ] 26 | end) 27 | end 28 | 29 | defp generate_id do 30 | <> = :crypto.strong_rand_bytes(8) 31 | "#{part1}#{part2}" 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/flume/support/time.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Support.Time do 2 | import DateTime, only: [utc_now: 0, to_unix: 2, from_unix!: 2] 3 | 4 | def offset_from_now(offset_in_ms, current_time \\ utc_now()) do 5 | now_in_µs = current_time |> to_unix(:microsecond) 6 | offset_in_µs = offset_in_ms * 1_000 7 | 8 | now_in_µs 9 | |> Kernel.+(offset_in_µs) 10 | |> round() 11 | |> from_unix!(:microsecond) 12 | end 13 | 14 | def offset_before(offset_in_ms, current_time \\ utc_now()) do 15 | now_in_µs = current_time |> to_unix(:microsecond) 16 | offset_in_µs = offset_in_ms * 1_000 17 | 18 | now_in_µs 19 | |> Kernel.-(offset_in_µs) 20 | |> round() 21 | |> from_unix!(:microsecond) 22 | end 23 | 24 | def time_to_score(time \\ utc_now()) do 25 | time 26 | |> unix_seconds 27 | |> Float.to_string() 28 | end 29 | 30 | def unix_seconds(time \\ utc_now()) do 31 | to_unix(time, :microsecond) / 1_000_000.0 32 | end 33 | 34 | def format_current_date(current_date) do 35 | date_time = 36 | %{current_date | microsecond: {0, 0}} 37 | |> DateTime.to_string() 38 | 39 | date = 40 | current_date 41 | |> DateTime.to_date() 42 | |> Date.to_string() 43 | 44 | {date_time, date} 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/flume/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Utils do 2 | def payload_size(list) when is_list(list) do 3 | Enum.reduce(list, 0, fn item, acc -> acc + byte_size(item) end) 4 | end 5 | 6 | def safe_apply(function, timeout) do 7 | task = Task.Supervisor.async_nolink(Flume.SafeApplySupervisor, function) 8 | 9 | case Task.yield(task, timeout) || Task.shutdown(task) do 10 | {:ok, result} -> {:ok, result} 11 | {:exit, reason} -> {:exit, reason} 12 | nil -> {:timeout, "Timed out after #{timeout} ms"} 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/flume/utils/integer_extension.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.Utils.IntegerExtension do 2 | def parse(value, default \\ :error) 3 | 4 | def parse(nil, nil), do: nil 5 | 6 | def parse(nil, default), do: default 7 | 8 | def parse(value, _default) when is_integer(value), do: value 9 | 10 | def parse(value, default) when is_binary(value) do 11 | case Integer.parse(value) do 12 | {count, _} -> 13 | count 14 | 15 | :error -> 16 | default 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/mix/tasks/redis_benchmark.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Flume.RedisBenchmark do 2 | use Mix.Task 3 | 4 | alias Flume.Config 5 | alias Flume.Queue.Manager 6 | 7 | @namespace Config.namespace() 8 | 9 | @rate_limit_opts %{rate_limit_count: 50_000, rate_limit_scale: 1000} 10 | 11 | @defaults [ 12 | count: 10_000, 13 | queues: 20, 14 | dequeue_batch: 50, 15 | enqueue_concurrency: 500, 16 | # (count * pre_seed_multiplier) jobs get pre-seeded split into all queues 17 | pre_seed_multiplier: 1, 18 | dequeue_poll_timeout: 1000 19 | ] 20 | 21 | def run(args \\ []) do 22 | Mix.Task.run("app.start") 23 | 24 | user_options = 25 | OptionParser.parse( 26 | args, 27 | strict: [ 28 | count: :integer, 29 | queues: :integer, 30 | dequeue_batch: :integer, 31 | enqueue_concurrency: :integer, 32 | pre_seed_multiplier: :integer, 33 | dequeue_poll_timeout: :integer, 34 | arg_count: :integer 35 | ] 36 | ) 37 | |> elem(0) 38 | 39 | opts = 40 | @defaults 41 | |> Keyword.merge(user_options) 42 | |> Enum.into(%{}) 43 | 44 | IO.inspect("Running benchmark with config") 45 | IO.inspect(opts) 46 | noop = fn _, _ -> [] end 47 | 48 | Benchee.run( 49 | %{ 50 | "enqueue" => fn jobs_queue_mapping -> 51 | start_enqueue_dequeue( 52 | &enqueue/2, 53 | noop, 54 | jobs_queue_mapping, 55 | opts 56 | ) 57 | end, 58 | "bulk_dequeue" => fn jobs_queue_mapping -> 59 | start_enqueue_dequeue( 60 | noop, 61 | &bulk_dequeue/2, 62 | jobs_queue_mapping, 63 | opts 64 | ) 65 | end, 66 | "interleaved_enqueue_dequeue" => fn jobs_queue_mapping -> 67 | start_enqueue_dequeue( 68 | &enqueue/2, 69 | &bulk_dequeue/2, 70 | jobs_queue_mapping, 71 | opts 72 | ) 73 | end 74 | }, 75 | before_each: fn arg_count -> 76 | clear_redis() 77 | 78 | jobs_queue_mapping = build_jobs_queue_mapping(opts[:count], opts[:queues], arg_count) 79 | 80 | pre_seed_queues(jobs_queue_mapping, opts[:pre_seed_multiplier]) 81 | jobs_queue_mapping 82 | end, 83 | after_each: fn _ -> clear_redis() end, 84 | inputs: %{ 85 | "0.5 kb" => 150, 86 | "1 kb" => 250, 87 | "2.5 kb" => 650 88 | } 89 | ) 90 | end 91 | 92 | def poll(function, poll_timeout) do 93 | function.() 94 | Process.sleep(poll_timeout) 95 | poll(function, poll_timeout) 96 | end 97 | 98 | defp start_enqueue_dequeue( 99 | enqueue_fn, 100 | dequeue_fn, 101 | jobs_queue_mapping, 102 | _opts = %{ 103 | count: _count, 104 | queues: _queues, 105 | dequeue_batch: dequeue_batch, 106 | enqueue_concurrency: enqueue_concurrency, 107 | pre_seed_multiplier: _pre_seed_multiplier, 108 | dequeue_poll_timeout: dequeue_poll_timeout 109 | } 110 | ) do 111 | dequeue_tasks = 112 | start_dequeue(jobs_queue_mapping, dequeue_fn, dequeue_batch, dequeue_poll_timeout) 113 | 114 | enqueue_fn.(jobs_queue_mapping, enqueue_concurrency) 115 | Enum.each(dequeue_tasks, &Process.exit(&1, :kill)) 116 | end 117 | 118 | defp enqueue(jobs_queue_mapping, concurrency) do 119 | enqueue_each = fn {jobs, queue} -> 120 | Enum.each(jobs, fn job -> 121 | {:ok, _} = Manager.enqueue(@namespace, queue, :worker, :perform, [job]) 122 | end) 123 | end 124 | 125 | enqueue = fn {jobs, queue} -> 126 | chunks = Enum.split(jobs, concurrency) |> Tuple.to_list() 127 | 128 | Enum.map(chunks, fn chunk -> 129 | Task.async(fn -> enqueue_each.({chunk, queue}) end) 130 | end) 131 | end 132 | 133 | Enum.flat_map(jobs_queue_mapping, enqueue) 134 | |> Enum.each(&Task.await(&1, :infinity)) 135 | end 136 | 137 | defp clear_redis do 138 | pool_size = Config.redis_pool_size() 139 | conn_key = :"#{Flume.Redis.Supervisor.redix_worker_prefix()}_#{pool_size - 1}" 140 | keys = Redix.command!(conn_key, ["KEYS", "#{Config.namespace()}:*"]) 141 | Enum.map(keys, fn key -> Redix.command(conn_key, ["DEL", key]) end) 142 | end 143 | 144 | defp build_jobs_queue_mapping(count, queue_nos, arg_count) do 145 | jobs = build_jobs(count, arg_count) 146 | 147 | chunked = Enum.chunk_every(jobs, round(count / queue_nos)) 148 | 149 | Enum.with_index(chunked) 150 | |> Enum.map(fn {jobs, idx} -> 151 | {jobs, "test:#{idx}"} 152 | end) 153 | end 154 | 155 | defp pre_seed_queues(jobs_queue_mapping, multiplier) do 156 | enqueue = fn {job_ids, queue} -> 157 | jobs = Enum.map(job_ids, fn id -> [:worker, :perform, [id]] end) 158 | {:ok, _} = Manager.bulk_enqueue(@namespace, queue, jobs) 159 | end 160 | 161 | Enum.each(1..multiplier, fn _ -> 162 | Enum.each(jobs_queue_mapping, enqueue) 163 | end) 164 | end 165 | 166 | defp start_dequeue(jobs_queue_mapping, dequeue_fn, dequeue_batch, dequeue_poll_timeout) do 167 | Enum.map(jobs_queue_mapping, fn {_jobs, queue} -> 168 | {:ok, pid} = 169 | start_poll_server( 170 | fn -> 171 | dequeue_fn.(queue, dequeue_batch) 172 | end, 173 | dequeue_poll_timeout 174 | ) 175 | 176 | pid 177 | end) 178 | end 179 | 180 | defp start_poll_server(function, poll_timeout) do 181 | Task.start(__MODULE__, :poll, [function, poll_timeout]) 182 | end 183 | 184 | defp bulk_dequeue(queue, batch) do 185 | {:ok, _jobs} = 186 | Manager.fetch_jobs( 187 | @namespace, 188 | queue, 189 | batch, 190 | @rate_limit_opts 191 | ) 192 | end 193 | 194 | defp build_jobs(nos, arg_count), do: Enum.map(1..nos, &build_job(&1, arg_count)) 195 | 196 | defp build_job(id, arg_count), do: %{id: id, args: Enum.map(1..arg_count, fn idx -> idx end)} 197 | end 198 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :flume, 7 | version: "0.2.0", 8 | elixir: "~> 1.8", 9 | elixirc_paths: elixirc_paths(Mix.env()), 10 | start_permanent: Mix.env() == :prod, 11 | deps: deps(), 12 | description: description(), 13 | package: package(), 14 | source_url: "https://github.com/scripbox/flume", 15 | homepage_url: "https://github.com/scripbox/flume", 16 | test_coverage: [tool: ExCoveralls], 17 | preferred_cli_env: [coveralls: :test] 18 | ] 19 | end 20 | 21 | # Specifies which paths to compile per environment. 22 | defp elixirc_paths(:test), do: ["lib", "test/support", "test/factories"] 23 | defp elixirc_paths(_), do: ["lib"] 24 | 25 | # Run "mix help compile.app" to learn about applications. 26 | def application do 27 | [ 28 | applications: [ 29 | :redix, 30 | :logger_file_backend, 31 | :gen_stage, 32 | :jason, 33 | :poolboy, 34 | :retry, 35 | :telemetry 36 | ], 37 | extra_applications: [:logger], 38 | mod: {Flume, []} 39 | ] 40 | end 41 | 42 | # Run "mix help deps" to learn about dependencies. 43 | defp deps do 44 | [ 45 | {:redix, "~> 1.0"}, 46 | {:gen_stage, "~> 0.14.0"}, 47 | {:jason, "~> 1.1.0"}, 48 | {:poolboy, "~> 1.5.1"}, 49 | {:elixir_uuid, "~> 1.2"}, 50 | {:logger_file_backend, "~> 0.0.10"}, 51 | {:retry, "0.8.2"}, 52 | {:benchee, "~> 1.0"}, 53 | {:telemetry, "~> 1.0"}, 54 | {:excoveralls, "~> 0.10.6", only: :test} 55 | ] 56 | end 57 | 58 | defp description do 59 | "Flume is a job processing system backed by GenStage & Redis" 60 | end 61 | 62 | defp package do 63 | [ 64 | licenses: ["Apache 2.0"], 65 | links: %{"GitHub" => "https://github.com/scripbox/flume"} 66 | ] 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"}, 3 | "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, 4 | "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, 5 | "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, 6 | "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, 7 | "excoveralls": {:hex, :excoveralls, "0.10.6", "e2b9718c9d8e3ef90bc22278c3f76c850a9f9116faf4ebe9678063310742edc2", [:mix], [{:hackney, "~> 1.13", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b06c73492aa9940c4c29cfc1356bcf5540ae318f17b423749a0615a66ee3e049"}, 8 | "gen_stage": {:hex, :gen_stage, "0.14.0", "65ae78509f85b59d360690ce3378d5096c3130a0694bab95b0c4ae66f3008fad", [:mix], [], "hexpm", "095d38418e538af99ac82043985d26724164b78736a1d0f137c308332ad46250"}, 9 | "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "c2790c9f0f7205f4a362512192dee8179097394400e745e4d20bab7226a8eaad"}, 10 | "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, 11 | "jason": {:hex, :jason, "1.1.0", "9634bca30f2f7468dde3e704d5865319b1eb88e4a8cded5c995baf0aa957524f", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "23e9d96104cce9e74a63356f280ad7c7abc8ee68a45fb051d0845236bc94386c"}, 12 | "logger_file_backend": {:hex, :logger_file_backend, "0.0.11", "3bbc5f31d3669e8d09d7a9443e86056fae7fc18e45c6f748c33b8c79a7e147a1", [:mix], [], "hexpm", "62be826f04644c62b0a2bc98a13e2e7ae52c0a4eda020f4c59d7287356d5e445"}, 13 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 14 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, 15 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, 16 | "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, 17 | "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm", "8f7168911120e13419e086e78d20e4d1a6776f1eee2411ac9f790af10813389f"}, 18 | "redix": {:hex, :redix, "1.1.5", "6fc460d66a5c2287e83e6d73dddc8d527ff59cb4d4f298b41e03a4db8c3b2bd5", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "679afdd4c14502fe9c11387ff1cdcb33065a1cf511097da1eee407f17c7a418b"}, 19 | "retry": {:hex, :retry, "0.8.2", "7b57bd5e1e7efeca04dd740cabdc3930c472cfa7e0186949de180c64417e9c35", [:mix], [], "hexpm", "ae8969bfea65bf2adf9e7cc6e59bc2d1b6d9e04bcb7d2c53b996d9b8a023fe78"}, 20 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm", "603561dc0fd62f4f2ea9b890f4e20e1a0d388746d6e20557cafb1b16950de88c"}, 21 | "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, 22 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"} 23 | } 24 | -------------------------------------------------------------------------------- /priv/scripts/enqueue_processing_jobs.lua: -------------------------------------------------------------------------------- 1 | -- Script to enqueue jobs stuck in the processing-sorted-set. 2 | --[[ 3 | * Fetch jobs from the processing-sorted-set whose score is older than the `current_score`. 4 | * If these jobs count is greater than zero then proceed. 5 | * Enqueue these jobs to the main queue. 6 | * Remove these jobs from the processing-sorted-set. 7 | ]] 8 | 9 | local processing_sorted_set_key = KEYS[1] 10 | local queue_key = KEYS[2] 11 | 12 | local current_score = ARGV[1] 13 | local limit = ARGV[2] 14 | 15 | local jobs = redis.call('ZRANGEBYSCORE', processing_sorted_set_key, '-inf', current_score, 'LIMIT', 0, limit) 16 | local jobs_count = table.getn(jobs) 17 | 18 | if jobs_count > 0 then 19 | redis.call('RPUSH', queue_key, unpack(jobs)) 20 | redis.call('ZREM', processing_sorted_set_key, unpack(jobs)) 21 | end 22 | 23 | return jobs_count 24 | -------------------------------------------------------------------------------- /priv/scripts/release_lock.lua: -------------------------------------------------------------------------------- 1 | -- Release lock: https://redis.io/commands/set#patterns 2 | 3 | if redis.call("get",KEYS[1]) == ARGV[1] 4 | then 5 | return redis.call("del",KEYS[1]) 6 | else 7 | return 0 8 | end 9 | -------------------------------------------------------------------------------- /test/factories/job_factory.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.JobFactory do 2 | def generate_jobs(module_name, count) do 3 | Enum.map(1..count, fn _ -> generate(module_name) end) 4 | end 5 | 6 | def generate(module_name, args \\ []) do 7 | %{ 8 | class: module_name, 9 | function: "perform", 10 | queue: "test", 11 | jid: "1082fd87-2508-4eb4-8fba-#{:rand.uniform(9_999_999)}a60e3", 12 | args: args, 13 | retry_count: 0, 14 | enqueued_at: 1_514_367_662, 15 | finished_at: nil, 16 | failed_at: nil, 17 | retried_at: nil, 18 | error_message: nil, 19 | error_backtrace: nil 20 | } 21 | |> Jason.encode!() 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/flume/pipeline/control/options_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Control.OptionsTest do 2 | use ExUnit.Case, async: true 3 | 4 | doctest Flume.Pipeline.Control.Options 5 | end 6 | -------------------------------------------------------------------------------- /test/flume/pipeline/event/consumer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.ConsumerTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Redis.Job 5 | alias Flume.{Pipeline, JobFactory} 6 | alias Flume.Pipeline.Event.Consumer 7 | 8 | @namespace Flume.Config.namespace() 9 | 10 | def event_attributes do 11 | %{ 12 | class: "EchoWorker", 13 | function: "perform", 14 | queue: "test", 15 | jid: "1082fd87-2508-4eb4-8fba-2958584a60e3", 16 | args: [], 17 | retry_count: 0, 18 | enqueued_at: 1_514_367_662, 19 | finished_at: nil, 20 | failed_at: nil, 21 | retried_at: nil, 22 | error_message: nil, 23 | error_backtrace: nil 24 | } 25 | end 26 | 27 | describe "handle_events/3" do 28 | test "processes event if it is parseable" do 29 | # Start the worker process 30 | {:ok, _} = EchoWorker.start_link() 31 | 32 | pipeline_name = "pipeline_1" 33 | queue_name = "test" 34 | caller_name = :calling_process 35 | message = "hello world" 36 | 37 | pipeline = %Pipeline{ 38 | name: pipeline_name, 39 | queue: queue_name, 40 | max_demand: 10, 41 | batch_size: 2 42 | } 43 | 44 | Process.register(self(), caller_name) 45 | 46 | serialized_event = JobFactory.generate("EchoWorker", [caller_name, message]) 47 | 48 | # Push the event to Redis 49 | Job.enqueue("#{@namespace}:queue:test", serialized_event) 50 | 51 | {:ok, producer} = 52 | TestProducer.start_link(%{ 53 | process_name: "#{pipeline_name}_producer_consumer", 54 | queue: "test" 55 | }) 56 | 57 | {:ok, _} = Consumer.start_link(pipeline) 58 | 59 | assert_receive {:received, ^message} 60 | 61 | # The consumer will also stop, since it is subscribed to the stage 62 | GenStage.stop(producer) 63 | end 64 | 65 | test "fails if event is not parseable" do 66 | # Start the worker process 67 | {:ok, _} = EchoWorker.start_link() 68 | 69 | pipeline_name = "pipeline_1" 70 | queue_name = "test" 71 | caller_name = :calling_process 72 | message = "hello world" 73 | 74 | pipeline = %Pipeline{ 75 | name: pipeline_name, 76 | queue: queue_name, 77 | max_demand: 10, 78 | batch_size: 2 79 | } 80 | 81 | Process.register(self(), caller_name) 82 | 83 | serialized_event = %{queue: "test", args: [caller_name, message]} |> Jason.encode!() 84 | 85 | # Push the event to Redis 86 | Job.enqueue("#{@namespace}:test", serialized_event) 87 | 88 | {:ok, producer} = 89 | TestProducer.start_link(%{ 90 | process_name: "#{pipeline_name}_producer_consumer", 91 | queue: "test" 92 | }) 93 | 94 | {:ok, _} = Consumer.start_link(pipeline) 95 | 96 | refute_receive {:received, ^message} 97 | 98 | # The consumer will also stop, since it is subscribed to the stage 99 | GenStage.stop(producer) 100 | end 101 | 102 | test "fails if bad/missing arguments are passed to the worker" do 103 | # Start the worker process 104 | {:ok, _} = EchoWorker.start_link() 105 | 106 | pipeline_name = "pipeline_1" 107 | queue_name = "test" 108 | caller_name = :calling_process 109 | message = "hello world" 110 | 111 | pipeline = %Pipeline{ 112 | name: pipeline_name, 113 | queue: queue_name, 114 | max_demand: 10, 115 | batch_size: 2 116 | } 117 | 118 | Process.register(self(), caller_name) 119 | 120 | serialized_event = %{event_attributes() | args: [caller_name]} |> Jason.encode!() 121 | 122 | # Push the event to Redis 123 | Job.enqueue("#{@namespace}:test", serialized_event) 124 | 125 | {:ok, producer} = 126 | TestProducer.start_link(%{ 127 | process_name: "#{pipeline_name}_producer_consumer", 128 | queue: "test" 129 | }) 130 | 131 | {:ok, _} = Consumer.start_link(pipeline) 132 | 133 | refute_receive {:received, ^message} 134 | 135 | # The consumer will also stop, since it is subscribed to the stage 136 | GenStage.stop(producer) 137 | end 138 | end 139 | end 140 | -------------------------------------------------------------------------------- /test/flume/pipeline/event/producer_consumer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.ProducerConsumerTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Redis.Job 5 | alias Flume.{Pipeline, JobFactory} 6 | alias Flume.Pipeline.Event.ProducerConsumer 7 | 8 | @namespace Flume.Config.namespace() 9 | 10 | describe "handle_events/3" do 11 | test "groups similar events for bulk pipeline" do 12 | pipeline_name = "batch_pipeline" 13 | queue_name = "batch" 14 | caller_name = :calling_process 15 | 16 | Process.register(self(), caller_name) 17 | 18 | # Start the producer 19 | {:ok, producer} = 20 | TestProducer.start_link(%{ 21 | process_name: "#{pipeline_name}_producer", 22 | queue: queue_name 23 | }) 24 | 25 | # Push the event to Redis 26 | Enum.each(1..4, fn i -> 27 | Job.enqueue( 28 | "#{@namespace}:queue:#{queue_name}", 29 | JobFactory.generate("EchoWorker1", [i]) 30 | ) 31 | end) 32 | 33 | Enum.each(3..6, fn i -> 34 | Job.enqueue( 35 | "#{@namespace}:queue:#{queue_name}", 36 | JobFactory.generate("EchoWorker2", [i]) 37 | ) 38 | end) 39 | 40 | pipeline = %Pipeline{ 41 | name: pipeline_name, 42 | max_demand: 10, 43 | batch_size: 2 44 | } 45 | 46 | # Start the producer_consumer 47 | {:ok, producer_consumer} = ProducerConsumer.start_link(pipeline) 48 | 49 | # Start the consumer 50 | {:ok, _} = 51 | EchoConsumer.start_link( 52 | producer_consumer, 53 | caller_name, 54 | name: :"#{pipeline_name}_consumer" 55 | ) 56 | 57 | assert_receive {:received, [%Flume.BulkEvent{args: [[[1], [2]]], class: "EchoWorker1"}]} 58 | assert_receive {:received, [%Flume.BulkEvent{args: [[[3], [4]]], class: "EchoWorker1"}]} 59 | 60 | assert_receive {:received, [%Flume.BulkEvent{args: [[[3], [4]]], class: "EchoWorker2"}]} 61 | assert_receive {:received, [%Flume.BulkEvent{args: [[[5], [6]]], class: "EchoWorker2"}]} 62 | 63 | # The will stop the whole pipeline 64 | GenStage.stop(producer) 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test/flume/pipeline/event/producer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.ProducerTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.{Pipeline, JobFactory} 5 | alias Flume.Pipeline.Event.Producer 6 | alias Flume.Redis.Job 7 | 8 | @namespace Flume.Config.namespace() 9 | 10 | describe "handle_demand/2" do 11 | test "pull events from redis" do 12 | pipeline = %Pipeline{ 13 | name: "pipeline_1", 14 | queue: "test", 15 | max_demand: 1000 16 | } 17 | 18 | downstream_name = Enum.join([pipeline.name, "producer_consumer"], "_") |> String.to_atom() 19 | 20 | events = JobFactory.generate_jobs("EchoWorker1", 3) 21 | Job.bulk_enqueue("#{@namespace}:queue:#{pipeline.queue}", events) 22 | 23 | {:ok, producer} = Producer.start_link(pipeline) 24 | {:ok, _} = EchoConsumer.start_link(producer, self(), name: downstream_name) 25 | 26 | Enum.each(events, fn event -> 27 | assert_receive {:received, [^event]} 28 | end) 29 | 30 | # The consumer will also stop, since it is subscribed to the stage 31 | GenStage.stop(producer) 32 | end 33 | 34 | test "should schedule one outstanding fetch" do 35 | pipeline = %Pipeline{ 36 | name: "pipeline_1", 37 | queue: "test", 38 | max_demand: 1, 39 | batch_size: 1, 40 | rate_limit_count: 2, 41 | rate_limit_scale: 1000, 42 | rate_limit_key: "pipeline_1" 43 | } 44 | 45 | downstream_name = Enum.join([pipeline.name, "producer_consumer"], "_") |> String.to_atom() 46 | 47 | events = JobFactory.generate_jobs("EchoWorker1", 15) 48 | Job.bulk_enqueue("#{@namespace}:queue:#{pipeline.queue}", events) 49 | 50 | {:ok, producer} = Producer.start_link(pipeline) 51 | 52 | {:ok, _} = 53 | EchoConsumer.start_link(producer, self(), 54 | name: downstream_name, 55 | max_demand: 5, 56 | min_demand: 3 57 | ) 58 | 59 | assert_receive_events(15, []) 60 | 61 | # The consumer will also stop, since it is subscribed to the stage 62 | GenStage.stop(producer) 63 | end 64 | end 65 | 66 | describe "pause/1" do 67 | test "pauses the producer from fetching more events" do 68 | pipeline_name = "batch_pipeline" 69 | queue_name = "batch" 70 | caller_name = :calling_process 71 | 72 | Process.register(self(), caller_name) 73 | 74 | pipeline = %Pipeline{ 75 | name: pipeline_name, 76 | queue: queue_name, 77 | max_demand: 10, 78 | batch_size: 2 79 | } 80 | 81 | # Push events to Redis 82 | Enum.each(1..4, fn i -> 83 | Job.enqueue( 84 | "#{@namespace}:queue:#{queue_name}", 85 | JobFactory.generate("EchoWorker1", [i]) 86 | ) 87 | end) 88 | 89 | # Start the producer 90 | {:ok, producer} = Producer.start_link(pipeline) 91 | 92 | # Start the consumer 93 | {:ok, _} = 94 | EchoConsumer.start_link( 95 | producer, 96 | caller_name, 97 | name: :"#{pipeline_name}_consumer" 98 | ) 99 | 100 | assert_receive {:received, [event_1]} 101 | assert_receive {:received, [event_2]} 102 | assert_receive {:received, [event_3]} 103 | assert_receive {:received, [event_4]} 104 | 105 | decoded_event_1 = Jason.decode!(event_1) 106 | decoded_event_2 = Jason.decode!(event_2) 107 | decoded_event_3 = Jason.decode!(event_3) 108 | decoded_event_4 = Jason.decode!(event_4) 109 | 110 | assert match?(%{"args" => [1], "class" => "EchoWorker1"}, decoded_event_1) 111 | assert match?(%{"args" => [2], "class" => "EchoWorker1"}, decoded_event_2) 112 | assert match?(%{"args" => [3], "class" => "EchoWorker1"}, decoded_event_3) 113 | assert match?(%{"args" => [4], "class" => "EchoWorker1"}, decoded_event_4) 114 | 115 | Producer.pause(pipeline_name, false) 116 | 117 | Enum.each(3..6, fn i -> 118 | Job.enqueue( 119 | "#{@namespace}:queue:#{queue_name}", 120 | JobFactory.generate("EchoWorker2", [i]) 121 | ) 122 | end) 123 | 124 | refute_receive {:received, [_event_1]} 125 | refute_receive {:received, [_event_2]} 126 | refute_receive {:received, [_event_3]} 127 | refute_receive {:received, [_event_4]} 128 | 129 | # The will stop the whole pipeline 130 | GenStage.stop(producer) 131 | end 132 | end 133 | 134 | describe "resume/1" do 135 | test "resumes the producer to fetch more events from the source" do 136 | pipeline_name = "batch_pipeline" 137 | queue_name = "batch" 138 | caller_name = :calling_process 139 | 140 | Process.register(self(), caller_name) 141 | 142 | pipeline = %Pipeline{ 143 | name: pipeline_name, 144 | queue: queue_name, 145 | max_demand: 10, 146 | batch_size: 2 147 | } 148 | 149 | # Start the producer 150 | {:ok, producer} = Producer.start_link(pipeline) 151 | 152 | # Start the consumer 153 | {:ok, _} = 154 | EchoConsumer.start_link( 155 | producer, 156 | caller_name, 157 | name: :"#{pipeline_name}_consumer" 158 | ) 159 | 160 | :ok = Producer.pause(pipeline_name, false) 161 | 162 | Enum.each(1..4, fn i -> 163 | Job.enqueue( 164 | "#{@namespace}:queue:#{queue_name}", 165 | JobFactory.generate("EchoWorker1", [i]) 166 | ) 167 | end) 168 | 169 | Enum.each(3..6, fn i -> 170 | Job.enqueue( 171 | "#{@namespace}:queue:#{queue_name}", 172 | JobFactory.generate("EchoWorker2", [i]) 173 | ) 174 | end) 175 | 176 | refute_receive {:received, [_event_1]} 177 | refute_receive {:received, [_event_2]} 178 | refute_receive {:received, [_event_3]} 179 | refute_receive {:received, [_event_4]} 180 | 181 | Producer.resume(pipeline_name, false) 182 | 183 | assert_receive {:received, [event_1]}, 2000 184 | assert_receive {:received, [event_2]}, 2000 185 | assert_receive {:received, [event_3]}, 2000 186 | assert_receive {:received, [event_4]}, 2000 187 | 188 | decoded_event_1 = Jason.decode!(event_1) 189 | decoded_event_2 = Jason.decode!(event_2) 190 | decoded_event_3 = Jason.decode!(event_3) 191 | decoded_event_4 = Jason.decode!(event_4) 192 | 193 | assert match?(%{"args" => [1], "class" => "EchoWorker1"}, decoded_event_1) 194 | assert match?(%{"args" => [2], "class" => "EchoWorker1"}, decoded_event_2) 195 | assert match?(%{"args" => [3], "class" => "EchoWorker1"}, decoded_event_3) 196 | assert match?(%{"args" => [4], "class" => "EchoWorker1"}, decoded_event_4) 197 | 198 | # The will stop the whole pipeline 199 | GenStage.stop(producer) 200 | end 201 | end 202 | 203 | describe "handle_cast/2 for pause related messages" do 204 | test "returns unchanged state if pipline has already been paused" do 205 | assert Producer.handle_cast(:pause, %{paused: true}) == {:noreply, [], %{paused: true}} 206 | end 207 | 208 | test "updates the paused state for a running pipeline" do 209 | assert Producer.handle_cast(:pause, %{paused: false}) == {:noreply, [], %{paused: true}} 210 | end 211 | end 212 | 213 | describe "handle_cast/2 for resume related messages" do 214 | test "returns unchanged state for running pipeline" do 215 | assert Producer.handle_cast(:resume, %{paused: false}) == {:noreply, [], %{paused: false}} 216 | end 217 | 218 | test "updates the paused state for a paused pipeline" do 219 | assert Producer.handle_cast(:resume, %{paused: true}) == {:noreply, [], %{paused: false}} 220 | end 221 | end 222 | 223 | def assert_receive_events(expected, received_so_far) do 224 | if length(received_so_far) == expected do 225 | :ok 226 | else 227 | receive do 228 | {:received, events} -> 229 | assert_receive_events(expected, events ++ received_so_far) 230 | after 231 | 10_000 -> 232 | flunk("Failed to receive message after 10 seconds") 233 | end 234 | end 235 | end 236 | end 237 | -------------------------------------------------------------------------------- /test/flume/pipeline/event/worker_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.Event.WorkerTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.{Event, BulkEvent} 5 | alias Flume.Pipeline.Event.Worker 6 | 7 | describe "process/2" do 8 | test "processes single event" do 9 | caller_name = :calling_process 10 | message = "hello world" 11 | 12 | Process.register(self(), caller_name) 13 | 14 | serialized_event = 15 | %{event_attributes() | "args" => [caller_name, message]} |> Jason.encode!() 16 | 17 | # Start the worker process 18 | {:ok, _pid} = Worker.start_link(default_pipeline(), serialized_event) 19 | 20 | assert_receive {:received, ^message} 21 | end 22 | 23 | test "emits telemetry event on successful pipelines" do 24 | pipeline = %{ 25 | name: "Pipeline1", 26 | queue: "default", 27 | rate_limit_count: 1000, 28 | rate_limit_scale: 5000, 29 | instrument: true 30 | } 31 | 32 | pipeline_name_atom = String.to_atom(pipeline.name) 33 | 34 | {test_name, _arity} = __ENV__.function 35 | 36 | parent = self() 37 | 38 | handler = fn event, measurements, meta, _config -> 39 | send(parent, {event, measurements, meta}) 40 | end 41 | 42 | :telemetry.attach( 43 | [to_string(test_name), :worker], 44 | [pipeline_name_atom, :worker], 45 | handler, 46 | :no_config 47 | ) 48 | 49 | :telemetry.attach( 50 | [to_string(test_name), :worker, :job], 51 | [pipeline_name_atom, :worker, :job], 52 | handler, 53 | :no_config 54 | ) 55 | 56 | caller_name = :calling_process 57 | message = "hello world" 58 | 59 | Process.register(self(), caller_name) 60 | 61 | serialized_event = 62 | %{event_attributes() | "args" => [caller_name, message]} |> Jason.encode!() 63 | 64 | # Start the worker process 65 | {:ok, _pid} = Worker.start_link(pipeline, serialized_event) 66 | 67 | assert_receive { 68 | [^pipeline_name_atom, :worker, :job], 69 | %{duration: value}, 70 | %{module: "echoworker"} 71 | } 72 | 73 | assert value >= 0 74 | 75 | assert_receive { 76 | [^pipeline_name_atom, :worker], 77 | %{duration: value}, 78 | %{module: "echoworker"} 79 | } 80 | 81 | assert value >= 0 82 | 83 | :telemetry.detach(to_string(test_name)) 84 | end 85 | 86 | test "processes bulk event" do 87 | pipeline = Map.put(default_pipeline(), :batch_size, 10) 88 | 89 | caller_name = :calling_process 90 | message = "hello world" 91 | 92 | Process.register(self(), caller_name) 93 | 94 | single_event = %{event_attributes() | "args" => [caller_name, message]} |> Event.new() 95 | bulk_event = BulkEvent.new(single_event) 96 | 97 | # Start the worker process 98 | {:ok, _pid} = Worker.start_link(pipeline, bulk_event) 99 | 100 | assert_receive {:received, ^message} 101 | end 102 | 103 | test "single worker receives context" do 104 | caller_name = :calling_process 105 | Process.register(self(), caller_name) 106 | context = %{"request_id" => 123} 107 | 108 | serialized_event = context_event(context, caller_name) |> Jason.encode!() 109 | 110 | {:ok, _pid} = Worker.start_link(default_pipeline(), serialized_event) 111 | assert_receive {:context, ^context} 112 | end 113 | 114 | test "bulk worker receives context" do 115 | caller_name = :calling_process 116 | Process.register(self(), caller_name) 117 | context = %{"request_id" => 123} 118 | 119 | bulk_event = context_event(context, caller_name) |> Event.new() |> BulkEvent.new() 120 | 121 | {:ok, _pid} = Worker.start_link(default_pipeline(), bulk_event) 122 | assert_receive {:context, [^context]} 123 | end 124 | end 125 | 126 | defp context_event(context, caller_name) do 127 | %{ 128 | event_attributes() 129 | | "args" => [caller_name], 130 | "class" => "EchoContextWorker", 131 | "context" => context 132 | } 133 | end 134 | 135 | def default_pipeline do 136 | %{ 137 | name: "Pipeline1", 138 | queue: "default", 139 | rate_limit_count: 1000, 140 | rate_limit_scale: 5000 141 | } 142 | end 143 | 144 | def event_attributes do 145 | %{ 146 | "class" => "EchoWorker", 147 | "function" => "perform", 148 | "queue" => "test", 149 | "jid" => "1082fd87-2508-4eb4-8fba-2958584a60e3", 150 | "args" => [], 151 | "retry_count" => 0, 152 | "enqueued_at" => 1_514_367_662, 153 | "finished_at" => nil, 154 | "failed_at" => nil, 155 | "retried_at" => nil, 156 | "error_message" => nil, 157 | "error_backtrace" => nil, 158 | "context" => %{request_id: 123} 159 | } 160 | end 161 | end 162 | -------------------------------------------------------------------------------- /test/flume/pipeline/event_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Pipeline.EventTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Pipeline.Event 5 | alias Flume.Redis 6 | 7 | @namespace Flume.Config.namespace() 8 | 9 | describe "pause/2" do 10 | test "does a permanent pipeline pause" do 11 | pipeline_name = :default_pipeline 12 | process_name = Event.Producer.process_name(pipeline_name) 13 | assert :ok == Event.pause(pipeline_name, async: false, temporary: false, timeout: 5000) 14 | 15 | assert match?( 16 | %{ 17 | state: %{ 18 | paused: true 19 | } 20 | }, 21 | :sys.get_state(process_name) 22 | ) 23 | 24 | assert Redis.Client.get!("#{@namespace}:pipeline:#{pipeline_name}:paused") == "true" 25 | end 26 | 27 | test "does not set a redis key for temporary pause" do 28 | pipeline_name = :default_pipeline 29 | process_name = Event.Producer.process_name(pipeline_name) 30 | assert :ok == Event.pause(pipeline_name, async: false, temporary: true, timeout: 6000) 31 | 32 | assert match?( 33 | %{ 34 | state: %{ 35 | paused: true 36 | } 37 | }, 38 | :sys.get_state(process_name) 39 | ) 40 | 41 | assert Redis.Client.get!("#{@namespace}:pipeline:#{pipeline_name}:paused") == nil 42 | end 43 | 44 | test "does an asynchronous and permanent pause" do 45 | pipeline_name = :default_pipeline 46 | assert :ok == Event.pause(pipeline_name, async: true, temporary: false, timeout: 5000) 47 | 48 | assert Redis.Client.get!("#{@namespace}:pipeline:#{pipeline_name}:paused") == "true" 49 | end 50 | end 51 | 52 | describe "resume/2" do 53 | test "does a permanent pipeline resume" do 54 | pipeline_name = :default_pipeline 55 | process_name = Event.Producer.process_name(pipeline_name) 56 | pause_key = "#{@namespace}:pipeline:#{pipeline_name}:paused" 57 | 58 | # Permanent pause 59 | assert :ok == Event.pause(pipeline_name, async: false, temporary: false, timeout: 5000) 60 | 61 | assert match?( 62 | %{ 63 | state: %{ 64 | paused: true 65 | } 66 | }, 67 | :sys.get_state(process_name) 68 | ) 69 | 70 | assert Redis.Client.get!(pause_key) == "true" 71 | 72 | # Permanent resume 73 | assert :ok == Event.resume(pipeline_name, async: false, temporary: false, timeout: 5000) 74 | 75 | assert match?( 76 | %{ 77 | state: %{ 78 | paused: false 79 | } 80 | }, 81 | :sys.get_state(process_name) 82 | ) 83 | 84 | assert Redis.Client.get!(pause_key) == nil 85 | end 86 | 87 | test "does not delete a redis key for a temporary resume" do 88 | pipeline_name = :default_pipeline 89 | process_name = Event.Producer.process_name(pipeline_name) 90 | pause_key = "#{@namespace}:pipeline:#{pipeline_name}:paused" 91 | 92 | # Permanent pause 93 | assert :ok == Event.pause(pipeline_name, async: true, temporary: false, timeout: 6000) 94 | 95 | assert Redis.Client.get!(pause_key) == "true" 96 | 97 | # Temporary resume 98 | assert :ok == Event.resume(pipeline_name, async: false, temporary: true, timeout: 6000) 99 | 100 | assert match?( 101 | %{ 102 | state: %{ 103 | paused: false 104 | } 105 | }, 106 | :sys.get_state(process_name) 107 | ) 108 | 109 | assert Redis.Client.get!(pause_key) == "true" 110 | end 111 | 112 | test "does an asynchronous and permanent pipeline resume" do 113 | pipeline_name = :default_pipeline 114 | pause_key = "#{@namespace}:pipeline:#{pipeline_name}:paused" 115 | 116 | # Permanent pause 117 | assert :ok == Event.pause(pipeline_name, async: true, temporary: false, timeout: 6000) 118 | 119 | assert Redis.Client.get!(pause_key) == "true" 120 | 121 | # Async permanent resume 122 | assert :ok == Event.resume(pipeline_name, async: true, temporary: false, timeout: 6000) 123 | 124 | assert Redis.Client.get!(pause_key) == nil 125 | end 126 | end 127 | end 128 | -------------------------------------------------------------------------------- /test/flume/pipeline_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.PipelineTest do 2 | use ExUnit.Case, async: true 3 | 4 | import Flume.Mock 5 | 6 | alias Flume.Pipeline 7 | 8 | describe "pause/2" do 9 | test "returns error for an invalid pipeline" do 10 | assert Pipeline.pause("invalid-pipeline", []) == 11 | {:error, "pipeline invalid-pipeline has not been configured"} 12 | 13 | assert Pipeline.pause(123, []) == {:error, "invalid value for a pipeline name"} 14 | end 15 | 16 | test "returns error for invalid options" do 17 | assert Pipeline.pause("default_pipeline", async: 0) == 18 | {:error, "expected :async to be a boolean, got: 0"} 19 | end 20 | 21 | test "crashes with an exit signal when the pipeline pause timeouts" do 22 | try do 23 | Pipeline.pause("default_pipeline", async: false, temporary: true, timeout: 0) 24 | catch 25 | :exit, error -> 26 | assert error == {:timeout, {GenServer, :call, [:default_pipeline_producer, :pause, 0]}} 27 | end 28 | end 29 | 30 | test "pauses a valid pipeline" do 31 | with_flume_mock do 32 | Pipeline.pause("default_pipeline", async: true, temporary: true, timeout: 300) 33 | 34 | assert_receive(%{ 35 | action: :pause, 36 | pipeline_name: "default_pipeline", 37 | options: [temporary: true, timeout: 300, async: true] 38 | }) 39 | end 40 | end 41 | end 42 | 43 | describe "resume/2" do 44 | test "returns error for an invalid pipeline" do 45 | assert Pipeline.resume("invalid-pipeline", []) == 46 | {:error, "pipeline invalid-pipeline has not been configured"} 47 | 48 | assert Pipeline.resume(123, []) == {:error, "invalid value for a pipeline name"} 49 | end 50 | 51 | test "returns error for invalid options" do 52 | assert Pipeline.resume("default_pipeline", temporary: 0) == 53 | {:error, "expected :temporary to be a boolean, got: 0"} 54 | end 55 | 56 | test "resumes a valid pipeline" do 57 | name = :default_pipeline 58 | process_name = Pipeline.Event.Producer.process_name(name) 59 | assert :ok == Pipeline.pause(name, async: false, temporary: true) 60 | 61 | assert match?( 62 | %{ 63 | state: %{ 64 | paused: true 65 | } 66 | }, 67 | :sys.get_state(process_name) 68 | ) 69 | 70 | assert :ok == Pipeline.resume(name, async: false, temporary: true, timeout: 4000) 71 | 72 | assert match?( 73 | %{ 74 | state: %{ 75 | paused: false 76 | } 77 | }, 78 | :sys.get_state(process_name) 79 | ) 80 | end 81 | 82 | test "crashes with an exit signal when the pipeline resume timeouts" do 83 | try do 84 | Pipeline.resume("default_pipeline", async: false, temporary: true, timeout: 0) 85 | catch 86 | :exit, error -> 87 | assert error == {:timeout, {GenServer, :call, [:default_pipeline_producer, :resume, 0]}} 88 | end 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /test/flume/queue/manager_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Queue.ManagerTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Queue.Manager 5 | alias Flume.Support.Time 6 | alias Flume.{Config, Event, JobFactory} 7 | alias Flume.Redis.{Client, Job, SortedSet} 8 | 9 | @namespace Config.namespace() 10 | 11 | def max_time_range do 12 | Flume.Queue.Backoff.calc_next_backoff(1) 13 | |> Flume.Support.Time.offset_from_now() 14 | |> Flume.Support.Time.time_to_score() 15 | end 16 | 17 | describe "enqueue/5" do 18 | test "enqueues a job into a queue" do 19 | assert {:ok, _} = Manager.enqueue(@namespace, "test", Worker, "process", [1]) 20 | end 21 | end 22 | 23 | describe "bulk_enqueue/3" do 24 | test "enqueues array of jobs into a queue" do 25 | assert {:ok, 2} = 26 | Manager.bulk_enqueue(@namespace, "test", [ 27 | [Worker, "process", [1]], 28 | [Worker, "process", [2]] 29 | ]) 30 | end 31 | end 32 | 33 | describe "enqueue_in/6" do 34 | test "enqueues a job at a scheduled time" do 35 | assert {:ok, _} = Manager.enqueue_in(@namespace, "test", 10, Worker, "process", [1]) 36 | end 37 | end 38 | 39 | describe "fetch_jobs/5" do 40 | test "fetch multiple jobs and queues it to a new list" do 41 | jobs = JobFactory.generate_jobs("Elixir.Worker", 10) 42 | 43 | Job.bulk_enqueue("#{@namespace}:queue:test", jobs) 44 | 45 | assert {:ok, jobs} == 46 | Manager.fetch_jobs(@namespace, "test", 10, %{ 47 | rate_limit_count: 1000, 48 | rate_limit_scale: 500 49 | }) 50 | 51 | assert 10 == Client.zcount!("#{@namespace}:queue:limit:test") 52 | end 53 | 54 | test "maintains rate-limit for a given key" do 55 | jobs = JobFactory.generate_jobs("Elixir.Worker", 10) 56 | 57 | Job.bulk_enqueue("#{@namespace}:queue:test", jobs) 58 | 59 | assert {:ok, jobs} == 60 | Manager.fetch_jobs( 61 | @namespace, 62 | "test", 63 | 10, 64 | %{rate_limit_count: 1000, rate_limit_scale: 500, rate_limit_key: "test"} 65 | ) 66 | 67 | assert 10 == Client.zcount!("#{@namespace}:limit:test") 68 | assert 0 == Client.zcount!("#{@namespace}:queue:limit:test") 69 | end 70 | 71 | test "dequeues multiple jobs from an empty queue" do 72 | assert {:ok, []} == 73 | Manager.fetch_jobs(@namespace, "test", 5, %{ 74 | rate_limit_count: 1000, 75 | rate_limit_scale: 500 76 | }) 77 | end 78 | end 79 | 80 | describe "Concurrent fetches" do 81 | test "fetches unique jobs" do 82 | count = 2 83 | jobs = JobFactory.generate_jobs("Elixir.Worker", count) 84 | 85 | Job.bulk_enqueue("#{@namespace}:queue:test", jobs) 86 | 87 | results = 88 | Enum.map(1..count, fn _ -> 89 | Task.async(fn -> 90 | {:ok, jobs} = 91 | Manager.fetch_jobs( 92 | @namespace, 93 | "test", 94 | 1, 95 | %{rate_limit_count: count, rate_limit_scale: 50000} 96 | ) 97 | 98 | jobs 99 | end) 100 | end) 101 | |> Enum.flat_map(&Task.await/1) 102 | 103 | assert 1 == length(results) 104 | assert 1 == Client.zcount!("#{@namespace}:queue:limit:test") 105 | end 106 | end 107 | 108 | describe "retry_or_fail_job/4" do 109 | test "adds job to retry queue by incrementing count" do 110 | job = 111 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958522a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 112 | 113 | SortedSet.add( 114 | "#{@namespace}:queue:processing:test", 115 | DateTime.utc_now() |> Time.time_to_score(), 116 | job 117 | ) 118 | 119 | Manager.retry_or_fail_job(@namespace, "test", job, "Failed") 120 | 121 | # make sure the job is removed from the backup queue 122 | assert [] = Client.zrange!("#{@namespace}:queue:processing:test") 123 | 124 | {:ok, [{"#{@namespace}:retry", [retried_job]}]} = 125 | Job.scheduled_jobs(["#{@namespace}:retry"], max_time_range()) 126 | 127 | retried_job = Flume.Event.decode!(retried_job) 128 | assert retried_job.jid == "1082fd87-2508-4eb4-8fba-2958522a60e3" 129 | end 130 | 131 | test "adds job to dead queue if count exceeds max retries" do 132 | job = 133 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2959994a60e3\",\"enqueued_at\":1514367662,\"args\":[1], \"retry_count\": 0}" 134 | 135 | SortedSet.add( 136 | "#{@namespace}:queue:processing:test", 137 | DateTime.utc_now() |> Time.time_to_score(), 138 | job 139 | ) 140 | 141 | Manager.retry_or_fail_job(@namespace, "test", job, "Failed") 142 | 143 | # make sure the job is removed from the backup queue 144 | assert [] = Client.zrange!("#{@namespace}:queue:processing:test") 145 | 146 | Enum.map(1..Config.max_retries(), fn _retry_count -> 147 | {:ok, [{"#{@namespace}:retry", [job_to_retry]}]} = 148 | Job.scheduled_jobs(["#{@namespace}:retry"], max_time_range()) 149 | 150 | Manager.retry_or_fail_job(@namespace, "test", job_to_retry, "Failed") 151 | end) 152 | 153 | # job will not be pushed to the retry queue 154 | assert {:ok, [{"#{@namespace}:retry", []}]} == 155 | Job.scheduled_jobs(["#{@namespace}:retry"], max_time_range()) 156 | 157 | {:ok, [{"#{@namespace}:dead", [job_in_dead_queue]}]} = 158 | Job.scheduled_jobs(["#{@namespace}:dead"], max_time_range()) 159 | 160 | job_in_dead_queue = Flume.Event.decode!(job_in_dead_queue) 161 | assert job_in_dead_queue.jid == "1082fd87-2508-4eb4-8fba-2959994a60e3" 162 | end 163 | end 164 | 165 | describe "remove_retry/3" do 166 | test "remove job from a retry queue" do 167 | queue = "#{@namespace}:retry" 168 | 169 | job = 170 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 171 | 172 | Job.schedule_job(queue, DateTime.utc_now() |> Time.unix_seconds(), job) 173 | 174 | assert {:ok, 1} == Manager.remove_retry(@namespace, job) 175 | assert {:ok, [{^queue, []}]} = Job.scheduled_jobs([queue], max_time_range()) 176 | end 177 | end 178 | 179 | describe "remove_processing/3" do 180 | test "removes a job from processing sorted-set" do 181 | serialized_job = 182 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1084fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 183 | 184 | Job.schedule_job( 185 | "#{@namespace}:queue:processing:test", 186 | Flume.Support.Time.unix_seconds(), 187 | serialized_job 188 | ) 189 | 190 | assert {:ok, 1} == Manager.remove_processing(@namespace, "test", serialized_job) 191 | end 192 | end 193 | 194 | describe "remove_and_enqueue_scheduled_jobs/3" do 195 | test "remove and enqueue scheduled jobs" do 196 | Job.schedule_job( 197 | "#{@namespace}:scheduled", 198 | DateTime.utc_now() |> Time.unix_seconds(), 199 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 200 | ) 201 | 202 | Job.schedule_job( 203 | "#{@namespace}:retry", 204 | DateTime.utc_now() |> Time.unix_seconds(), 205 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd97-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 206 | ) 207 | 208 | queue = "#{@namespace}:queue:test" 209 | 210 | assert {:ok, 2} == 211 | Manager.remove_and_enqueue_scheduled_jobs( 212 | @namespace, 213 | Time.time_to_score() 214 | ) 215 | 216 | jobs = [ 217 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 218 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd97-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 219 | ] 220 | 221 | assert jobs == Job.fetch_all!(queue) 222 | end 223 | end 224 | 225 | describe "enqueue_processing_jobs/2" do 226 | test "enqueues jobs to main queue if job's score in processing sorted-set is less than the current-score" do 227 | job = 228 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 229 | 230 | SortedSet.add( 231 | "#{@namespace}:queue:processing:test", 232 | DateTime.utc_now() |> Time.time_to_score(), 233 | job 234 | ) 235 | 236 | Client.query!(["SCRIPT", "FLUSH"]) 237 | assert {:ok, _} = Manager.enqueue_processing_jobs(@namespace, DateTime.utc_now(), "test", 1) 238 | 239 | assert [] = Client.zrange!("#{@namespace}:queue:processing:test") 240 | 241 | assert match?( 242 | %Event{jid: "1082fd87-2508-4eb4-8fba-2958584a60e3", enqueued_at: 1_514_367_662}, 243 | Client.lrange!("#{@namespace}:queue:test") |> List.first() |> Event.decode!() 244 | ) 245 | end 246 | 247 | test "enqueues only 1 job to main queue if job's score in processing sorted-set is less than the current-score" do 248 | job_1 = 249 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test_limit\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 250 | 251 | SortedSet.add( 252 | "#{@namespace}:queue:processing:test_limit", 253 | DateTime.utc_now() |> Time.time_to_score(), 254 | job_1 255 | ) 256 | 257 | job_2 = 258 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test_limit\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367771,\"args\":[1]}" 259 | 260 | SortedSet.add( 261 | "#{@namespace}:queue:processing:test_limit", 262 | Time.offset_from_now(1000) |> Time.time_to_score(), 263 | job_2 264 | ) 265 | 266 | Manager.enqueue_processing_jobs(@namespace, DateTime.utc_now(), "test_limit", 1) 267 | 268 | assert [^job_2] = Client.zrange!("#{@namespace}:queue:processing:test_limit") 269 | 270 | [event] = Client.lrange!("#{@namespace}:queue:test_limit") 271 | 272 | assert match?( 273 | %Event{jid: "1082fd87-2508-4eb4-8fba-2958584a60e3", enqueued_at: 1_514_367_662}, 274 | Event.decode!(event) 275 | ) 276 | end 277 | end 278 | 279 | describe "job_counts/2" do 280 | test "returns counts of the requested queues" do 281 | jobs = JobFactory.generate_jobs("Elixir.Worker", 10) 282 | 283 | [ 284 | queue_1, 285 | queue_2, 286 | _ 287 | ] = Enum.map(1..3, &"test_#{&1}") 288 | 289 | Enum.map(1..2, &Job.bulk_enqueue("#{@namespace}:queue:test_#{&1}", jobs)) 290 | 291 | assert { 292 | :ok, 293 | [ 294 | {queue_1, 10}, 295 | {queue_2, 10}, 296 | {"unknown-queue", 0} 297 | ] 298 | } == Manager.job_counts(@namespace, [queue_1, queue_2, "unknown-queue"]) 299 | end 300 | end 301 | end 302 | -------------------------------------------------------------------------------- /test/flume/redis/client_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.ClientTest do 2 | use Flume.TestWithRedis 3 | 4 | setup_all _redis do 5 | on_exit(fn -> 6 | clear_redis("flume") 7 | end) 8 | 9 | :ok 10 | end 11 | 12 | doctest Flume.Redis.Client 13 | end 14 | -------------------------------------------------------------------------------- /test/flume/redis/command_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.CommandTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Redis.Command 5 | 6 | describe "hdel/1" do 7 | test "returns list of HDEL commands" do 8 | hash_key_list = [{"hash_1", "key_1"}, {"hash_1", "key_2"}, {"hash_2", "key_1"}] 9 | 10 | assert Command.hdel(hash_key_list) == [ 11 | ["HDEL", "hash_1", "key_1", "key_2"], 12 | ["HDEL", "hash_2", "key_1"] 13 | ] 14 | end 15 | end 16 | 17 | describe "hdel/2" do 18 | test "returns HDEL command for list of keys to be deleted from a hash" do 19 | hash = "hash_1" 20 | keys = ["key_1", "key_2", "key_3"] 21 | 22 | assert Command.hdel(hash, keys) == ["HDEL", "hash_1", "key_1", "key_2", "key_3"] 23 | end 24 | 25 | test "returns HDEL command for a key to be deleted from a hash" do 26 | assert Command.hdel("hash", "key") == ["HDEL", "hash", "key"] 27 | end 28 | end 29 | 30 | describe "hscan/1" do 31 | test "returns list of HSCAN commands" do 32 | expected_commands = [ 33 | ["HSCAN", "hash_1", 0, "MATCH", "xyz"], 34 | ["HSCAN", "hash_2", 0], 35 | ["HSCAN", "hash_3", 0, "MATCH", "abc"] 36 | ] 37 | 38 | attr_list = [ 39 | ["hash_1", 0, "xyz"], 40 | ["hash_2", 0], 41 | ["hash_3", 0, "abc"] 42 | ] 43 | 44 | assert Command.hscan(attr_list) == expected_commands 45 | end 46 | end 47 | 48 | describe "hmget/1" do 49 | test "returns list of HMGET commands" do 50 | expected_commands = [ 51 | ["HMGET", "hash_1", "abc", "xyz"], 52 | ["HMGET", "hash_2", "xyz"], 53 | ["HMGET", "hash_3", "abc"] 54 | ] 55 | 56 | attr_list = [ 57 | {"hash_1", ["abc", "xyz"]}, 58 | {"hash_2", "xyz"}, 59 | {"hash_3", ["abc"]} 60 | ] 61 | 62 | assert Command.hmget(attr_list) == expected_commands 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /test/flume/redis/job_test.exs: -------------------------------------------------------------------------------- 1 | defmodule JobTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Config 5 | alias Flume.Redis.Job 6 | alias Flume.Support.Time, as: TimeExtension 7 | 8 | @namespace Config.namespace() 9 | @serialized_job "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1083fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 10 | 11 | describe "enqueue/2" do 12 | test "enqueues a job to a queue" do 13 | assert {:ok, _} = Job.enqueue("#{@namespace}:test", @serialized_job) 14 | end 15 | end 16 | 17 | describe "bulk_enqueue/3" do 18 | test "enqueues array of jobs to a queue" do 19 | assert {:ok, 2} = 20 | Job.bulk_enqueue("#{@namespace}:test", [ 21 | @serialized_job, 22 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1083fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[2]}" 23 | ]) 24 | end 25 | end 26 | 27 | describe "bulk_dequeue/3" do 28 | test "dequeues multiple jobs and queues it to new list" do 29 | jobs = [ 30 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1082fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 31 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1182fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 32 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1282fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 33 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1382fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 34 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1482fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 35 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1582fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 36 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1682fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 37 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1782fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 38 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1882fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}", 39 | "{\"class\":\"Elixir.Worker\",\"queue\":\"test\",\"jid\":\"1982fd87-2508-4eb4-8fba-2958584a60e3\",\"enqueued_at\":1514367662,\"args\":[1]}" 40 | ] 41 | 42 | Enum.map(jobs, fn job -> Job.enqueue("#{@namespace}:test", job) end) 43 | 44 | assert {:ok, jobs} == 45 | Job.bulk_dequeue( 46 | "#{@namespace}:test", 47 | "#{@namespace}:processing:test", 48 | 10, 49 | TimeExtension.time_to_score() 50 | ) 51 | end 52 | 53 | test "dequeues multiple jobs from an empty queue" do 54 | assert {:ok, []} == 55 | Job.bulk_dequeue( 56 | "#{@namespace}:test", 57 | "#{@namespace}:processing:test", 58 | 5, 59 | TimeExtension.time_to_score() 60 | ) 61 | end 62 | end 63 | 64 | describe "schedule_job/5" do 65 | test "schedules a job" do 66 | assert {:ok, 1} == 67 | Job.schedule_job( 68 | "#{@namespace}:test", 69 | DateTime.utc_now() |> TimeExtension.unix_seconds(), 70 | @serialized_job 71 | ) 72 | end 73 | end 74 | 75 | describe "bulk_enqueue_scheduled!/1" do 76 | def build_scheduled_queue_and_job(scheduled_queue, queue, job), 77 | do: {scheduled_queue, queue, job} 78 | 79 | def build_scheduled_queue_and_jobs(scheduled_queue, queue, count), 80 | do: Enum.map(1..count, &build_scheduled_queue_and_job(scheduled_queue, queue, "#{&1}")) 81 | 82 | test "groups scheduled queues_and_jobs by queue" do 83 | group1 = build_scheduled_queue_and_jobs("s1", "q1", 10) 84 | group2 = build_scheduled_queue_and_jobs("s2", "q2", 10) 85 | group3 = build_scheduled_queue_and_jobs("s3", "q3", 10) 86 | result = Job.bulk_enqueue_scheduled!(group1 ++ group2 ++ group3) 87 | 88 | assert length(result) == 30 89 | 90 | Enum.each(group1 ++ group2 ++ group3, fn {scheduled, _, job} -> 91 | refute Enum.find(result, fn {s, j} -> 92 | {s, j} == {scheduled, job} 93 | end) == nil 94 | end) 95 | end 96 | end 97 | 98 | describe "remove_job/3" do 99 | test "removes a job from a queue" do 100 | Job.enqueue("#{@namespace}:test", @serialized_job) 101 | 102 | assert 1 == Job.remove_job!("#{@namespace}:test", @serialized_job) 103 | end 104 | end 105 | 106 | describe "remove_retry_job/3" do 107 | test "removes a job from a retry queue" do 108 | Job.fail_job!("#{@namespace}:test", @serialized_job) 109 | 110 | assert 1 == Job.remove_scheduled_job!("#{@namespace}:test", @serialized_job) 111 | end 112 | end 113 | 114 | describe "fail_job/3" do 115 | test "adds a job to retry queue" do 116 | assert 1 == Job.fail_job!("#{@namespace}:test", @serialized_job) 117 | end 118 | end 119 | 120 | describe "fetch_all/2" do 121 | test "fetches all jobs from a list" do 122 | Job.enqueue("#{@namespace}:test", @serialized_job) 123 | 124 | assert [@serialized_job] == Job.fetch_all!("#{@namespace}:test") 125 | end 126 | end 127 | 128 | describe "scheduled_job/3" do 129 | test "returns scheduled jobs" do 130 | {:ok, _jid} = 131 | Job.schedule_job( 132 | "#{@namespace}:test", 133 | DateTime.utc_now() |> TimeExtension.unix_seconds(), 134 | @serialized_job 135 | ) 136 | 137 | {:ok, jobs} = 138 | Job.scheduled_jobs( 139 | ["#{@namespace}:test"], 140 | TimeExtension.time_to_score() 141 | ) 142 | 143 | assert [{"#{@namespace}:test", [@serialized_job]}] == jobs 144 | end 145 | end 146 | end 147 | -------------------------------------------------------------------------------- /test/flume/redis/lock_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.Redis.LockTest do 2 | use Flume.TestWithRedis 3 | 4 | alias Flume.Redis.Lock 5 | 6 | @namespace Flume.Config.namespace() 7 | @lock_key "#{@namespace}:acquire_lock_test" 8 | 9 | describe "acquire_lock/3" do 10 | test "mutual exclusion" do 11 | ttl = 10_000 12 | 13 | Enum.each(1..3, fn _ -> 14 | processes = 1..10 15 | 16 | locks_acquired = 17 | Enum.map(processes, fn _ -> 18 | Task.async(fn -> 19 | Lock.acquire(@lock_key, ttl) 20 | end) 21 | end) 22 | |> Enum.map(&Task.await(&1, 1000)) 23 | |> Keyword.delete(:error) 24 | 25 | assert length(locks_acquired) == 1 26 | assert Lock.release(@lock_key, locks_acquired[:ok]) == :ok 27 | end) 28 | end 29 | 30 | test "lock expires after ttl" do 31 | ttl = 1000 32 | {:ok, _} = Lock.acquire(@lock_key, ttl) 33 | Process.sleep(1100) 34 | {:ok, token} = Lock.acquire(@lock_key, ttl) 35 | assert Lock.release(@lock_key, token) == :ok 36 | end 37 | 38 | test "one process cannot clean another process's lock" do 39 | ttl = 10_000 40 | {:ok, _token} = Lock.acquire(@lock_key, ttl) 41 | assert Lock.release(@lock_key, "other_token") == :ok 42 | assert {:error, :locked} = Lock.acquire(@lock_key, ttl) 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /test/flume/utils_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Flume.UtilsTest do 2 | use ExUnit.Case 3 | 4 | alias Flume.Utils 5 | 6 | describe "safe_apply/2" do 7 | test "returns result correctly" do 8 | response = "Test" 9 | result = Utils.safe_apply(fn -> response end, 1000) 10 | assert {:ok, response} == result 11 | end 12 | 13 | test "allows processes to crash safely" do 14 | error_msg = "Test" 15 | result = Utils.safe_apply(fn -> raise error_msg end, 1000) 16 | assert {:exit, {foo, _stack}} = result 17 | assert error_msg == foo.message 18 | end 19 | 20 | test "times out processes" do 21 | result = Utils.safe_apply(fn -> Process.sleep(1200) end, 1000) 22 | assert {:timeout, _} = result 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /test/flume_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FlumeTest do 2 | use Flume.TestWithRedis 3 | 4 | import Flume.Mock 5 | 6 | alias Flume.Redis.Job 7 | alias Flume.{Redis, Config, Pipeline, JobFactory} 8 | alias Flume.Pipeline.Event.{ProducerConsumer, Consumer, Producer} 9 | 10 | @namespace Config.namespace() 11 | 12 | describe "pending_jobs_count/0" do 13 | defmodule TestWorker do 14 | def start_link(pipeline, event) do 15 | Task.start_link(__MODULE__, :run, [pipeline, event]) 16 | end 17 | 18 | def run(_pipeline, event) do 19 | %{"args" => [caller_name]} = Jason.decode!(event) 20 | caller_name = caller_name |> String.to_atom() 21 | send(caller_name, {:hello, self()}) 22 | 23 | receive do 24 | {:ack, pid} -> 25 | send(pid, :done) 26 | end 27 | end 28 | end 29 | 30 | test "returns pending jobs count" do 31 | pipeline = %Pipeline{ 32 | name: "test_pipeline", 33 | queue: "test", 34 | max_demand: 1000 35 | } 36 | 37 | caller_name = :test_process 38 | Process.register(self(), caller_name) 39 | 40 | {:ok, producer} = Producer.start_link(pipeline) 41 | {:ok, _} = ProducerConsumer.start_link(pipeline) 42 | {:ok, _} = Consumer.start_link(pipeline, TestWorker) 43 | 44 | # Push events to Redis 45 | Job.enqueue( 46 | "#{@namespace}:queue:#{pipeline.queue}", 47 | JobFactory.generate("TestWorker", [caller_name]) 48 | ) 49 | 50 | receive do 51 | {:hello, pid} -> 52 | assert 1 == Flume.pending_jobs_count([:test_pipeline]) 53 | send(pid, {:ack, caller_name}) 54 | end 55 | 56 | GenStage.stop(producer) 57 | end 58 | end 59 | 60 | @tag :slow_test 61 | describe "regular pipelines" do 62 | test "pulls max_demand events from redis" do 63 | max_demand = 2 64 | sleep_time = 500 65 | 66 | pipeline = %Pipeline{ 67 | name: "test_pipeline", 68 | queue: "test", 69 | max_demand: max_demand 70 | } 71 | 72 | caller_name = :test_process 73 | Process.register(self(), caller_name) 74 | 75 | # Push events to Redis 76 | Enum.each(1..4, fn _ -> 77 | Job.enqueue( 78 | "#{@namespace}:queue:#{pipeline.queue}", 79 | JobFactory.generate("TestWorker") 80 | ) 81 | end) 82 | 83 | {:ok, producer} = Producer.start_link(pipeline) 84 | {:ok, producer_consumer} = ProducerConsumer.start_link(pipeline) 85 | 86 | {:ok, _} = 87 | EchoConsumerWithTimestamp.start_link(%{ 88 | upstream: producer_consumer, 89 | owner: caller_name, 90 | max_demand: max_demand, 91 | sleep_time: sleep_time 92 | }) 93 | 94 | assert_receive {:received, events, received_time_1}, 4_000 95 | assert length(events) == 2 96 | 97 | assert_receive {:received, events, received_time_2}, 4_000 98 | assert length(events) == 2 99 | 100 | assert received_time_2 > received_time_1 101 | 102 | GenStage.stop(producer) 103 | end 104 | end 105 | 106 | @tag :slow_test 107 | describe "rate-limited pipelines" do 108 | test "pulls max_demand events from redis within the rate-limit-scale" do 109 | max_demand = 10 110 | sleep_time = 500 111 | 112 | pipeline = %Pipeline{ 113 | name: "test_pipeline", 114 | queue: "test", 115 | max_demand: max_demand, 116 | rate_limit_count: 2, 117 | rate_limit_scale: 1000 118 | } 119 | 120 | caller_name = :test_process 121 | Process.register(self(), caller_name) 122 | 123 | {:ok, producer} = Producer.start_link(pipeline) 124 | {:ok, producer_consumer} = ProducerConsumer.start_link(pipeline) 125 | 126 | {:ok, _} = 127 | EchoConsumerWithTimestamp.start_link(%{ 128 | upstream: producer_consumer, 129 | owner: caller_name, 130 | max_demand: max_demand, 131 | sleep_time: sleep_time 132 | }) 133 | 134 | # Push events to Redis 135 | jobs = JobFactory.generate_jobs("Elixir.Worker", 10) 136 | Job.bulk_enqueue("#{@namespace}:queue:#{pipeline.queue}", jobs) 137 | 138 | assert_receive {:received, events, received_time_1}, 4_000 139 | assert length(events) == 2 140 | 141 | assert_receive {:received, events, received_time_2}, 4_000 142 | assert length(events) == 2 143 | assert received_time_2 > received_time_1 144 | 145 | assert_receive {:received, events, received_time_3}, 4_000 146 | assert received_time_3 > received_time_2 147 | assert length(events) == 2 148 | 149 | assert_receive {:received, events, received_time_4}, 4_000 150 | assert received_time_4 > received_time_3 151 | assert length(events) == 2 152 | 153 | assert_receive {:received, events, received_time_5}, 4_000 154 | assert received_time_5 > received_time_4 155 | assert length(events) == 2 156 | 157 | GenStage.stop(producer) 158 | end 159 | end 160 | 161 | describe "enqueue/4" do 162 | test "mock works" do 163 | with_flume_mock do 164 | Flume.enqueue(:test, List, :last, [[1]]) 165 | 166 | assert_receive %{ 167 | queue: :test, 168 | worker: List, 169 | function_name: :last, 170 | args: [[1]] 171 | } 172 | end 173 | end 174 | end 175 | 176 | describe "enqueue_in/5" do 177 | test "mock works" do 178 | with_flume_mock do 179 | Flume.enqueue_in(:test, 10, List, :last, [[1]]) 180 | 181 | assert_receive %{ 182 | schedule_in: 10, 183 | queue: :test, 184 | worker: List, 185 | function_name: :last, 186 | args: [[1]] 187 | } 188 | end 189 | end 190 | end 191 | 192 | describe "bulk_enqueue/4" do 193 | test "mock works" do 194 | with_flume_mock do 195 | Flume.bulk_enqueue( 196 | :test, 197 | [ 198 | [List, "last", [[1]]], 199 | [List, "last", [[2, 3]]] 200 | ] 201 | ) 202 | 203 | assert_receive %{ 204 | queue: :test, 205 | jobs: [ 206 | [List, "last", [[1]]], 207 | [List, "last", [[2, 3]]] 208 | ] 209 | } 210 | end 211 | end 212 | end 213 | 214 | test "pipelines/0" do 215 | assert Flume.pipelines() == [%{name: "default_pipeline", queue: "default", max_demand: 1000}] 216 | end 217 | 218 | describe "pause_all/1" do 219 | test "pauses all the pipelines temporarily" do 220 | assert Flume.pause_all(async: false, temporary: true) == [:ok] 221 | 222 | Enum.each(Flume.Config.pipeline_names(), fn pipeline_name -> 223 | process_name = Pipeline.Event.Producer.process_name(pipeline_name) 224 | 225 | assert match?( 226 | %{ 227 | state: %{ 228 | paused: true 229 | } 230 | }, 231 | :sys.get_state(process_name) 232 | ) 233 | 234 | assert Redis.Client.get!("flume_test:pipeline:#{pipeline_name}:paused") == nil 235 | end) 236 | end 237 | 238 | test "pauses all the pipelines permanently" do 239 | assert Flume.pause_all(async: false, temporary: false) == [:ok] 240 | 241 | Enum.each(Flume.Config.pipeline_names(), fn pipeline_name -> 242 | process_name = Pipeline.Event.Producer.process_name(pipeline_name) 243 | 244 | assert match?( 245 | %{ 246 | state: %{ 247 | paused: true 248 | } 249 | }, 250 | :sys.get_state(process_name) 251 | ) 252 | 253 | assert Redis.Client.get!("flume_test:pipeline:#{pipeline_name}:paused") == "true" 254 | end) 255 | end 256 | end 257 | 258 | describe "resume_all/2" do 259 | test "resumes all the pipelines temporarily" do 260 | assert Flume.pause_all(async: false, temporary: false) == [:ok] 261 | assert Flume.resume_all(async: false, temporary: true) == [:ok] 262 | 263 | Enum.each(Flume.Config.pipeline_names(), fn pipeline_name -> 264 | process_name = Pipeline.Event.Producer.process_name(pipeline_name) 265 | 266 | assert match?( 267 | %{ 268 | state: %{ 269 | paused: false 270 | } 271 | }, 272 | :sys.get_state(process_name) 273 | ) 274 | 275 | assert Redis.Client.get!("flume_test:pipeline:#{pipeline_name}:paused") == "true" 276 | end) 277 | end 278 | 279 | test "resumes all the pipelines permanently" do 280 | assert Flume.pause_all(async: false, temporary: false) == [:ok] 281 | assert Flume.resume_all(async: false, temporary: false) == [:ok] 282 | 283 | Enum.each(Flume.Config.pipeline_names(), fn pipeline_name -> 284 | process_name = Pipeline.Event.Producer.process_name(pipeline_name) 285 | 286 | assert match?( 287 | %{ 288 | state: %{ 289 | paused: false 290 | } 291 | }, 292 | :sys.get_state(process_name) 293 | ) 294 | 295 | assert Redis.Client.get!("flume_test:pipeline:#{pipeline_name}:paused") == nil 296 | end) 297 | end 298 | end 299 | end 300 | -------------------------------------------------------------------------------- /test/support/echo_consumer.ex: -------------------------------------------------------------------------------- 1 | defmodule EchoConsumer do 2 | use GenStage 3 | 4 | def start_link(producer, owner, options) do 5 | GenStage.start_link( 6 | __MODULE__, 7 | %{ 8 | producer: producer, 9 | owner: owner, 10 | max_demand: Keyword.get(options, :max_demand, 1), 11 | min_demand: Keyword.get(options, :min_demand, 0) 12 | }, 13 | name: Keyword.fetch!(options, :name) 14 | ) 15 | end 16 | 17 | def init(%{producer: producer, owner: owner, min_demand: min_demand, max_demand: max_demand}) do 18 | {:consumer, owner, subscribe_to: [{producer, min_demand: min_demand, max_demand: max_demand}]} 19 | end 20 | 21 | def handle_events(events, _, owner) do 22 | send(owner, {:received, events}) 23 | {:noreply, [], owner} 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /test/support/echo_consumer_with_timestamp.ex: -------------------------------------------------------------------------------- 1 | defmodule EchoConsumerWithTimestamp do 2 | use GenStage 3 | 4 | alias Flume.Support.Time, as: TimeExtension 5 | 6 | def start_link(state \\ %{}) do 7 | GenStage.start_link(__MODULE__, state) 8 | end 9 | 10 | def init(state) do 11 | {:consumer, state, 12 | subscribe_to: [{state.upstream, min_demand: 0, max_demand: state.max_demand}]} 13 | end 14 | 15 | def handle_events(events, _, state) do 16 | Process.sleep(state.sleep_time) 17 | send(state.owner, {:received, events, TimeExtension.unix_seconds()}) 18 | {:noreply, [], state} 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/support/echo_context_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule EchoContextWorker do 2 | def perform([[caller]]), do: send_back_context(caller, :context) 3 | def perform(caller), do: String.to_atom(caller) |> send_back_context(:context) 4 | 5 | defp send_back_context(caller, scope), do: send(caller, {scope, Flume.worker_context()}) 6 | end 7 | -------------------------------------------------------------------------------- /test/support/echo_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule EchoWorker do 2 | use GenServer 3 | 4 | def start_link(opts \\ []) do 5 | GenServer.start_link(__MODULE__, opts, name: __MODULE__) 6 | end 7 | 8 | def init(opts \\ []) do 9 | {:ok, opts} 10 | end 11 | 12 | def perform(owner, message) do 13 | send(String.to_atom(owner), {:received, message}) 14 | end 15 | 16 | def perform(args) when is_list(args) do 17 | args 18 | |> Enum.each(fn [owner, message] -> 19 | send(owner, {:received, message}) 20 | end) 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/support/test_producer.ex: -------------------------------------------------------------------------------- 1 | defmodule TestProducer do 2 | use GenStage 3 | 4 | alias Flume.Config 5 | alias Flume.Queue.Manager, as: QueueManager 6 | 7 | # Client API 8 | def start_link(%{process_name: process_name, queue: _queue} = state) do 9 | GenStage.start_link(__MODULE__, state, name: String.to_atom(process_name)) 10 | end 11 | 12 | # Server callbacks 13 | def init(state) do 14 | {:producer, state} 15 | end 16 | 17 | def handle_demand(demand, state) when demand > 0 do 18 | {_count, events} = take(demand, state.queue) 19 | 20 | {:noreply, events, state} 21 | end 22 | 23 | def handle_call({:consumer_done, _val}, _from, state) do 24 | {:reply, :ok, [], state} 25 | end 26 | 27 | # Private API 28 | defp take(demand, queue_name) do 29 | events = 30 | case QueueManager.fetch_jobs(Config.namespace(), queue_name, demand) do 31 | {:error, _error} -> 32 | [] 33 | 34 | {:ok, events} -> 35 | events 36 | end 37 | 38 | count = length(events) 39 | 40 | {count, events} 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /test/support/test_with_redis.ex: -------------------------------------------------------------------------------- 1 | defmodule Flume.TestWithRedis do 2 | use ExUnit.CaseTemplate, async: true 3 | 4 | using do 5 | quote do 6 | alias Flume.Config 7 | 8 | setup _tags do 9 | clear_redis() 10 | 11 | on_exit(fn -> 12 | clear_redis() 13 | end) 14 | 15 | :ok 16 | end 17 | 18 | def clear_redis(namespace \\ Config.namespace()) do 19 | pool_size = Config.redis_pool_size() 20 | conn = :"#{Flume.Redis.Supervisor.redix_worker_prefix()}_#{pool_size - 1}" 21 | keys = Redix.command!(conn, ["KEYS", "#{namespace}:*"]) 22 | 23 | Enum.map(keys, fn key -> ["DEL", key] end) 24 | |> case do 25 | [] -> 26 | [] 27 | 28 | commands -> 29 | Redix.pipeline(conn, commands) 30 | end 31 | end 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | # Start flume supervision tree 3 | Flume.Supervisor.start_link() 4 | --------------------------------------------------------------------------------