├── priv └── plts │ └── .gitkeep ├── .github ├── FUNDING.yml ├── renovate.json ├── dependabot.yml └── workflows │ ├── branch_main.yml │ ├── pr.yml │ ├── tag-beta.yml │ ├── tag-stable.yml │ ├── part_dependabot.yml │ ├── part_release.yml │ ├── part_docs.yml │ └── part_test.yml ├── .tool-versions ├── test ├── quantum_test.exs ├── test_helper.exs ├── support │ ├── capture_log_extend.ex │ ├── test_producer.ex │ ├── test_consumer.ex │ └── test_storage.ex ├── quantum │ ├── job_test.exs │ ├── run_strategy_test.exs │ ├── task_registry_test.exs │ ├── node_selector_broadcaster_test.exs │ ├── clock_broadcaster_test.exs │ ├── normalizer_test.exs │ ├── scheduler_test.exs │ ├── execution_broadcaster_test.exs │ ├── executor_test.exs │ └── job_broadcaster_test.exs └── quantum_startup_test.exs ├── .formatter.exs ├── config ├── config.exs └── .credo.exs ├── lib ├── quantum │ ├── task_registry │ │ ├── init_opts.ex │ │ ├── start_opts.ex │ │ └── state.ex │ ├── execution_broadcaster │ │ ├── event.ex │ │ ├── init_opts.ex │ │ ├── state.ex │ │ └── start_opts.ex │ ├── clock_broadcaster │ │ ├── event.ex │ │ ├── state.ex │ │ ├── init_opts.ex │ │ └── start_opts.ex │ ├── node_selector_broadcaster │ │ ├── event.ex │ │ ├── state.ex │ │ ├── init_opts.ex │ │ └── start_opts.ex │ ├── job_broadcaster │ │ ├── init_opts.ex │ │ ├── start_opts.ex │ │ └── state.ex │ ├── executor │ │ └── start_opts.ex │ ├── run_strategy.ex │ ├── executor_supervisor │ │ ├── init_opts.ex │ │ └── start_opts.ex │ ├── run_strategy │ │ ├── local.ex │ │ ├── all.ex │ │ └── random.ex │ ├── storage │ │ └── noop.ex │ ├── executor_supervisor.ex │ ├── task_registry.ex │ ├── storage.ex │ ├── node_selector_broadcaster.ex │ ├── clock_broadcaster.ex │ ├── supervisor.ex │ ├── executor.ex │ ├── normalizer.ex │ ├── job.ex │ ├── execution_broadcaster.ex │ └── job_broadcaster.ex └── quantum.ex ├── .gitignore ├── dialyzer.ignore-warnings ├── assets └── quantum-elixir-logo.svg ├── pages ├── supervision-tree.md ├── run-strategies.md ├── runtime-configuration.md ├── crontab-format.md └── configuration.md ├── mix.exs ├── README.md └── LICENSE /priv/plts/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: maennchen 2 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | erlang 28.3 2 | elixir 1.19.4 3 | -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "enabledManagers": ["asdf"] 3 | } -------------------------------------------------------------------------------- /test/quantum_test.exs: -------------------------------------------------------------------------------- 1 | defmodule QuantumTest do 2 | use ExUnit.Case 3 | doctest Quantum 4 | end 5 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | Application.ensure_all_started(:quantum) 2 | 3 | ExUnit.start(capture_log: true) 4 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :logger, :console, metadata: [:all, :crash_reason] 4 | config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase 5 | -------------------------------------------------------------------------------- /lib/quantum/task_registry/init_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TaskRegistry.InitOpts do 2 | @moduledoc false 3 | 4 | # Init Options for Quantum.TaskRegistry 5 | 6 | @type t :: %__MODULE__{} 7 | 8 | @enforce_keys [] 9 | defstruct @enforce_keys 10 | end 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: mix 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | - package-ecosystem: github-actions 9 | directory: "/" 10 | schedule: 11 | interval: daily 12 | open-pull-requests-limit: 10 13 | -------------------------------------------------------------------------------- /lib/quantum/execution_broadcaster/event.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutionBroadcaster.Event do 2 | @moduledoc false 3 | 4 | # Execute Event 5 | 6 | alias Quantum.Job 7 | 8 | @type t :: %__MODULE__{ 9 | job: Job.t() 10 | } 11 | 12 | @enforce_keys [:job] 13 | 14 | defstruct @enforce_keys 15 | end 16 | -------------------------------------------------------------------------------- /lib/quantum/clock_broadcaster/event.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ClockBroadcaster.Event do 2 | @moduledoc false 3 | 4 | # Clock Event 5 | 6 | @type t :: %__MODULE__{ 7 | time: DateTime.t(), 8 | catch_up: boolean() 9 | } 10 | 11 | @enforce_keys [:time, :catch_up] 12 | 13 | defstruct @enforce_keys 14 | end 15 | -------------------------------------------------------------------------------- /.github/workflows/branch_main.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - "main" 5 | workflow_dispatch: {} 6 | 7 | name: "Main Branch" 8 | 9 | jobs: 10 | test: 11 | name: "Test" 12 | 13 | uses: ./.github/workflows/part_test.yml 14 | 15 | docs: 16 | name: "Docs" 17 | 18 | uses: ./.github/workflows/part_docs.yml 19 | -------------------------------------------------------------------------------- /lib/quantum/node_selector_broadcaster/event.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NodeSelectorBroadcaster.Event do 2 | @moduledoc false 3 | 4 | # Execute Event 5 | 6 | alias Quantum.Job 7 | 8 | @type t :: %__MODULE__{ 9 | job: Job.t(), 10 | node: Node.t() 11 | } 12 | 13 | @enforce_keys [:job, :node] 14 | 15 | defstruct @enforce_keys 16 | end 17 | -------------------------------------------------------------------------------- /lib/quantum/task_registry/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TaskRegistry.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options for Quantum.TaskRegistry 5 | 6 | @type t :: %__MODULE__{ 7 | name: GenServer.server(), 8 | listeners: [atom] 9 | } 10 | 11 | @enforce_keys [:name] 12 | defstruct @enforce_keys ++ [listeners: []] 13 | end 14 | -------------------------------------------------------------------------------- /lib/quantum/task_registry/state.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TaskRegistry.State do 2 | @moduledoc false 3 | 4 | # State of Quantum.TaskRegistry 5 | 6 | alias Quantum.Job 7 | 8 | @type t :: %__MODULE__{ 9 | running_tasks: %{optional(Job.name()) => [Node.t()]} 10 | } 11 | 12 | @enforce_keys [:running_tasks] 13 | defstruct @enforce_keys 14 | end 15 | -------------------------------------------------------------------------------- /lib/quantum/node_selector_broadcaster/state.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NodeSelectorBroadcaster.State do 2 | @moduledoc false 3 | 4 | # Internal State 5 | 6 | @type t :: %__MODULE__{ 7 | task_supervisor_reference: GenServer.server() 8 | } 9 | 10 | @enforce_keys [ 11 | :task_supervisor_reference 12 | ] 13 | 14 | defstruct @enforce_keys 15 | end 16 | -------------------------------------------------------------------------------- /lib/quantum/clock_broadcaster/state.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ClockBroadcaster.State do 2 | @moduledoc false 3 | 4 | # Internal State 5 | 6 | @type t :: %__MODULE__{ 7 | debug_logging: boolean(), 8 | time: DateTime.t(), 9 | remaining_demand: non_neg_integer 10 | } 11 | 12 | @enforce_keys [:debug_logging, :time, :remaining_demand] 13 | 14 | defstruct @enforce_keys 15 | end 16 | -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | branches: 4 | - "*" 5 | workflow_dispatch: {} 6 | 7 | name: "Pull Request" 8 | 9 | jobs: 10 | test: 11 | name: "Test" 12 | 13 | uses: ./.github/workflows/part_test.yml 14 | 15 | docs: 16 | name: "Docs" 17 | 18 | uses: ./.github/workflows/part_docs.yml 19 | 20 | dependabot: 21 | name: "Dependabot" 22 | 23 | uses: ./.github/workflows/part_dependabot.yml -------------------------------------------------------------------------------- /test/support/capture_log_extend.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.CaptureLogExtend do 2 | @moduledoc false 3 | 4 | import ExUnit.CaptureLog 5 | 6 | def capture_log_with_return(fun) do 7 | ref = make_ref() 8 | 9 | logs = 10 | capture_log(fn -> 11 | return = fun.() 12 | send(self(), {:return, ref, return}) 13 | end) 14 | 15 | receive do 16 | {:return, ^ref, return} -> 17 | {return, logs} 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/quantum/clock_broadcaster/init_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ClockBroadcaster.InitOpts do 2 | @moduledoc false 3 | 4 | # Init Options 5 | 6 | alias Quantum.{Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | start_time: DateTime.t(), 10 | storage: Storage, 11 | scheduler: Scheduler, 12 | debug_logging: boolean() 13 | } 14 | 15 | @enforce_keys [:start_time, :storage, :scheduler, :debug_logging] 16 | 17 | defstruct @enforce_keys 18 | end 19 | -------------------------------------------------------------------------------- /lib/quantum/job_broadcaster/init_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.JobBroadcaster.InitOpts do 2 | @moduledoc false 3 | 4 | # Init Options for Quantum.JobBroadcaster 5 | 6 | alias Quantum.{Job, Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | jobs: [Job.t()], 10 | storage: Storage, 11 | scheduler: Scheduler, 12 | debug_logging: boolean 13 | } 14 | 15 | @enforce_keys [:jobs, :storage, :scheduler, :debug_logging] 16 | defstruct @enforce_keys 17 | end 18 | -------------------------------------------------------------------------------- /lib/quantum/node_selector_broadcaster/init_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NodeSelectorBroadcaster.InitOpts do 2 | @moduledoc false 3 | 4 | # Init Options for Quantum.NodeSelectorBroadcaster 5 | 6 | @type t :: %__MODULE__{ 7 | execution_broadcaster_reference: GenServer.server(), 8 | task_supervisor_reference: GenServer.server() 9 | } 10 | 11 | @enforce_keys [ 12 | :execution_broadcaster_reference, 13 | :task_supervisor_reference 14 | ] 15 | defstruct @enforce_keys 16 | end 17 | -------------------------------------------------------------------------------- /.github/workflows/tag-beta.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" 5 | workflow_dispatch: {} 6 | 7 | name: "Beta Tag" 8 | 9 | jobs: 10 | release: 11 | name: "Release" 12 | 13 | uses: ./.github/workflows/part_release.yml 14 | with: 15 | releaseName: "${{ github.ref_name }}" 16 | 17 | docs: 18 | name: "Docs" 19 | 20 | needs: ['release'] 21 | 22 | uses: ./.github/workflows/part_docs.yml 23 | with: 24 | releaseName: "${{ github.ref_name }}" 25 | 26 | -------------------------------------------------------------------------------- /.github/workflows/tag-stable.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | tags: 4 | - "v[0-9]+.[0-9]+.[0-9]+" 5 | workflow_dispatch: {} 6 | 7 | name: "Stable Tag" 8 | 9 | jobs: 10 | release: 11 | name: "Release" 12 | 13 | uses: ./.github/workflows/part_release.yml 14 | with: 15 | releaseName: "${{ github.ref_name }}" 16 | stable: true 17 | 18 | docs: 19 | name: "Docs" 20 | 21 | needs: ['release'] 22 | 23 | uses: ./.github/workflows/part_docs.yml 24 | with: 25 | releaseName: "${{ github.ref_name }}" 26 | -------------------------------------------------------------------------------- /lib/quantum/clock_broadcaster/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ClockBroadcaster.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options 5 | 6 | alias Quantum.{Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | name: GenServer.server(), 10 | start_time: DateTime.t(), 11 | storage: Storage, 12 | scheduler: Scheduler, 13 | debug_logging: boolean() 14 | } 15 | 16 | @enforce_keys [:start_time, :name, :storage, :scheduler, :debug_logging] 17 | 18 | defstruct @enforce_keys 19 | end 20 | -------------------------------------------------------------------------------- /lib/quantum/job_broadcaster/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.JobBroadcaster.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options for Quantum.JobBroadcaster 5 | 6 | alias Quantum.{Job, Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | name: GenServer.server(), 10 | jobs: [Job.t()], 11 | storage: Storage, 12 | scheduler: Scheduler, 13 | debug_logging: boolean 14 | } 15 | 16 | @enforce_keys [:name, :jobs, :storage, :scheduler, :debug_logging] 17 | defstruct @enforce_keys 18 | end 19 | -------------------------------------------------------------------------------- /.github/workflows/part_dependabot.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: {} 3 | 4 | name: "Dependabot" 5 | 6 | jobs: 7 | automerge_dependabot: 8 | name: "Automerge PRs" 9 | 10 | runs-on: ubuntu-latest 11 | 12 | permissions: 13 | pull-requests: write 14 | contents: write 15 | 16 | steps: 17 | - uses: fastify/github-action-merge-dependabot@v3.11.2 18 | with: 19 | github-token: ${{ github.token }} 20 | use-github-auto-merge: true 21 | # Major Updates need to be merged manually 22 | target: minor 23 | -------------------------------------------------------------------------------- /lib/quantum/executor/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Executor.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options for Quantum.Executor 5 | 6 | @type t :: %__MODULE__{ 7 | task_supervisor_reference: GenServer.server(), 8 | task_registry_reference: GenServer.server(), 9 | debug_logging: boolean, 10 | scheduler: atom() 11 | } 12 | 13 | @enforce_keys [ 14 | :task_supervisor_reference, 15 | :task_registry_reference, 16 | :debug_logging, 17 | :scheduler 18 | ] 19 | defstruct @enforce_keys 20 | end 21 | -------------------------------------------------------------------------------- /lib/quantum/node_selector_broadcaster/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NodeSelectorBroadcaster.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options for Quantum.NodeSelectorBroadcaster 5 | 6 | @type t :: %__MODULE__{ 7 | name: GenServer.server(), 8 | execution_broadcaster_reference: GenServer.server(), 9 | task_supervisor_reference: GenServer.server() 10 | } 11 | 12 | @enforce_keys [ 13 | :name, 14 | :execution_broadcaster_reference, 15 | :task_supervisor_reference 16 | ] 17 | defstruct @enforce_keys 18 | end 19 | -------------------------------------------------------------------------------- /config/.credo.exs: -------------------------------------------------------------------------------- 1 | %{ 2 | configs: [ 3 | %{ 4 | name: "default", 5 | files: %{ 6 | included: ["lib/", "test/"], 7 | excluded: [] 8 | }, 9 | checks: %{ 10 | enabled: [ 11 | # For others you can also set parameters 12 | {Credo.Check.Readability.MaxLineLength, priority: :low, max_length: 120}, 13 | {Credo.Check.Design.TagTODO, exit_status: 0} 14 | ], 15 | disabled: [ 16 | {Credo.Check.Warning.MissedMetadataKeyInLoggerConfig, []} 17 | ] 18 | } 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /test/support/test_producer.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TestProducer do 2 | @moduledoc false 3 | 4 | use GenStage 5 | 6 | def start_link do 7 | GenStage.start_link(__MODULE__, nil) 8 | end 9 | 10 | def child_spec(_) do 11 | %{super([]) | start: {__MODULE__, :start_link, []}} 12 | end 13 | 14 | def handle_demand(_demand, state) do 15 | {:noreply, [], state} 16 | end 17 | 18 | def send(stage, message) do 19 | GenStage.cast(stage, message) 20 | end 21 | 22 | def init(_) do 23 | {:producer, nil} 24 | end 25 | 26 | def handle_cast(message, state) do 27 | {:noreply, [message], state} 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/quantum/job_broadcaster/state.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.JobBroadcaster.State do 2 | @moduledoc false 3 | 4 | # State of Quantum.JobBroadcaster 5 | 6 | alias Quantum.{Job, JobBroadcaster, Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | jobs: %{optional(Job.name()) => Job.t()}, 10 | buffer: JobBroadcaster.event(), 11 | storage: Storage, 12 | storage_pid: Storage.storage_pid(), 13 | scheduler: Scheduler, 14 | debug_logging: boolean() 15 | } 16 | 17 | @enforce_keys [:jobs, :buffer, :storage, :storage_pid, :scheduler, :debug_logging] 18 | defstruct @enforce_keys 19 | end 20 | -------------------------------------------------------------------------------- /lib/quantum/run_strategy.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.RunStrategy do 2 | @moduledoc """ 3 | Config Normalizer of a `Quantum.RunStrategy.NodeList`. 4 | """ 5 | 6 | @doc """ 7 | Normalize given config to a value that has `Quantum.RunStrategy.NodeList` implemented. 8 | 9 | Raise / Do not Match on invalid config. 10 | """ 11 | @callback normalize_config!(any) :: any 12 | 13 | defprotocol NodeList do 14 | @moduledoc """ 15 | Strategy to run Jobs over nodes 16 | """ 17 | 18 | @doc """ 19 | Get nodes to run on 20 | """ 21 | @spec nodes(any, Quantum.Job.t()) :: [Node.t()] 22 | def nodes(strategy, job) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /test/support/test_consumer.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TestConsumer do 2 | @moduledoc false 3 | 4 | use GenStage 5 | 6 | def start_link(producer, target) do 7 | GenStage.start_link(__MODULE__, {producer, target}) 8 | end 9 | 10 | def child_spec([producer, target]) do 11 | %{super([]) | start: {__MODULE__, :start_link, [producer, target]}} 12 | end 13 | 14 | def init({producer, owner}) do 15 | {:consumer, owner, subscribe_to: [producer]} 16 | end 17 | 18 | def handle_events(events, _from, owner) do 19 | for event <- events do 20 | send(owner, {:received, event}) 21 | end 22 | 23 | {:noreply, [], owner} 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/quantum/execution_broadcaster/init_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutionBroadcaster.InitOpts do 2 | @moduledoc false 3 | 4 | # Init Options for Quantum.ExecutionBroadcaster 5 | 6 | alias Quantum.{Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | job_broadcaster_reference: GenServer.server(), 10 | clock_broadcaster_reference: GenServer.server(), 11 | storage: Storage, 12 | scheduler: Scheduler, 13 | debug_logging: boolean 14 | } 15 | 16 | @enforce_keys [ 17 | :job_broadcaster_reference, 18 | :clock_broadcaster_reference, 19 | :storage, 20 | :scheduler, 21 | :debug_logging 22 | ] 23 | defstruct @enforce_keys 24 | end 25 | -------------------------------------------------------------------------------- /lib/quantum/executor_supervisor/init_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutorSupervisor.InitOpts do 2 | @moduledoc false 3 | 4 | # Init Options for Quantum.ExecutorSupervisor 5 | 6 | @type t :: %__MODULE__{ 7 | node_selector_broadcaster_reference: GenServer.server(), 8 | task_supervisor_reference: GenServer.server(), 9 | task_registry_reference: GenServer.server(), 10 | debug_logging: boolean, 11 | scheduler: atom() 12 | } 13 | 14 | @enforce_keys [ 15 | :node_selector_broadcaster_reference, 16 | :task_supervisor_reference, 17 | :task_registry_reference, 18 | :debug_logging, 19 | :scheduler 20 | ] 21 | defstruct @enforce_keys 22 | end 23 | -------------------------------------------------------------------------------- /lib/quantum/execution_broadcaster/state.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutionBroadcaster.State do 2 | @moduledoc false 3 | 4 | # Internal State 5 | 6 | alias Quantum.Job 7 | alias Quantum.Storage, as: StorageAdapter 8 | 9 | @type t :: %__MODULE__{ 10 | uninitialized_jobs: [Job.t()], 11 | execution_timeline: [{DateTime.t(), [Job.t()]}], 12 | storage: StorageAdapter, 13 | storage_pid: StorageAdapter.storage_pid(), 14 | scheduler: Quantum, 15 | debug_logging: boolean() 16 | } 17 | 18 | @enforce_keys [ 19 | :uninitialized_jobs, 20 | :execution_timeline, 21 | :storage, 22 | :storage_pid, 23 | :scheduler, 24 | :debug_logging 25 | ] 26 | 27 | defstruct @enforce_keys 28 | end 29 | -------------------------------------------------------------------------------- /lib/quantum/execution_broadcaster/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutionBroadcaster.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options for Quantum.ExecutionBroadcaster 5 | 6 | alias Quantum.{Scheduler, Storage} 7 | 8 | @type t :: %__MODULE__{ 9 | name: GenServer.server(), 10 | job_broadcaster_reference: GenServer.server(), 11 | clock_broadcaster_reference: GenServer.server(), 12 | storage: Storage, 13 | scheduler: Scheduler, 14 | debug_logging: boolean 15 | } 16 | 17 | @enforce_keys [ 18 | :name, 19 | :job_broadcaster_reference, 20 | :clock_broadcaster_reference, 21 | :storage, 22 | :scheduler, 23 | :debug_logging 24 | ] 25 | defstruct @enforce_keys 26 | end 27 | -------------------------------------------------------------------------------- /lib/quantum/executor_supervisor/start_opts.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutorSupervisor.StartOpts do 2 | @moduledoc false 3 | 4 | # Start Options for Quantum.ExecutorSupervisor 5 | 6 | @type t :: %__MODULE__{ 7 | name: GenServer.server(), 8 | node_selector_broadcaster_reference: GenServer.server(), 9 | task_supervisor_reference: GenServer.server(), 10 | task_registry_reference: GenServer.server(), 11 | debug_logging: boolean(), 12 | scheduler: atom() 13 | } 14 | 15 | @enforce_keys [ 16 | :name, 17 | :node_selector_broadcaster_reference, 18 | :task_supervisor_reference, 19 | :task_registry_reference, 20 | :debug_logging, 21 | :scheduler 22 | ] 23 | defstruct @enforce_keys 24 | end 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | quantum-*.tar 24 | 25 | # Temporary files for e.g. tests. 26 | /tmp/ 27 | 28 | # Misc. 29 | mix.lock 30 | /priv/plts/*.plt 31 | /priv/plts/*.plt.hash 32 | -------------------------------------------------------------------------------- /lib/quantum/run_strategy/local.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.RunStrategy.Local do 2 | @moduledoc """ 3 | Run job on local node 4 | 5 | ### Mix Configuration 6 | 7 | config :my_app, MyApp.Scheduler, 8 | jobs: [ 9 | # Run on local node 10 | [schedule: "* * * * *", run_strategy: Quantum.RunStrategy.Local] 11 | ] 12 | 13 | """ 14 | 15 | @typedoc false 16 | @type t :: %__MODULE__{nodes: any} 17 | 18 | defstruct nodes: nil 19 | 20 | @behaviour Quantum.RunStrategy 21 | 22 | alias Quantum.Job 23 | 24 | @impl Quantum.RunStrategy 25 | @spec normalize_config!(any) :: t 26 | def normalize_config!(_), do: %__MODULE__{} 27 | 28 | defimpl Quantum.RunStrategy.NodeList do 29 | @spec nodes(any, Job.t()) :: [Node.t()] 30 | def nodes(_, _) do 31 | [node()] 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /dialyzer.ignore-warnings: -------------------------------------------------------------------------------- 1 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Atom':'__impl__'/1 2 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.BitString':'__impl__'/1 3 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Float':'__impl__'/1 4 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Function':'__impl__'/1 5 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Integer':'__impl__'/1 6 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.List':'__impl__'/1 7 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Map':'__impl__'/1 8 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.PID':'__impl__'/1 9 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Port':'__impl__'/1 10 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Reference':'__impl__'/1 11 | :0: Unknown function 'Elixir.Quantum.RunStrategy.NodeList.Tuple':'__impl__'/1 12 | -------------------------------------------------------------------------------- /test/quantum/job_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.JobTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Quantum.Job 5 | import Crontab.CronExpression 6 | 7 | defmodule Scheduler do 8 | @moduledoc false 9 | 10 | use Quantum, otp_app: :quantum_test 11 | end 12 | 13 | test "new/1 returns a new job" do 14 | assert %Job{} = Scheduler.config() |> Job.new() 15 | end 16 | 17 | test "new/1 returns new job with proper configs" do 18 | configs = Scheduler.config(schedule: "*/7", overlap: false) 19 | schedule = ~e[*/7] 20 | 21 | assert %Job{schedule: ^schedule, overlap: false} = Job.new(configs) 22 | end 23 | 24 | test "is is possible to initialize a job as inactive" do 25 | configs = Scheduler.config(schedule: "*/7", overlap: false, state: :inactive) 26 | schedule = ~e[*/7] 27 | 28 | assert %Job{schedule: ^schedule, overlap: false, state: :inactive} = Job.new(configs) 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /assets/quantum-elixir-logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | quantum-elixir logo 5 | Logo of quantum-elixir, a cron-like job scheduler for elixir 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /test/quantum/run_strategy_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.RunStrategyTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Quantum.Job 5 | alias Quantum.RunStrategy.NodeList 6 | 7 | defmodule Scheduler do 8 | @moduledoc false 9 | 10 | use Quantum, otp_app: :quantum_test 11 | end 12 | 13 | test "run strategy local" do 14 | job = Scheduler.new_job(run_strategy: Quantum.RunStrategy.Local) 15 | assert %Job{} = job 16 | assert [Node.self()] == NodeList.nodes(job.run_strategy, job) 17 | end 18 | 19 | test "run strategy random" do 20 | node_list = [:node1, :node2] 21 | job = Scheduler.new_job(run_strategy: {Quantum.RunStrategy.Random, node_list}) 22 | assert [node] = NodeList.nodes(job.run_strategy, job) 23 | assert Enum.member?(node_list, node) 24 | end 25 | 26 | test "run strategy all" do 27 | node_list = [:node1, :node2] 28 | 29 | job = Scheduler.new_job(run_strategy: {Quantum.RunStrategy.All, node_list}) 30 | assert [:node1, :node2] == NodeList.nodes(job.run_strategy, job) 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/quantum/storage/noop.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Storage.Noop do 2 | @moduledoc """ 3 | Empty implementation of a `Quantum.Storage`. 4 | """ 5 | 6 | @behaviour Quantum.Storage 7 | 8 | use GenServer 9 | 10 | @doc false 11 | @impl GenServer 12 | def init(_args), do: {:ok, nil} 13 | 14 | @doc false 15 | def start_link(_opts), do: :ignore 16 | 17 | @doc false 18 | @impl Quantum.Storage 19 | def jobs(_storage_pid), do: :not_applicable 20 | 21 | @doc false 22 | @impl Quantum.Storage 23 | def add_job(_storage_pid, _job), do: :ok 24 | 25 | @doc false 26 | @impl Quantum.Storage 27 | def delete_job(_storage_pid, _job_name), do: :ok 28 | 29 | @doc false 30 | @impl Quantum.Storage 31 | def update_job_state(_storage_pid, _job_name, _state), do: :ok 32 | 33 | @doc false 34 | @impl Quantum.Storage 35 | def last_execution_date(_storage_pid), do: :unknown 36 | 37 | @doc false 38 | @impl Quantum.Storage 39 | def update_last_execution_date(_storage_pid, _last_execution_date), do: :ok 40 | 41 | @doc false 42 | @impl Quantum.Storage 43 | def purge(_storage_pid), do: :ok 44 | end 45 | -------------------------------------------------------------------------------- /pages/supervision-tree.md: -------------------------------------------------------------------------------- 1 | # Supervision Tree 2 | 3 | * `YourApp.Scheduler` (`Quantum`) - Your primary Interface to interact with. (Like `add_job/1` etc.) 4 | - `YourApp.Scheduler.Supervisor` (`Quantum.Supervisor`) - The Supervisor that coordinates configuration, the runner and task supervisor. 5 | * `YourApp.Scheduler.TaskRegistry` (`Quantum.TaskRegistry`) - The `GenServer` that keeps track of running tasks and prevents overlap. 6 | * `YourApp.Scheduler.JobBroadcaster` (`Quantum.JobBroadcaster`) - The `GenStage` that keeps track of all jobs. 7 | * `YourApp.Scheduler.ExecutionBroadcaster` (`Quantum.ExecutionBroadcaster`) - The `GenStage` that notifies execution of jobs. 8 | * `YourApp.Scheduler.ExecutorSupervisor` (`Quantum.ExecutorSupervisor`) - The `ConsumerSupervisor` that spawns an Executor for every execution. 9 | - `no_name` (`YourApp.Scheduler.Executor`) - The `Task` that calls the `YourApp.Scheduler.TaskSupervisor` with the execution of the Cron (per Node). 10 | * `YourApp.Scheduler.TaskSupervisor` (`Task.Supervisor`) - The `Task.Supervisor` where all Cron jobs run in. 11 | - `Task` - The place where the defined Cron job action gets called. 12 | 13 | ## Error Handling 14 | 15 | The OTP Supervision Tree is initiated by the user of the library. Therefore the error handling can be implemented via normal OTP means. See `Supervisor` module for more information. 16 | -------------------------------------------------------------------------------- /.github/workflows/part_release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | inputs: 4 | releaseName: 5 | required: true 6 | type: string 7 | stable: 8 | required: false 9 | type: boolean 10 | default: false 11 | 12 | name: "Release" 13 | 14 | jobs: 15 | create_prerelease: 16 | name: Create Prerelease 17 | 18 | if: ${{ !inputs.stable }} 19 | 20 | runs-on: ubuntu-latest 21 | 22 | permissions: 23 | contents: write 24 | 25 | steps: 26 | - name: Create draft prerelease 27 | env: 28 | GITHUB_TOKEN: ${{ github.token }} 29 | run: | 30 | gh release create \ 31 | --repo ${{ github.repository }} \ 32 | --title ${{ inputs.releaseName }} \ 33 | --prerelease \ 34 | --notes '' \ 35 | ${{ inputs.releaseName }} 36 | 37 | create_stable: 38 | name: Create Stable 39 | 40 | if: ${{ inputs.stable }} 41 | 42 | runs-on: ubuntu-latest 43 | 44 | permissions: 45 | contents: write 46 | 47 | steps: 48 | - name: Create draft release 49 | env: 50 | GITHUB_TOKEN: ${{ github.token }} 51 | run: | 52 | gh release create \ 53 | --repo ${{ github.repository }} \ 54 | --title ${{ inputs.releaseName }} \ 55 | --notes '' \ 56 | --draft \ 57 | ${{ inputs.releaseName }} 58 | -------------------------------------------------------------------------------- /test/quantum/task_registry_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TaskRegistryTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | alias Quantum.TaskRegistry 7 | alias Quantum.TaskRegistry.StartOpts 8 | 9 | doctest TaskRegistry, 10 | except: [mark_running: 3, mark_finished: 3] 11 | 12 | setup do 13 | {:ok, _registry} = start_supervised({TaskRegistry, %StartOpts{name: __MODULE__}}) 14 | 15 | {:ok, %{registry: __MODULE__}} 16 | end 17 | 18 | describe "running" do 19 | test "not running => running", %{registry: registry} do 20 | task = make_ref() 21 | 22 | assert :marked_running = TaskRegistry.mark_running(registry, task, self()) 23 | end 24 | 25 | test "running => already running", %{registry: registry} do 26 | task = make_ref() 27 | 28 | TaskRegistry.mark_running(registry, task, self()) 29 | 30 | assert :already_running = TaskRegistry.mark_running(registry, task, self()) 31 | end 32 | end 33 | 34 | describe "finished" do 35 | test "finish existing", %{registry: registry} do 36 | task = make_ref() 37 | 38 | TaskRegistry.mark_running(registry, task, self()) 39 | 40 | assert :ok = TaskRegistry.mark_finished(registry, task, self()) 41 | end 42 | 43 | test "finish not existing", %{registry: registry} do 44 | task = make_ref() 45 | 46 | assert :ok = TaskRegistry.mark_finished(registry, task, self()) 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /pages/run-strategies.md: -------------------------------------------------------------------------------- 1 | # Run Strategies 2 | 3 | Tasks can be executed via different run strategies. 4 | 5 | ## Configuration 6 | 7 | ### Mix 8 | 9 | ```elixir 10 | config :my_app, MyApp.Scheduler, 11 | jobs: [ 12 | [schedule: "* * * * *", run_strategy: {StrategyName, options}], 13 | ] 14 | ``` 15 | 16 | The run strategy can be configured by providing a tuple of the strategy module name and it's options. If you choose `Local Node` strategy, the config should be: 17 | 18 | ```elixir 19 | config :my_app, MyApp.Scheduler, 20 | jobs: [ 21 | [schedule: "* * * * *", run_strategy: Quantum.RunStrategy.Local], 22 | ] 23 | ``` 24 | 25 | ### Runtime 26 | 27 | Provide a value that implements the `Quantum.RunStrategy.NodeList` protocol. The value will not be normalized. 28 | 29 | ## Provided Strategies 30 | 31 | ### All Nodes 32 | 33 | `Quantum.RunStrategy.All` 34 | 35 | If you want to run a task on all nodes of either a list or in the whole cluster, use this strategy. 36 | 37 | ### Random Node 38 | 39 | `Quantum.RunStrategy.Random` 40 | 41 | If you want to run a task on any node of either a list or in the whole cluster, use this strategy. 42 | 43 | ### Local Node 44 | 45 | `Quantum.RunStrategy.Local` 46 | 47 | If you want to run a task on local node, use this strategy. 48 | 49 | ## Custom Run Strategy 50 | 51 | Custom run strategies, can be implemented by implementing the `Quantum.RunStrategy` behaviour and the `Quantum.RunStrategy.NodeList` protocol. 52 | -------------------------------------------------------------------------------- /lib/quantum/run_strategy/all.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.RunStrategy.All do 2 | @moduledoc """ 3 | Run job on all node of the node list. 4 | 5 | If the node list is `:cluster`, all nodes of the cluster will be used. 6 | 7 | ### Mix Configuration 8 | 9 | config :my_app, MyApp.Scheduler, 10 | jobs: [ 11 | # Run on all nodes in cluster 12 | [schedule: "* * * * *", run_strategy: {Quantum.RunStrategy.All, :cluster}], 13 | # Run on all nodes of given list 14 | [schedule: "* * * * *", run_strategy: {Quantum.RunStrategy.All, [:"node@host1", :"node@host2"]}], 15 | ] 16 | 17 | """ 18 | 19 | @typedoc false 20 | @type t :: %__MODULE__{nodes: [Node.t() | :cluster]} 21 | 22 | defstruct nodes: nil 23 | 24 | @behaviour Quantum.RunStrategy 25 | 26 | alias Quantum.Job 27 | 28 | @impl Quantum.RunStrategy 29 | @spec normalize_config!([Node.t()] | :cluster) :: t 30 | def normalize_config!(nodes) when is_list(nodes) do 31 | %__MODULE__{nodes: Enum.map(nodes, &normalize_node/1)} 32 | end 33 | 34 | def normalize_config!(:cluster), do: %__MODULE__{nodes: :cluster} 35 | 36 | @spec normalize_node(Node.t() | binary) :: Node.t() 37 | defp normalize_node(node) when is_atom(node), do: node 38 | defp normalize_node(node) when is_binary(node), do: String.to_atom(node) 39 | 40 | defimpl Quantum.RunStrategy.NodeList do 41 | @spec nodes(Quantum.RunStrategy.All.t(), Job.t()) :: [Node.t()] 42 | def nodes(%Quantum.RunStrategy.All{nodes: :cluster}, _) do 43 | [node() | Node.list()] 44 | end 45 | 46 | def nodes(%Quantum.RunStrategy.All{nodes: nodes}, _) do 47 | nodes 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/quantum/executor_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutorSupervisor do 2 | @moduledoc false 3 | 4 | # This `ConsumerSupervisor` is responsible to start a job for every execute event. 5 | 6 | use ConsumerSupervisor 7 | 8 | alias Quantum.Executor.StartOpts, as: ExecutorStartOpts 9 | 10 | alias __MODULE__.{InitOpts, StartOpts} 11 | 12 | @spec start_link(StartOpts.t()) :: GenServer.on_start() 13 | def start_link(%StartOpts{name: name} = opts) do 14 | __MODULE__ 15 | |> ConsumerSupervisor.start_link( 16 | struct!( 17 | InitOpts, 18 | Map.take(opts, [ 19 | :node_selector_broadcaster_reference, 20 | :task_supervisor_reference, 21 | :task_registry_reference, 22 | :debug_logging, 23 | :scheduler 24 | ]) 25 | ), 26 | name: name 27 | ) 28 | |> case do 29 | {:ok, pid} -> 30 | {:ok, pid} 31 | 32 | {:error, {:already_started, pid}} -> 33 | Process.monitor(pid) 34 | {:ok, pid} 35 | 36 | {:error, _reason} = error -> 37 | error 38 | end 39 | end 40 | 41 | @impl ConsumerSupervisor 42 | def init( 43 | %InitOpts{ 44 | node_selector_broadcaster_reference: node_selector_broadcaster 45 | } = opts 46 | ) do 47 | executor_opts = 48 | struct!( 49 | ExecutorStartOpts, 50 | Map.take(opts, [ 51 | :task_supervisor_reference, 52 | :task_registry_reference, 53 | :debug_logging, 54 | :scheduler 55 | ]) 56 | ) 57 | 58 | ConsumerSupervisor.init( 59 | [{Quantum.Executor, executor_opts}], 60 | strategy: :one_for_one, 61 | subscribe_to: [{node_selector_broadcaster, max_demand: 50}] 62 | ) 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /pages/runtime-configuration.md: -------------------------------------------------------------------------------- 1 | # Runtime Configuration 2 | 3 | If you want to add jobs on runtime, this is possible too: 4 | 5 | ```elixir 6 | import Crontab.CronExpression 7 | 8 | YourApp.Scheduler.add_job({~e[1 * * * *], fn -> :ok end}) 9 | ``` 10 | 11 | Add a named job at runtime: 12 | 13 | ```elixir 14 | import Crontab.CronExpression 15 | 16 | YourApp.Scheduler.new_job() 17 | |> Quantum.Job.set_name(:ticker) 18 | |> Quantum.Job.set_schedule(~e[1 * * * *]) 19 | |> Quantum.Job.set_task(fn -> :ok end) 20 | |> YourApp.Scheduler.add_job() 21 | ``` 22 | 23 | Deactivate a job, i.e. it will not be performed until job is activated again: 24 | ```elixir 25 | YourApp.Scheduler.deactivate_job(:ticker) 26 | ``` 27 | 28 | Activate an inactive job: 29 | ```elixir 30 | YourApp.Scheduler.activate_job(:ticker) 31 | ``` 32 | 33 | Run a job once outside of normal schedule: 34 | ```elixir 35 | YourApp.Scheduler.run_job(:ticker) 36 | ``` 37 | 38 | Find a job: 39 | ```elixir 40 | YourApp.Scheduler.find_job(:ticker) 41 | # %Quantum.Job{...} 42 | ``` 43 | 44 | Delete a job: 45 | ```elixir 46 | YourApp.Scheduler.delete_job(:ticker) 47 | # %Quantum.Job{...} 48 | ``` 49 | 50 | ## Jobs with Second granularity 51 | 52 | It is possible to specify jobs with second granularity. 53 | To do this the `schedule` parameter has to be provided with either a `%Crontab.CronExpression{extended: true, ...}` or 54 | with a set `e` flag on the `e` sigil. (The sigil must be imported from `Crontab.CronExpression`) 55 | 56 | The following example will put a tick into the `stdout` every first second of every minute. 57 | 58 | ```elixir 59 | import Crontab.CronExpression 60 | 61 | YourApp.Scheduler.new_job() 62 | |> Quantum.Job.set_name(:ticker) 63 | |> Quantum.Job.set_schedule(~e[1 * * * *]e) 64 | |> Quantum.Job.set_task(fn -> IO.puts "tick" end) 65 | |> YourApp.Scheduler.add_job() 66 | ``` 67 | -------------------------------------------------------------------------------- /lib/quantum/run_strategy/random.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.RunStrategy.Random do 2 | @moduledoc """ 3 | Run job on one node of the list randomly. 4 | 5 | If the node list is `:cluster`, one node of the cluster will be used. 6 | 7 | This run strategy also makes sure, that the node doesn't run in two places at the same time 8 | if `job.overlap` is falsy. 9 | 10 | ### Mix Configuration 11 | 12 | config :my_app, MyApp.Scheduler, 13 | jobs: [ 14 | # Run on any node in cluster 15 | [schedule: "* * * * *", run_strategy: {Quantum.RunStrategy.Random, :cluster}], 16 | # Run on any node of given list 17 | [schedule: "* * * * *", run_strategy: {Quantum.RunStrategy.Random, [:"node@host1", :"node@host2"]}], 18 | ] 19 | 20 | """ 21 | 22 | @typedoc false 23 | @type t :: %__MODULE__{nodes: [Node.t()] | :cluster} 24 | 25 | defstruct nodes: nil 26 | 27 | @behaviour Quantum.RunStrategy 28 | 29 | alias Quantum.Job 30 | 31 | @impl Quantum.RunStrategy 32 | @spec normalize_config!([Node.t()] | :cluster) :: t 33 | def normalize_config!(nodes) when is_list(nodes) do 34 | %__MODULE__{nodes: Enum.map(nodes, &normalize_node/1)} 35 | end 36 | 37 | def normalize_config!(:cluster), do: %__MODULE__{nodes: :cluster} 38 | 39 | @spec normalize_node(Node.t() | binary) :: Node.t() 40 | defp normalize_node(node) when is_atom(node), do: node 41 | defp normalize_node(node) when is_binary(node), do: String.to_atom(node) 42 | 43 | defimpl Quantum.RunStrategy.NodeList do 44 | @spec nodes(Quantum.RunStrategy.Random.t(), Job.t()) :: [Node.t()] 45 | def nodes(%Quantum.RunStrategy.Random{nodes: :cluster}, _job) do 46 | [Enum.random([node() | Node.list()])] 47 | end 48 | 49 | def nodes(%Quantum.RunStrategy.Random{nodes: nodes}, _job) do 50 | [Enum.random(nodes)] 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/quantum_startup_test.exs: -------------------------------------------------------------------------------- 1 | defmodule QuantumStartupTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case 5 | 6 | import ExUnit.CaptureLog 7 | 8 | import Crontab.CronExpression 9 | 10 | defmodule Scheduler do 11 | @moduledoc false 12 | 13 | use Quantum, otp_app: :quantum 14 | end 15 | 16 | @tag :startup 17 | test "prevent duplicate job names on startup" do 18 | capture_log(fn -> 19 | test_jobs = [ 20 | {:test_job, [schedule: ~e[1 * * * *], task: fn -> :ok end]}, 21 | {:test_job, [schedule: ~e[2 * * * *], task: fn -> :ok end]}, 22 | {:inactive_job, [schedule: ~e[* * * * *], task: fn -> :ok end, state: :inactive]}, 23 | {"3 * * * *", fn -> :ok end}, 24 | {"4 * * * *", fn -> :ok end} 25 | ] 26 | 27 | Application.put_env(:quantum, QuantumStartupTest.Scheduler, jobs: test_jobs) 28 | 29 | start_supervised!(Scheduler) 30 | 31 | assert Enum.count(QuantumStartupTest.Scheduler.jobs()) == 4 32 | assert QuantumStartupTest.Scheduler.find_job(:test_job).schedule == ~e[1 * * * *] 33 | assert QuantumStartupTest.Scheduler.find_job(:inactive_job).state == :inactive 34 | 35 | :ok = stop_supervised(Scheduler) 36 | end) 37 | end 38 | 39 | @tag :startup 40 | test "prevent unexported functions on startup" do 41 | log = 42 | capture_log(fn -> 43 | test_jobs = [ 44 | {:existing_function, [schedule: ~e[2 * * * *], task: {IO, :puts, ["hey"]}]}, 45 | {:another_existing_function, [schedule: ~e[2 * * * *], task: {Kernel, :floor, [5.4]}]}, 46 | {:inexistent_function, 47 | [schedule: ~e[2 * * * *], task: {UndefinedModule, :undefined_function, ["argument"]}]} 48 | ] 49 | 50 | Application.put_env(:quantum, QuantumStartupTest.Scheduler, jobs: test_jobs) 51 | 52 | start_supervised!(Scheduler) 53 | 54 | assert Enum.count(QuantumStartupTest.Scheduler.jobs()) == 2 55 | :ok = stop_supervised(Scheduler) 56 | end) 57 | 58 | assert log =~ 59 | "Job with name :inexistent_function of scheduler QuantumStartupTest.Scheduler not started: invalid task function" 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/quantum/task_registry.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.TaskRegistry do 2 | @moduledoc false 3 | 4 | # Registry to check if a task is already running on a node. 5 | 6 | alias __MODULE__.StartOpts 7 | alias Quantum.Job 8 | 9 | # Start the registry 10 | @spec start_link(StartOpts.t()) :: GenServer.on_start() 11 | def start_link(%StartOpts{name: name, listeners: listeners}) do 12 | [keys: :unique, name: name, listeners: listeners] 13 | |> Registry.start_link() 14 | |> case do 15 | {:ok, pid} -> 16 | {:ok, pid} 17 | 18 | {:error, {:already_started, pid}} -> 19 | Process.monitor(pid) 20 | {:ok, pid} 21 | 22 | {:error, _reason} = error -> 23 | error 24 | end 25 | end 26 | 27 | @spec child_spec(options :: StartOpts.t()) :: Supervisor.child_spec() 28 | def child_spec(options), 29 | do: 30 | [] 31 | |> Registry.child_spec() 32 | |> Map.put(:start, {__MODULE__, :start_link, [options]}) 33 | 34 | # Mark a task as Running 35 | # 36 | # ### Examples 37 | # 38 | # iex> Quantum.TaskRegistry.mark_running(server, running_job.name, Node.self()) 39 | # :already_running 40 | # 41 | # iex> Quantum.TaskRegistry.mark_running(server, not_running_job.name, Node.self()) 42 | # :marked_running 43 | @spec mark_running(server :: atom, task :: Job.name(), node :: Node.t()) :: 44 | :already_running | :marked_running 45 | def mark_running(server, task, node) do 46 | server 47 | |> Registry.register({task, node}, true) 48 | |> case do 49 | {:ok, _pid} -> :marked_running 50 | {:error, {:already_registered, _other_pid}} -> :already_running 51 | end 52 | end 53 | 54 | # Mark a task as Finished 55 | # 56 | # ### Examples 57 | # 58 | # iex> Quantum.TaskRegistry.mark_running(server, running_job.name, Node.self()) 59 | # :ok 60 | # 61 | # iex> Quantum.TaskRegistry.mark_running(server, not_running_job.name, Node.self()) 62 | # :ok 63 | @spec mark_finished(server :: atom, task :: Job.name(), node :: Node.t()) :: :ok 64 | def mark_finished(server, task, node) do 65 | Registry.unregister(server, {task, node}) 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /.github/workflows/part_docs.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | inputs: 4 | releaseName: 5 | required: false 6 | type: string 7 | 8 | name: "Documentation" 9 | 10 | env: 11 | BUILD_EMBEDDED: true 12 | 13 | jobs: 14 | generate: 15 | name: "Generate" 16 | 17 | runs-on: ubuntu-latest 18 | 19 | env: 20 | MIX_ENV: dev 21 | 22 | steps: 23 | - uses: actions/checkout@v6 24 | - uses: erlef/setup-elixir@v1 25 | id: setupBEAM 26 | with: 27 | version-file: '.tool-versions' 28 | version-type: strict 29 | - uses: actions/cache@v5 30 | with: 31 | path: deps 32 | key: deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 33 | restore-keys: | 34 | deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 35 | - run: mix deps.get 36 | - uses: actions/cache@v5 37 | with: 38 | path: _build/dev 39 | key: compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 40 | restore-keys: | 41 | compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 42 | - run: mix deps.compile 43 | - run: mix compile --warning-as-errors 44 | - run: mix docs 45 | - uses: actions/upload-artifact@v6 46 | with: 47 | name: docs 48 | path: doc 49 | 50 | upload: 51 | name: "Upload" 52 | 53 | runs-on: ubuntu-latest 54 | 55 | if: ${{ inputs.releaseName }} 56 | 57 | needs: ['generate'] 58 | 59 | permissions: 60 | contents: write 61 | 62 | steps: 63 | - uses: actions/checkout@v6 64 | - uses: actions/download-artifact@v7 65 | with: 66 | name: docs 67 | path: docs 68 | - run: | 69 | tar -czvf docs.tar.gz docs 70 | - name: Upload 71 | env: 72 | GITHUB_TOKEN: ${{ github.token }} 73 | run: | 74 | gh release upload --clobber "${{ inputs.releaseName }}" \ 75 | docs.tar.gz -------------------------------------------------------------------------------- /pages/crontab-format.md: -------------------------------------------------------------------------------- 1 | # Crontab format 2 | 3 | ## Basics 4 | 5 | | Field | Allowed values | 6 | | ------------ | ------------------------------------------- | 7 | | second | 0-59 | 8 | | minute | 0-59 | 9 | | hour | 0-23 | 10 | | day of month | 1-31 | 11 | | month | 1-12 (or names) | 12 | | day of week | 0-6 (0 is Sunday, or use abbreviated names) | 13 | 14 | The `second` field can only be used in extended Cron expressions. 15 | 16 | Names can also be used for the `month` and `day of week` fields. 17 | Use the first three letters of the particular day or month (case does not matter). 18 | 19 | ## Special expressions 20 | 21 | Instead of the first five fields, one of these special strings may be used: 22 | 23 | | String | Description | 24 | | ----------- | --------------------------------------------------------- | 25 | | `@annually` | Run once a year, same as `~e["0 0 1 1 *"]` or `@yearly` | 26 | | `@daily` | Run once a day, same as `~e["0 0 * * *"]` or `@midnight` | 27 | | `@hourly` | Run once an hour, same as `~e["0 * * * *"]` | 28 | | `@midnight` | Run once a day, same as `~e["0 0 * * *"]` or `@daily` | 29 | | `@minutely` | Run once a minute, same as `~e["* * * * *"]` | 30 | | `@monthly` | Run once a month, same as `~e["0 0 1 * *"]` | 31 | | `@reboot` | Run once, at startup | 32 | | `@secondly` | Run once a second, same as `~e["* * * * * *"]e` | 33 | | `@weekly` | Run once a week, same as `~e["0 0 * * 0"]` | 34 | | `@yearly` | Run once a year, same as `~e["0 0 1 1 *"]` or `@annually` | 35 | 36 | ## Supported Notations 37 | 38 | * [Oracle](https://docs.oracle.com/cd/E12058_01/doc/doc.1014/e12030/cron_expressions.htm) 39 | * [Cron Format](http://www.nncron.ru/help/EN/working/cron-format.htm) 40 | * [Wikipedia](https://en.wikipedia.org/wiki/Cron) 41 | 42 | ## Crontab Dependency 43 | 44 | All Cron Expressions are parsed and evaluated by [crontab](https://hex.pm/packages/crontab). 45 | 46 | Issues with parsing a Cron expression can be reported here: 47 | [crontab GitHub issues](https://github.com/jshmrtn/crontab/issues) 48 | -------------------------------------------------------------------------------- /test/quantum/node_selector_broadcaster_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NodeSelectorBroadcasterTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import ExUnit.CaptureLog 7 | import Quantum.CaptureLogExtend 8 | 9 | alias Quantum.ExecutionBroadcaster.Event, as: ExecuteEvent 10 | alias Quantum.Job 11 | alias Quantum.NodeSelectorBroadcaster 12 | alias Quantum.NodeSelectorBroadcaster.Event 13 | alias Quantum.NodeSelectorBroadcaster.StartOpts 14 | alias Quantum.RunStrategy.All 15 | alias Quantum.{TestConsumer, TestProducer} 16 | 17 | doctest NodeSelectorBroadcaster 18 | 19 | defmodule TestScheduler do 20 | @moduledoc false 21 | 22 | use Quantum, otp_app: :job_broadcaster_test 23 | end 24 | 25 | setup _ do 26 | task_supervisor = 27 | start_supervised!({Task.Supervisor, [name: Module.concat(__MODULE__, TaskSupervisor)]}) 28 | 29 | producer = start_supervised!({TestProducer, []}) 30 | 31 | {broadcaster, _} = 32 | capture_log_with_return(fn -> 33 | start_supervised!( 34 | {NodeSelectorBroadcaster, 35 | %StartOpts{ 36 | name: __MODULE__, 37 | execution_broadcaster_reference: producer, 38 | task_supervisor_reference: task_supervisor 39 | }} 40 | ) 41 | end) 42 | 43 | start_supervised!({TestConsumer, [broadcaster, self()]}) 44 | 45 | {:ok, %{producer: producer, broadcaster: broadcaster}} 46 | end 47 | 48 | describe "execute" do 49 | test "schedules execution", %{ 50 | producer: producer 51 | } do 52 | caller = self() 53 | 54 | job = 55 | TestScheduler.new_job() 56 | |> Job.set_task(fn -> send(caller, :executed) end) 57 | 58 | TestProducer.send(producer, %ExecuteEvent{job: job}) 59 | 60 | node_self = Node.self() 61 | assert_receive {:received, %Event{job: ^job, node: ^node_self}} 62 | end 63 | 64 | test "doesn't execute on invalid node", %{ 65 | producer: producer 66 | } do 67 | node = :"invalid-name@invalid-host" 68 | 69 | job = 70 | TestScheduler.new_job() 71 | |> Job.set_run_strategy(%All{nodes: [node]}) 72 | 73 | assert capture_log(fn -> 74 | TestProducer.send(producer, %ExecuteEvent{job: job}) 75 | 76 | refute_receive %Event{} 77 | end) =~ 78 | "Node #{inspect(node)} is not running. Job #{inspect(job.name)} could not be executed." 79 | end 80 | end 81 | 82 | def send(caller) do 83 | send(caller, :executed) 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/quantum/storage.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Storage do 2 | @moduledoc """ 3 | Behaviour to be implemented by all Storage Adapters. 4 | 5 | The calls to the storage are blocking, make sure they're fast to not block the job execution. 6 | """ 7 | 8 | alias Quantum.Job 9 | 10 | @typedoc """ 11 | The location of the `server`. 12 | 13 | ### Values 14 | 15 | * `nil` if the storage was not started 16 | * `server()` if the storage was started 17 | 18 | """ 19 | @type storage_pid :: nil | GenServer.server() 20 | 21 | @doc """ 22 | Storage child spec 23 | 24 | If the storage does not need a process, specify a function that returns `:ignore`. 25 | 26 | ### Values 27 | 28 | * `:scheduler` - The Scheduler 29 | 30 | """ 31 | @callback child_spec(init_arg :: Keyword.t()) :: Supervisor.child_spec() 32 | 33 | @doc """ 34 | Load saved jobs from storage. 35 | 36 | Returns `:not_applicable` if the storage has never received an `add_job` call or after it has been purged. 37 | In this case the jobs from the configuration will be loaded. 38 | """ 39 | @callback jobs(storage_pid :: storage_pid) :: 40 | :not_applicable | [Job.t()] 41 | 42 | @doc """ 43 | Save new job in storage. 44 | """ 45 | @callback add_job(storage_pid :: storage_pid, job :: Job.t()) :: 46 | :ok 47 | 48 | @doc """ 49 | Delete new job in storage. 50 | """ 51 | @callback delete_job(storage_pid :: storage_pid, job :: Job.name()) :: :ok 52 | 53 | @doc """ 54 | Change Job State from given job. 55 | """ 56 | @callback update_job_state(storage_pid :: storage_pid, job :: Job.name(), state :: Job.state()) :: 57 | :ok 58 | 59 | @doc """ 60 | Load last execution time from storage. 61 | 62 | Returns `:unknown` if the storage does not know the last execution time. 63 | In this case all jobs will be run at the next applicable date. 64 | """ 65 | @callback last_execution_date(storage_pid :: storage_pid) :: :unknown | DateTime.t() 66 | 67 | @doc """ 68 | Update last execution time to given date. 69 | """ 70 | @callback update_last_execution_date( 71 | storage_pid :: storage_pid, 72 | last_execution_date :: DateTime.t() 73 | ) :: :ok 74 | 75 | @doc """ 76 | Purge all date from storage and go back to initial state. 77 | """ 78 | @callback purge(storage_pid :: storage_pid) :: :ok 79 | 80 | @doc """ 81 | Updates existing job in storage. 82 | 83 | This callback is optional. If not implemented then the `c:delete_job/2` 84 | and then the `c:add_job/2` callbacks will be called instead. 85 | """ 86 | @callback update_job(storage_pid :: storage_pid, job :: Job.t()) :: :ok 87 | 88 | @optional_callbacks update_job: 2 89 | end 90 | -------------------------------------------------------------------------------- /lib/quantum/node_selector_broadcaster.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NodeSelectorBroadcaster do 2 | @moduledoc false 3 | 4 | # Receives Added / Removed Jobs, Broadcasts Executions of Jobs 5 | 6 | use GenStage 7 | 8 | require Logger 9 | 10 | alias Quantum.ExecutionBroadcaster.Event, as: ExecuteEvent 11 | alias Quantum.Job 12 | alias Quantum.RunStrategy.NodeList 13 | 14 | alias __MODULE__.{Event, InitOpts, StartOpts, State} 15 | 16 | @type event :: {:add, Job.t()} | {:execute, Job.t()} 17 | 18 | # Start Stage 19 | @spec start_link(StartOpts.t()) :: GenServer.on_start() 20 | def start_link(%StartOpts{name: name} = opts) do 21 | __MODULE__ 22 | |> GenStage.start_link( 23 | struct!( 24 | InitOpts, 25 | Map.take(opts, [ 26 | :execution_broadcaster_reference, 27 | :task_supervisor_reference 28 | ]) 29 | ), 30 | name: name 31 | ) 32 | |> case do 33 | {:ok, pid} -> 34 | {:ok, pid} 35 | 36 | {:error, {:already_started, pid}} -> 37 | Process.monitor(pid) 38 | {:ok, pid} 39 | 40 | {:error, _reason} = error -> 41 | error 42 | end 43 | end 44 | 45 | @impl GenStage 46 | def init(%InitOpts{ 47 | execution_broadcaster_reference: execution_broadcaster, 48 | task_supervisor_reference: task_supervisor_reference 49 | }) do 50 | {:producer_consumer, 51 | %State{ 52 | task_supervisor_reference: task_supervisor_reference 53 | }, subscribe_to: [execution_broadcaster]} 54 | end 55 | 56 | @impl GenStage 57 | def handle_events(events, _, %{task_supervisor_reference: task_supervisor_reference} = state) do 58 | {:noreply, 59 | Enum.flat_map(events, fn %ExecuteEvent{job: job} -> 60 | job 61 | |> select_nodes(task_supervisor_reference) 62 | |> Enum.map(fn node -> 63 | %Event{job: job, node: node} 64 | end) 65 | end), state} 66 | end 67 | 68 | @impl GenStage 69 | def handle_info(_message, state) do 70 | {:noreply, [], state} 71 | end 72 | 73 | defp select_nodes(%Job{run_strategy: run_strategy} = job, task_supervisor) do 74 | run_strategy 75 | |> NodeList.nodes(job) 76 | |> Enum.filter(&check_node(&1, task_supervisor, job)) 77 | end 78 | 79 | @spec check_node(Node.t(), GenServer.server(), Job.t()) :: boolean 80 | defp check_node(node, task_supervisor, %{name: job_name}) do 81 | if running_node?(node, task_supervisor) do 82 | true 83 | else 84 | Logger.error( 85 | "Node #{inspect(node)} is not running. Job #{inspect(job_name)} could not be executed." 86 | ) 87 | 88 | false 89 | end 90 | end 91 | 92 | # Check if the task supervisor runs on a given node 93 | @spec running_node?(Node.t(), GenServer.server()) :: boolean 94 | defp running_node?(node, _) when node == node(), do: true 95 | 96 | defp running_node?(node, task_supervisor) do 97 | node 98 | |> :rpc.call(:erlang, :whereis, [task_supervisor]) 99 | |> is_pid() 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Mixfile do 2 | @moduledoc false 3 | 4 | use Mix.Project 5 | 6 | @source_url "https://github.com/quantum-elixir/quantum-core" 7 | @version "3.5.3" 8 | 9 | def project do 10 | [ 11 | app: :quantum, 12 | build_embedded: Mix.env() == :prod, 13 | deps: deps(), 14 | description: "Cron-like job scheduler for Elixir.", 15 | docs: docs(), 16 | elixir: "~> 1.15", 17 | name: "Quantum", 18 | elixirc_paths: elixirc_paths(Mix.env()), 19 | package: package(), 20 | start_permanent: Mix.env() == :prod, 21 | test_coverage: [tool: ExCoveralls], 22 | version: @version, 23 | build_embedded: (System.get_env("BUILD_EMBEDDED") || "false") in ["1", "true"], 24 | dialyzer: 25 | [list_unused_filters: true, ignore_warnings: "dialyzer.ignore-warnings"] ++ 26 | if (System.get_env("DIALYZER_PLT_PRIV") || "false") in ["1", "true"] do 27 | [plt_file: {:no_warn, "priv/plts/dialyzer.plt"}] 28 | else 29 | [] 30 | end, 31 | preferred_cli_env: [ 32 | coveralls: :test, 33 | "coveralls.detail": :test, 34 | "coveralls.html": :test, 35 | "coveralls.json": :test, 36 | "coveralls.post": :test, 37 | "coveralls.xml": :test 38 | ] 39 | ] 40 | end 41 | 42 | def application do 43 | [extra_applications: [:logger]] 44 | end 45 | 46 | defp elixirc_paths(:test), do: ["lib", "test/support"] 47 | defp elixirc_paths(_), do: ["lib"] 48 | 49 | defp package do 50 | %{ 51 | maintainers: [ 52 | "Constantin Rack", 53 | "Dan Swain", 54 | "Lenz Gschwendtner", 55 | "Lucas Charles", 56 | "Rodion Vshevtsov", 57 | "Stanislav Krasnoyarov", 58 | "Kai Faber", 59 | "Jonatan Männchen" 60 | ], 61 | exclude_patterns: [~r[priv/plts]], 62 | licenses: ["Apache-2.0"], 63 | links: %{ 64 | "Changelog" => "#{@source_url}/blob/main/CHANGELOG.md", 65 | "GitHub" => @source_url 66 | } 67 | } 68 | end 69 | 70 | defp docs do 71 | [ 72 | main: "readme", 73 | source_ref: "v#{@version}", 74 | source_url: @source_url, 75 | logo: "assets/quantum-elixir-logo.svg", 76 | extras: [ 77 | "CHANGELOG.md", 78 | "README.md", 79 | "pages/supervision-tree.md", 80 | "pages/configuration.md", 81 | "pages/runtime-configuration.md", 82 | "pages/crontab-format.md", 83 | "pages/run-strategies.md" 84 | ], 85 | groups_for_modules: [ 86 | "Run Strategy": [ 87 | Quantum.RunStrategy, 88 | Quantum.RunStrategy.All, 89 | Quantum.RunStrategy.Local, 90 | Quantum.RunStrategy.NodeList, 91 | Quantum.RunStrategy.Random 92 | ], 93 | Storage: [ 94 | Quantum.Storage, 95 | Quantum.Storage.Noop 96 | ] 97 | ] 98 | ] 99 | end 100 | 101 | defp deps do 102 | [ 103 | {:crontab, "~> 1.2"}, 104 | {:gen_stage, "~> 0.14 or ~> 1.0"}, 105 | {:telemetry, "~> 0.4.3 or ~> 1.0"}, 106 | {:tzdata, "~> 1.0", only: [:dev, :test]}, 107 | {:ex_doc, ">= 0.0.0", only: [:dev], runtime: false}, 108 | {:excoveralls, "~> 0.5", only: [:test], runtime: false}, 109 | {:dialyxir, "~> 1.0-rc", only: [:dev], runtime: false}, 110 | {:credo, "~> 1.0", only: [:dev], runtime: false}, 111 | {:telemetry_registry, "~> 0.2"} 112 | ] 113 | end 114 | end 115 | -------------------------------------------------------------------------------- /lib/quantum/clock_broadcaster.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ClockBroadcaster do 2 | @moduledoc false 3 | 4 | # Broadcasts the time to run jobs for 5 | 6 | use GenStage 7 | 8 | require Logger 9 | 10 | alias __MODULE__.{Event, InitOpts, StartOpts, State} 11 | 12 | @spec start_link(opts :: StartOpts.t()) :: GenServer.on_start() 13 | def start_link(%StartOpts{name: name} = opts) do 14 | __MODULE__ 15 | |> GenStage.start_link( 16 | struct!(InitOpts, Map.take(opts, [:start_time, :storage, :scheduler, :debug_logging])), 17 | name: name 18 | ) 19 | |> case do 20 | {:ok, pid} -> 21 | {:ok, pid} 22 | 23 | {:error, {:already_started, pid}} -> 24 | Process.monitor(pid) 25 | {:ok, pid} 26 | 27 | {:error, _reason} = error -> 28 | error 29 | end 30 | end 31 | 32 | @impl GenStage 33 | @spec init(opts :: InitOpts.t()) :: {:producer, State.t()} 34 | def init(%InitOpts{ 35 | debug_logging: debug_logging, 36 | storage: storage, 37 | scheduler: scheduler, 38 | start_time: start_time 39 | }) do 40 | start_time = 41 | scheduler 42 | |> Module.concat(Storage) 43 | |> GenServer.whereis() 44 | |> storage.last_execution_date() 45 | |> case do 46 | :unknown -> start_time 47 | date -> DateTime.from_naive!(date, "Etc/UTC") 48 | end 49 | |> DateTime.truncate(:second) 50 | # Roll back one second since handle_tick will start at `now + 1`. 51 | |> DateTime.add(-1, :second) 52 | 53 | :timer.send_interval(1000, :tick) 54 | 55 | {:producer, 56 | %State{ 57 | time: start_time, 58 | debug_logging: debug_logging, 59 | remaining_demand: 0 60 | }} 61 | end 62 | 63 | @impl GenStage 64 | def handle_demand(demand, %State{remaining_demand: remaining_demand} = state) do 65 | handle_tick(%State{state | remaining_demand: remaining_demand + demand}) 66 | end 67 | 68 | @impl GenStage 69 | def handle_info(:tick, state) do 70 | handle_tick(state) 71 | end 72 | 73 | def handle_info(_message, state) do 74 | {:noreply, [], state} 75 | end 76 | 77 | defp handle_tick(%State{remaining_demand: 0} = state) do 78 | {:noreply, [], state} 79 | end 80 | 81 | defp handle_tick(%State{remaining_demand: remaining_demand, time: time} = state) 82 | when remaining_demand > 0 do 83 | now = DateTime.truncate(DateTime.utc_now(), :second) 84 | 85 | {events, new_time} = 86 | Enum.reduce_while( 87 | 1..remaining_demand, 88 | {[], time}, 89 | fn _, {list, time} = acc -> 90 | new_time = DateTime.add(time, 1, :second) 91 | 92 | case DateTime.compare(new_time, now) do 93 | :lt -> 94 | {:cont, {[%Event{time: new_time, catch_up: true} | list], new_time}} 95 | 96 | :eq -> 97 | {:cont, {[%Event{time: new_time, catch_up: false} | list], new_time}} 98 | 99 | :gt -> 100 | {:halt, acc} 101 | end 102 | end 103 | ) 104 | 105 | events = Enum.reverse(events) 106 | 107 | new_remaining_demand = remaining_demand - Enum.count(events) 108 | 109 | if remaining_demand > 0 and new_remaining_demand == 0 do 110 | log_caught_up(state) 111 | end 112 | 113 | {:noreply, events, %State{state | time: new_time, remaining_demand: new_remaining_demand}} 114 | end 115 | 116 | defp log_caught_up(%State{debug_logging: false}), do: :ok 117 | 118 | defp log_caught_up(%State{debug_logging: true}), 119 | do: 120 | Logger.debug(fn -> 121 | {"Clock Producer caught up with past times and is now running in normal time", 122 | node: Node.self()} 123 | end) 124 | end 125 | -------------------------------------------------------------------------------- /test/support/test_storage.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Storage.Test do 2 | @moduledoc """ 3 | Test implementation of a `Quantum.Storage`. 4 | """ 5 | 6 | @behaviour Quantum.Storage 7 | 8 | use GenServer 9 | 10 | def start_link(_opts), do: send_and_wait(:jobs, :start_link, :ignore) 11 | 12 | @doc false 13 | @impl GenServer 14 | def init(_args), do: {:ok, nil} 15 | 16 | @impl Quantum.Storage 17 | def jobs(_storage_pid), do: send_and_wait(:jobs, nil, :not_applicable) 18 | 19 | @impl Quantum.Storage 20 | def add_job(_storage_pid, job), do: send_and_wait(:add_job, job) 21 | 22 | @impl Quantum.Storage 23 | def delete_job(_storage_pid, job_name), do: send_and_wait(:delete_job, job_name) 24 | 25 | @impl Quantum.Storage 26 | def update_job_state(_storage_pid, job_name, state), 27 | do: send_and_wait(:update_job_state, {job_name, state}) 28 | 29 | @impl Quantum.Storage 30 | def last_execution_date(_storage_pid), do: send_and_wait(:last_execution_date, nil, :unknown) 31 | 32 | @impl Quantum.Storage 33 | def update_last_execution_date(_storage_pid, last_execution_date), 34 | do: send_and_wait(:update_last_execution_date, last_execution_date) 35 | 36 | @impl Quantum.Storage 37 | def purge(_storage_pid), do: send_and_wait(:purge, nil) 38 | 39 | # Used for Small Test Storages 40 | defmacro __using__(_) do 41 | quote do 42 | @behaviour Quantum.Storage 43 | 44 | import Quantum.Storage.Test 45 | 46 | use GenServer 47 | 48 | def start_link(_opts), do: send_and_wait(:jobs, :start_link, :ignore) 49 | 50 | @doc false 51 | @impl GenServer 52 | def init(_args), do: {:ok, nil} 53 | 54 | @impl Quantum.Storage 55 | def jobs(_storage_pid), do: send_and_wait(:jobs, nil, :not_applicable) 56 | 57 | @impl Quantum.Storage 58 | def add_job(_storage_pid, job), do: send_and_wait(:add_job, job) 59 | 60 | @impl Quantum.Storage 61 | def delete_job(_storage_pid, job_name), do: send_and_wait(:delete_job, job_name) 62 | 63 | @impl Quantum.Storage 64 | def update_job_state(_storage_pid, job_name, state), 65 | do: send_and_wait(:update_job_state, job_name, state) 66 | 67 | @impl Quantum.Storage 68 | def last_execution_date(_storage_pid), 69 | do: send_and_wait(:last_execution_date, nil, :unknown) 70 | 71 | @impl Quantum.Storage 72 | def update_last_execution_date(_storage_pid, last_execution_date), 73 | do: send_and_wait(:update_last_execution_date, last_execution_date) 74 | 75 | @impl Quantum.Storage 76 | def purge(_storage_pid), do: send_and_wait(:purge, nil) 77 | 78 | defoverridable Quantum.Storage 79 | end 80 | end 81 | 82 | def send_and_wait(fun, args, default \\ :ok) do 83 | test_pid = find_test_pid(self()) 84 | 85 | if !is_nil(test_pid) do 86 | ref = make_ref() 87 | 88 | send(test_pid, {fun, args, {self(), ref}}) 89 | end 90 | 91 | default 92 | end 93 | 94 | defp find_test_pid(pid) do 95 | pid 96 | |> Process.info() 97 | |> case do 98 | nil -> [] 99 | other -> other 100 | end 101 | |> Keyword.get(:dictionary, []) 102 | |> Map.new() 103 | |> case do 104 | %{test_pid: pid} -> 105 | pid 106 | 107 | %{"$ancestors": ancestors} -> 108 | Enum.find_value(ancestors, fn ancestor_pid -> 109 | find_test_pid(ancestor_pid) 110 | end) 111 | 112 | _ -> 113 | nil 114 | end 115 | end 116 | end 117 | 118 | defmodule Quantum.Storage.TestWithUpdate do 119 | @moduledoc """ 120 | Test implementation of a `Quantum.Storage` that overrides `c:update_job/2`. 121 | """ 122 | use Quantum.Storage.Test 123 | 124 | @impl Quantum.Storage 125 | def update_job(_storage_pid, job), do: send_and_wait(:update_job, job) 126 | end 127 | -------------------------------------------------------------------------------- /lib/quantum/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Supervisor do 2 | @moduledoc false 3 | 4 | use Supervisor 5 | 6 | # Starts the quantum supervisor. 7 | @spec start_link(GenServer.server(), Keyword.t()) :: GenServer.on_start() 8 | def start_link(quantum, opts) do 9 | name = Keyword.take(opts, [:name]) 10 | Supervisor.start_link(__MODULE__, {quantum, opts}, name) 11 | end 12 | 13 | @impl Supervisor 14 | def init({scheduler, opts}) do 15 | %{ 16 | storage: storage, 17 | scheduler: ^scheduler, 18 | task_supervisor_name: task_supervisor_name, 19 | storage_name: storage_name, 20 | task_registry_name: task_registry_name, 21 | clock_broadcaster_name: clock_broadcaster_name, 22 | job_broadcaster_name: job_broadcaster_name, 23 | execution_broadcaster_name: execution_broadcaster_name, 24 | node_selector_broadcaster_name: node_selector_broadcaster_name, 25 | executor_supervisor_name: executor_supervisor_name 26 | } = 27 | opts = 28 | opts 29 | |> scheduler.config 30 | |> quantum_init(scheduler) 31 | |> Map.new() 32 | 33 | task_supervisor_opts = [name: task_supervisor_name] 34 | 35 | storage_opts = 36 | opts 37 | |> Map.get(:storage_opts, []) 38 | |> Keyword.put(:scheduler, scheduler) 39 | |> Keyword.put(:name, storage_name) 40 | 41 | task_registry_opts = %Quantum.TaskRegistry.StartOpts{name: task_registry_name} 42 | 43 | clock_broadcaster_opts = 44 | struct!( 45 | Quantum.ClockBroadcaster.StartOpts, 46 | opts 47 | |> Map.take([:debug_logging, :storage, :scheduler]) 48 | |> Map.put(:name, clock_broadcaster_name) 49 | |> Map.put(:start_time, DateTime.utc_now()) 50 | ) 51 | 52 | job_broadcaster_opts = 53 | struct!( 54 | Quantum.JobBroadcaster.StartOpts, 55 | opts 56 | |> Map.take([:jobs, :storage, :scheduler, :debug_logging]) 57 | |> Map.put(:name, job_broadcaster_name) 58 | ) 59 | 60 | execution_broadcaster_opts = 61 | struct!( 62 | Quantum.ExecutionBroadcaster.StartOpts, 63 | opts 64 | |> Map.take([ 65 | :storage, 66 | :scheduler, 67 | :debug_logging 68 | ]) 69 | |> Map.put(:job_broadcaster_reference, job_broadcaster_name) 70 | |> Map.put(:clock_broadcaster_reference, clock_broadcaster_name) 71 | |> Map.put(:name, execution_broadcaster_name) 72 | ) 73 | 74 | node_selector_broadcaster_opts = %Quantum.NodeSelectorBroadcaster.StartOpts{ 75 | execution_broadcaster_reference: execution_broadcaster_name, 76 | task_supervisor_reference: task_supervisor_name, 77 | name: node_selector_broadcaster_name 78 | } 79 | 80 | executor_supervisor_opts = 81 | struct!( 82 | Quantum.ExecutorSupervisor.StartOpts, 83 | opts 84 | |> Map.take([:debug_logging]) 85 | |> Map.put(:node_selector_broadcaster_reference, node_selector_broadcaster_name) 86 | |> Map.put(:task_supervisor_reference, task_supervisor_name) 87 | |> Map.put(:task_registry_reference, task_registry_name) 88 | |> Map.put(:name, executor_supervisor_name) 89 | |> Map.put(:scheduler, scheduler) 90 | ) 91 | 92 | Supervisor.init( 93 | [ 94 | {Task.Supervisor, task_supervisor_opts}, 95 | {storage, storage_opts}, 96 | {Quantum.ClockBroadcaster, clock_broadcaster_opts}, 97 | {Quantum.TaskRegistry, task_registry_opts}, 98 | {Quantum.JobBroadcaster, job_broadcaster_opts}, 99 | {Quantum.ExecutionBroadcaster, execution_broadcaster_opts}, 100 | {Quantum.NodeSelectorBroadcaster, node_selector_broadcaster_opts}, 101 | {Quantum.ExecutorSupervisor, executor_supervisor_opts} 102 | ], 103 | strategy: :rest_for_one 104 | ) 105 | end 106 | 107 | # Run Optional Callback in Quantum Scheduler Implementation 108 | @spec quantum_init(Keyword.t(), atom) :: Keyword.t() 109 | defp quantum_init(config, scheduler) do 110 | scheduler.init(config) 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /test/quantum/clock_broadcaster_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ClockBroadcasterTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case 5 | 6 | alias Quantum.ClockBroadcaster 7 | alias Quantum.ClockBroadcaster.Event 8 | alias Quantum.ClockBroadcaster.StartOpts 9 | 10 | @listen_events 10 11 | 12 | defmodule TestScheduler do 13 | @moduledoc false 14 | 15 | use Quantum, otp_app: :quantum 16 | end 17 | 18 | test "should only generate event from event struct", %{test: test} do 19 | events = 20 | test 21 | |> stream_broadcaster!() 22 | |> Stream.take(1) 23 | |> Enum.to_list() 24 | 25 | assert [%Event{}] = events 26 | end 27 | 28 | test "should only generate event every second", %{test: test} do 29 | self = self() 30 | 31 | test 32 | |> stream_broadcaster!() 33 | |> Stream.take(@listen_events) 34 | |> Stream.each(fn event -> 35 | send(self, {:event, event, DateTime.utc_now()}) 36 | end) 37 | |> Enum.to_list() 38 | 39 | for _ <- 1..@listen_events do 40 | assert_received {:event, _, _} = message 41 | 42 | assert_live_event(message) 43 | end 44 | end 45 | 46 | test "catches up fast and then one every second", %{test: test} do 47 | self = self() 48 | 49 | start_time = DateTime.add(DateTime.utc_now(), -@listen_events, :second) 50 | 51 | test 52 | |> stream_broadcaster!(start_time: start_time) 53 | |> Stream.take(@listen_events + 1) 54 | |> Stream.each(&receive_send(self, &1)) 55 | |> Enum.to_list() 56 | 57 | for _ <- 1..@listen_events do 58 | assert_received {:event, _, _} = message 59 | 60 | assert_catch_up_event(message) 61 | end 62 | 63 | assert_received {:event, _, _} = message 64 | 65 | assert_live_event(message) 66 | end 67 | 68 | test "should wait for future date until", %{test: test} do 69 | self = self() 70 | 71 | start_time = DateTime.add(DateTime.utc_now(), 2, :second) 72 | 73 | test 74 | |> stream_broadcaster!(start_time: start_time) 75 | |> Stream.take(@listen_events) 76 | |> Stream.each(&receive_send(self, &1)) 77 | |> Enum.to_list() 78 | 79 | for _ <- 1..@listen_events do 80 | assert_received {:event, _, _} = message 81 | 82 | assert_live_event(message) 83 | end 84 | end 85 | 86 | defp receive_send(pid, event) do 87 | send(pid, {:event, event, DateTime.utc_now()}) 88 | end 89 | 90 | defp assert_live_event(message) do 91 | assert {:event, 92 | %Event{ 93 | time: 94 | %DateTime{ 95 | year: year, 96 | month: month, 97 | day: day, 98 | hour: hour, 99 | minute: minute, 100 | second: second 101 | } = event_time, 102 | catch_up: false 103 | }, 104 | %DateTime{ 105 | year: year, 106 | month: month, 107 | day: day, 108 | hour: hour, 109 | minute: minute, 110 | second: second 111 | } = receive_time} = message 112 | 113 | assert DateTime.diff(event_time, receive_time, :millisecond) < 100 114 | end 115 | 116 | defp assert_catch_up_event(message) do 117 | assert {:event, 118 | %Event{ 119 | catch_up: true 120 | }, _} = message 121 | end 122 | 123 | defp start_broadcaster!(test, opts) do 124 | start_supervised!( 125 | {ClockBroadcaster, 126 | struct!( 127 | StartOpts, 128 | Keyword.merge( 129 | [ 130 | name: Module.concat(__MODULE__, test), 131 | debug_logging: false, 132 | start_time: DateTime.utc_now(), 133 | storage: Quantum.Storage.Test, 134 | scheduler: NotNeeded 135 | ], 136 | Enum.to_list(opts) 137 | ) 138 | )} 139 | ) 140 | end 141 | 142 | defp stream_broadcaster!(test, opts \\ %{}) do 143 | GenStage.stream([{start_broadcaster!(test, opts), max_demand: 1000}]) 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /lib/quantum/executor.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Executor do 2 | @moduledoc false 3 | 4 | # Task to actually execute a Task 5 | 6 | use Task 7 | 8 | require Logger 9 | 10 | alias Quantum.{ 11 | Job, 12 | NodeSelectorBroadcaster.Event, 13 | TaskRegistry 14 | } 15 | 16 | alias __MODULE__.StartOpts 17 | 18 | @spec start_link(StartOpts.t(), Event.t()) :: {:ok, pid} 19 | def start_link(opts, %Event{job: job, node: node}) do 20 | Task.start_link(fn -> 21 | execute(opts, job, node) 22 | end) 23 | end 24 | 25 | @spec execute(StartOpts.t(), Job.t(), Node.t()) :: :ok 26 | # Execute task on all given nodes without checking for overlap 27 | defp execute( 28 | %StartOpts{ 29 | task_supervisor_reference: task_supervisor, 30 | debug_logging: debug_logging, 31 | scheduler: scheduler 32 | }, 33 | %Job{overlap: true} = job, 34 | node 35 | ) do 36 | run(node, job, task_supervisor, debug_logging, scheduler) 37 | 38 | :ok 39 | end 40 | 41 | # Execute task on all given nodes with checking for overlap 42 | defp execute( 43 | %StartOpts{ 44 | task_supervisor_reference: task_supervisor, 45 | task_registry_reference: task_registry, 46 | debug_logging: debug_logging, 47 | scheduler: scheduler 48 | }, 49 | %Job{overlap: false, name: job_name} = job, 50 | node 51 | ) do 52 | debug_logging && 53 | Logger.debug(fn -> 54 | {"Start execution of job", node: Node.self(), name: job_name} 55 | end) 56 | 57 | case TaskRegistry.mark_running(task_registry, job_name, node) do 58 | :marked_running -> 59 | %Task{ref: ref} = run(node, job, task_supervisor, debug_logging, scheduler) 60 | 61 | receive do 62 | {^ref, _} -> 63 | TaskRegistry.mark_finished(task_registry, job_name, node) 64 | 65 | {:DOWN, ^ref, _, _, _} -> 66 | TaskRegistry.mark_finished(task_registry, job_name, node) 67 | 68 | :ok 69 | end 70 | 71 | _ -> 72 | :ok 73 | end 74 | end 75 | 76 | # TODO: Remove once the following issue is solved: 77 | # https://github.com/elixir-lang/elixir/issues/14655 78 | @dialyzer {:nowarn_function, run: 5} 79 | 80 | # Ececute the given function on a given node via the task supervisor 81 | @spec run(Node.t(), Job.t(), GenServer.server(), boolean(), atom()) :: Task.t() 82 | defp run( 83 | node, 84 | %Job{name: job_name, task: task} = job, 85 | task_supervisor, 86 | debug_logging, 87 | scheduler 88 | ) do 89 | debug_logging && 90 | Logger.debug(fn -> 91 | {"Task for job started on node", node: Node.self(), name: job_name, started_on: node} 92 | end) 93 | 94 | Task.Supervisor.async_nolink({task_supervisor, node}, fn -> 95 | debug_logging && 96 | Logger.debug(fn -> 97 | {"Execute started for job", node: Node.self(), name: job_name} 98 | end) 99 | 100 | try do 101 | :telemetry.span([:quantum, :job], %{job: job, node: node, scheduler: scheduler}, fn -> 102 | result = execute_task(task) 103 | {result, %{job: job, node: node, scheduler: scheduler, result: result}} 104 | end) 105 | catch 106 | type, value -> 107 | debug_logging && 108 | Logger.debug(fn -> 109 | { 110 | "Execution failed for job", 111 | node: Node.self(), name: job_name, type: type, value: value 112 | } 113 | end) 114 | 115 | log_exception(type, value, __STACKTRACE__) 116 | else 117 | result -> 118 | debug_logging && 119 | Logger.debug(fn -> 120 | {"Execution ended for job", node: Node.self(), name: job_name, result: result} 121 | end) 122 | end 123 | 124 | :ok 125 | end) 126 | end 127 | 128 | # Run function 129 | @spec execute_task(Quantum.Job.task()) :: any 130 | defp execute_task({mod, fun, args}) do 131 | :erlang.apply(mod, fun, args) 132 | end 133 | 134 | defp execute_task(fun) when is_function(fun, 0) do 135 | fun.() 136 | end 137 | 138 | def log_exception(kind, reason, stacktrace) do 139 | reason = Exception.normalize(kind, reason, stacktrace) 140 | 141 | # TODO: Remove in a future version and make elixir 1.10 minimum requirement 142 | if Version.match?(System.version(), "< 1.10.0") do 143 | Logger.error(Exception.format(kind, reason, stacktrace)) 144 | else 145 | crash_reason = 146 | case kind do 147 | :throw -> {{:nocatch, reason}, stacktrace} 148 | _ -> {reason, stacktrace} 149 | end 150 | 151 | Logger.error( 152 | Exception.format(kind, reason, stacktrace), 153 | crash_reason: crash_reason 154 | ) 155 | end 156 | end 157 | end 158 | -------------------------------------------------------------------------------- /lib/quantum/normalizer.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Normalizer do 2 | @moduledoc false 3 | 4 | # Normalize Config values into a `Quantum.Job`. 5 | 6 | alias Crontab.CronExpression 7 | alias Crontab.CronExpression.Parser, as: CronExpressionParser 8 | 9 | alias Quantum.{ 10 | Job, 11 | RunStrategy.NodeList 12 | } 13 | 14 | @type config_short_notation :: {config_schedule, config_task} 15 | @type config_full_notation :: {config_name | nil, Keyword.t() | map} 16 | 17 | @type config_schedule :: 18 | CronExpression.t() | String.t() | {:cron, String.t()} | {:extended, String.t()} 19 | @type config_task :: {module, fun, [any]} | (-> any) 20 | @type config_name :: String.t() | atom 21 | 22 | # Normalize Config Input into `Quantum.Job`. 23 | # 24 | # ### Parameters: 25 | # 26 | # * `base` - Empty `Quantum.Job` 27 | # * `job` - The Job To Normalize 28 | @spec normalize(Job.t(), config_full_notation | config_short_notation | Job.t()) :: 29 | Job.t() | no_return 30 | def normalize(base, job) 31 | 32 | def normalize(%Job{} = base, job) when is_list(job) do 33 | normalize_options(base, Map.new(job)) 34 | end 35 | 36 | def normalize(%Job{} = base, {job_name, opts}) when is_list(opts) do 37 | normalize(base, {job_name, Map.new(opts)}) 38 | end 39 | 40 | def normalize(%Job{} = base, {nil, opts}) when is_map(opts) do 41 | normalize_options(base, opts) 42 | end 43 | 44 | def normalize(%Job{} = base, {job_name, opts}) when is_map(opts) do 45 | opts = Map.put(opts, :name, job_name) 46 | 47 | normalize_options(base, opts) 48 | end 49 | 50 | def normalize(%Job{} = base, {schedule, task}) do 51 | normalize_options(base, %{schedule: schedule, task: task}) 52 | end 53 | 54 | def normalize(%Job{} = base, {schedule, task, opts}) when is_list(opts) do 55 | opts = 56 | Enum.into(opts, %{}) 57 | |> Map.merge(%{schedule: schedule, task: task}) 58 | 59 | normalize_options(base, opts) 60 | end 61 | 62 | def normalize(%Job{} = _base, %Job{} = job) do 63 | job 64 | end 65 | 66 | @spec normalize_options(Job.t(), map) :: Job.t() 67 | defp normalize_options(job, options) do 68 | Enum.reduce(options, job, &normalize_job_option/2) 69 | end 70 | 71 | @spec normalize_job_option({atom, any}, Job.t()) :: Job.t() 72 | defp normalize_job_option({:name, name}, job) do 73 | Job.set_name(job, normalize_name(name)) 74 | end 75 | 76 | defp normalize_job_option({:schedule, schedule}, job) do 77 | Job.set_schedule(job, normalize_schedule(schedule)) 78 | end 79 | 80 | defp normalize_job_option({:task, task}, job) do 81 | Job.set_task(job, normalize_task(task)) 82 | end 83 | 84 | defp normalize_job_option({:run_strategy, run_strategy}, job) do 85 | Job.set_run_strategy(job, normalize_run_strategy(run_strategy)) 86 | end 87 | 88 | defp normalize_job_option({:overlap, overlap}, job) do 89 | Job.set_overlap(job, overlap) 90 | end 91 | 92 | defp normalize_job_option({:timezone, timezone}, job) do 93 | Job.set_timezone(job, normalize_timezone(timezone)) 94 | end 95 | 96 | defp normalize_job_option({:state, state}, job) do 97 | Job.set_state(job, state) 98 | end 99 | 100 | defp normalize_job_option(_, job), do: job 101 | 102 | @spec normalize_task(config_task) :: Job.task() | no_return 103 | defp normalize_task({mod, fun, args}), do: {mod, fun, args} 104 | defp normalize_task(fun) when is_function(fun, 0), do: fun 105 | 106 | defp normalize_task(fun) when is_function(fun), 107 | do: raise("Only 0 arity functions are supported via the short syntax.") 108 | 109 | @spec normalize_schedule(config_schedule) :: Job.schedule() | no_return 110 | def normalize_schedule(nil), do: nil 111 | def normalize_schedule(%CronExpression{} = e), do: e 112 | 113 | def normalize_schedule(e) when is_binary(e), 114 | do: e |> String.downcase() |> CronExpressionParser.parse!() 115 | 116 | def normalize_schedule({:cron, e}) when is_binary(e), 117 | do: e |> String.downcase() |> CronExpressionParser.parse!() 118 | 119 | def normalize_schedule({:extended, e}) when is_binary(e), 120 | do: e |> String.downcase() |> CronExpressionParser.parse!(true) 121 | 122 | @spec normalize_name(atom | String.t()) :: atom 123 | defp normalize_name(name) when is_binary(name), do: String.to_atom(name) 124 | defp normalize_name(name) when is_atom(name), do: name 125 | 126 | @spec normalize_run_strategy({module, any} | module) :: NodeList 127 | defp normalize_run_strategy(strategy) when is_atom(strategy) do 128 | strategy.normalize_config!(nil) 129 | end 130 | 131 | defp normalize_run_strategy({strategy, options}) when is_atom(strategy) do 132 | strategy.normalize_config!(options) 133 | end 134 | 135 | @spec normalize_timezone(String.t() | :utc) :: String.t() | :utc 136 | defp normalize_timezone(timezone) when is_binary(timezone), do: timezone 137 | defp normalize_timezone(:utc), do: :utc 138 | defp normalize_timezone(timezone), do: raise("Invalid timezone: #{inspect(timezone)}") 139 | end 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Quantum 2 | 3 | [![.github/workflows/branch_main.yml](https://github.com/quantum-elixir/quantum-core/actions/workflows/branch_main.yml/badge.svg)](https://github.com/quantum-elixir/quantum-core/actions/workflows/branch_main.yml) 4 | [![Coverage Status](https://coveralls.io/repos/quantum-elixir/quantum-core/badge.svg?branch=main)](https://coveralls.io/r/quantum-elixir/quantum-core?branch=main) 5 | [![Module Version](https://img.shields.io/hexpm/v/quantum.svg)](https://hex.pm/packages/quantum) 6 | [![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/quantum/) 7 | [![Total Download](https://img.shields.io/hexpm/dt/quantum.svg)](https://hex.pm/packages/quantum) 8 | [![License](https://img.shields.io/hexpm/l/quantum.svg)](https://github.com/quantum-elixir/quantum-core/blob/main/LICENSE) 9 | [![Last Updated](https://img.shields.io/github/last-commit/quantum-elixir/quantum-core.svg)](https://github.com/quantum-elixir/quantum-core/commits/main) 10 | 11 | > **This README follows main, which may not be the currently published version**. Here are the 12 | [docs for the latest published version of Quantum](https://hexdocs.pm/quantum/readme.html). 13 | 14 | [Cron](https://en.wikipedia.org/wiki/Cron)-like job scheduler for [Elixir](http://elixir-lang.org/). 15 | 16 | ## Setup 17 | 18 | To use Quantum in your project, edit the `mix.exs` file and add `Quantum` to 19 | 20 | **1. the list of dependencies:** 21 | ```elixir 22 | defp deps do 23 | [ 24 | {:quantum, "~> 3.0"} 25 | ] 26 | end 27 | ``` 28 | 29 | **2. and create a scheduler for your app:** 30 | ```elixir 31 | defmodule Acme.Scheduler do 32 | use Quantum, otp_app: :your_app 33 | end 34 | ``` 35 | 36 | **3. and your application's supervision tree:** 37 | ```elixir 38 | defmodule Acme.Application do 39 | use Application 40 | 41 | def start(_type, _args) do 42 | children = [ 43 | # This is the new line 44 | Acme.Scheduler 45 | ] 46 | 47 | opts = [strategy: :one_for_one, name: Acme.Supervisor] 48 | Supervisor.start_link(children, opts) 49 | end 50 | end 51 | ``` 52 | 53 | ## Troubleshooting 54 | 55 | To see more transparently what `quantum` is doing, configure the `logger` to display `:debug` messages. 56 | 57 | ```elixir 58 | config :logger, level: :debug 59 | ``` 60 | 61 | If you want do use the logger in debug-level without the messages from quantum: 62 | 63 | ```elixir 64 | config :acme, Acme.Scheduler, 65 | debug_logging: false 66 | ``` 67 | 68 | If you encounter any problems with `quantum`, please search if there is already an 69 | [open issue](https://github.com/quantum-elixir/quantum-core/issues) addressing the problem. 70 | 71 | Otherwise feel free to [open an issue](https://github.com/quantum-elixir/quantum-core/issues/new). Please include debug logs. 72 | 73 | ## Usage 74 | 75 | Configure your cronjobs in your `config/config.exs` like this: 76 | 77 | ```elixir 78 | config :acme, Acme.Scheduler, 79 | jobs: [ 80 | # Every minute 81 | {"* * * * *", {Heartbeat, :send, []}}, 82 | # Every 15 minutes 83 | {"*/15 * * * *", fn -> System.cmd("rm", ["/tmp/tmp_"]) end}, 84 | # Runs on 18, 20, 22, 0, 2, 4, 6: 85 | {"0 18-6/2 * * *", fn -> :mnesia.backup('/var/backup/mnesia') end}, 86 | # Runs every midnight: 87 | {"@daily", {Backup, :backup, []}} 88 | ] 89 | ``` 90 | 91 | More details on the usage can be found in the [Documentation](https://hexdocs.pm/quantum/configuration.html) 92 | 93 | ## Contribution 94 | 95 | This project uses the [Collective Code Construction Contract (C4)](http://rfc.zeromq.org/spec:42/C4/) for all code changes. 96 | 97 | > "Everyone, without distinction or discrimination, SHALL have an equal right to become a Contributor under the terms of this contract." 98 | 99 | ### TL;DR 100 | 101 | 1. Check for [open issues](https://github.com/quantum-elixir/quantum-core/issues) or [open a new issue](https://github.com/quantum-elixir/quantum-core/issues/new) to start a discussion around [a problem](https://www.youtube.com/watch?v=_QF9sFJGJuc). 102 | 2. Issues SHALL be named as "Problem: _description of the problem_". 103 | 3. Fork the [quantum-elixir repository on GitHub](https://github.com/quantum-elixir/quantum-core) to start making your changes 104 | 4. If possible, write a test which shows that the problem was solved. 105 | 5. Send a pull request. 106 | 6. Pull requests SHALL be named as "Solution: _description of your solution_" 107 | 7. Your pull request is merged and you are added to the [list of contributors](https://github.com/quantum-elixir/quantum-core/graphs/contributors) 108 | 109 | ## Copyright and License 110 | 111 | Copyright (c) 2015 Constantin Rack 112 | 113 | Licensed under the Apache License, Version 2.0 (the "License"); 114 | you may not use this file except in compliance with the License. 115 | You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 116 | 117 | Unless required by applicable law or agreed to in writing, software 118 | distributed under the License is distributed on an "AS IS" BASIS, 119 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 120 | See the License for the specific language governing permissions and 121 | limitations under the License. 122 | -------------------------------------------------------------------------------- /pages/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | Configure your cronjobs in your `config/config.exs` like this: 4 | 5 | ```elixir 6 | config :your_app, YourApp.Scheduler, 7 | jobs: [ 8 | # Every minute 9 | {"* * * * *", {Heartbeat, :send, []}}, 10 | {{:cron, "* * * * *"}, {Heartbeat, :send, []}}, 11 | # Every second 12 | {{:extended, "* * * * *"}, {Heartbeat, :send, []}}, 13 | # Every 15 minutes 14 | {"*/15 * * * *", fn -> System.cmd("rm", ["/tmp/tmp_"]) end}, 15 | # Runs on 18, 20, 22, 0, 2, 4, 6: 16 | {"0 18-6/2 * * *", fn -> :mnesia.backup('/var/backup/mnesia') end}, 17 | # Runs every midnight: 18 | {"@daily", {Backup, :backup, []}, state: :inactive} 19 | ] 20 | ``` 21 | 22 | ## Persistent Storage 23 | 24 | Persistent storage can be used to track jobs and last execution times over restarts. 25 | 26 | **Note: If a storage is present, the jobs from the configuration will not be loaded to prevent conflicts.** 27 | 28 | ```elixir 29 | config :your_app, YourApp.Scheduler, 30 | storage: Quantum.Storage.Implementation 31 | ``` 32 | 33 | ### Storage Adapters 34 | 35 | Storage implementations must implement the `Quantum.Storage` behaviour. 36 | 37 | The following adapters are supported: 38 | 39 | * [`PersistentEts`](https://hex.pm/packages/quantum_storage_persistent_ets) 40 | * [`Mnesia`](https://hex.pm/packages/quantum_storage_mnesia) 41 | 42 | ## Release managers 43 | ( 44 | [conform](https://github.com/bitwalker/conform) / 45 | [distillery](https://github.com/bitwalker/distillery) / 46 | [exrm](https://github.com/bitwalker/exrm) / 47 | [edeliver](https://github.com/boldpoker/edeliver) 48 | ) 49 | 50 | Please note that the following config notation **is not supported** by release managers. 51 | 52 | ```elixir 53 | {"* * * * *", fn -> :anonymous_function end} 54 | ``` 55 | 56 | ## Named Jobs 57 | 58 | You can define named jobs in your config like this: 59 | 60 | ```elixir 61 | config :your_app, YourApp.Scheduler, 62 | jobs: [ 63 | news_letter: [ 64 | schedule: "@weekly", 65 | task: {Heartbeat, :send, [:arg1]}, 66 | ] 67 | ] 68 | ``` 69 | 70 | Possible options: 71 | - `schedule` cron schedule, ex: `"@weekly"` / `"1 * * * *"` / `{:cron, "1 * * * *"}` or `{:extended, "1 * * * *"}` 72 | - `task` function to be performed, ex: `{Heartbeat, :send, []}` or `fn -> :something end` 73 | - `run_strategy` strategy on how to run tasks inside of cluster, default: `%Quantum.RunStrategy.Random{nodes: :cluster}` 74 | - `overlap` set to false to prevent next job from being executed if previous job is still running, default: `true` 75 | - `state` set to `:inactive` to deactivate a job or `:active` to activate it 76 | 77 | It is possible to control the behavior of jobs at runtime. 78 | 79 | ## Override default settings 80 | 81 | The default job settings can be configured as shown in the example below. 82 | So if you have a lot of jobs and do not want to override the 83 | default setting in every job, you can set them globally. 84 | 85 | ```elixir 86 | config :your_app, YourApp.Scheduler, 87 | schedule: "* * * * *", 88 | overlap: false, 89 | timezone: :utc, 90 | jobs: [ 91 | # Your cronjobs 92 | ] 93 | ``` 94 | 95 | ## Jobs with Second granularity 96 | 97 | It is possible to specify jobs with second granularity. 98 | To do this the `schedule` parameter has to be provided with a `{:extended, "1 * * * *"}` expression. 99 | 100 | ```elixir 101 | config :your_app, YourApp.Scheduler, 102 | jobs: [ 103 | news_letter: [ 104 | schedule: {:extended, "*/2"}, # Runs every two seconds 105 | task: {Heartbeat, :send, [:arg1]} 106 | ] 107 | ] 108 | ``` 109 | 110 | ## GenServer timeout 111 | 112 | Sometimes, you may come across GenServer timeout errors esp. when you have 113 | too many jobs or high load. The default `GenServer.call` timeout is 5000. 114 | You can override this default by specifying `timeout` setting in configuration. 115 | 116 | ```elixir 117 | config :your_app, YourApp.Scheduler, 118 | timeout: 30_000 119 | ``` 120 | 121 | Or if you wish to wait indefinitely: 122 | 123 | ```elixir 124 | config :your_app, YourApp.Scheduler, 125 | timeout: :infinity 126 | ``` 127 | 128 | ## Timezone Support 129 | 130 | Please note that Quantum uses **UTC timezone** and not local timezone. 131 | 132 | Before changing the timezone you need to install [Tzdata](https://github.com/lau/tzdata) and add the following line in your config file. 133 | 134 | ```elixir 135 | config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase 136 | ``` 137 | 138 | Now you can specify another default timezone, add the following `timezone` option to your configuration: 139 | 140 | ```elixir 141 | config :your_app, YourApp.Scheduler, 142 | timezone: "America/Chicago", 143 | jobs: [ 144 | # Your cronjobs 145 | ] 146 | ``` 147 | 148 | Valid options are `:utc` or a timezone name such as `"America/Chicago"`. A full list of timezone names can be downloaded from https://www.iana.org/time-zones, or at https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. 149 | 150 | Timezones can also be configured on a per-job basis. This overrides the default Quantum timezone for a particular job. To set the timezone on a job, use the `timezone` key when creating the `Quantum.Job` structure. 151 | 152 | ```elixir 153 | %Quantum.Job{ 154 | # ... 155 | timezone: "America/New_York" 156 | } 157 | ``` 158 | -------------------------------------------------------------------------------- /test/quantum/normalizer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.NormalizerTest do 2 | use ExUnit.Case, async: true 3 | 4 | import Quantum.Normalizer 5 | import Crontab.CronExpression 6 | 7 | alias Quantum.Job 8 | 9 | defmodule Scheduler do 10 | use Quantum, otp_app: :quantum_test 11 | end 12 | 13 | setup_all do 14 | Application.put_env(:quantum_test, Scheduler, jobs: []) 15 | 16 | {:ok, _pid} = start_supervised(Scheduler) 17 | 18 | :ok 19 | end 20 | 21 | test "named job" do 22 | job = { 23 | :newsletter, 24 | [ 25 | schedule: ~e[@weekly], 26 | task: {MyModule, :my_method, [1, 2, 3]}, 27 | overlap: false 28 | ] 29 | } 30 | 31 | expected_job = 32 | Scheduler.new_job() 33 | |> Job.set_name(:newsletter) 34 | |> Job.set_schedule(~e[@weekly]) 35 | |> Job.set_task({MyModule, :my_method, [1, 2, 3]}) 36 | |> Job.set_overlap(false) 37 | 38 | assert normalize(Scheduler.new_job(), job) == expected_job 39 | end 40 | 41 | test "expression tuple extended" do 42 | job = { 43 | :newsletter, 44 | [ 45 | schedule: {:extended, "*"}, 46 | task: {MyModule, :my_method, [1, 2, 3]}, 47 | overlap: false 48 | ] 49 | } 50 | 51 | expected_job = 52 | Scheduler.new_job() 53 | |> Job.set_name(:newsletter) 54 | |> Job.set_schedule(~e[*]e) 55 | |> Job.set_task({MyModule, :my_method, [1, 2, 3]}) 56 | |> Job.set_overlap(false) 57 | 58 | assert normalize(Scheduler.new_job(), job) == expected_job 59 | end 60 | 61 | test "normalizer of run strategy" do 62 | job = { 63 | :newsletter, 64 | [ 65 | run_strategy: {Quantum.RunStrategy.All, [:node@host]} 66 | ] 67 | } 68 | 69 | expected_job = 70 | Scheduler.new_job() 71 | |> Job.set_name(:newsletter) 72 | |> Job.set_run_strategy(%Quantum.RunStrategy.All{nodes: [:node@host]}) 73 | 74 | assert normalize(Scheduler.new_job(), job) == expected_job 75 | end 76 | 77 | test "normalizer of state" do 78 | job = { 79 | :newsletter, 80 | [ 81 | state: :inactive 82 | ] 83 | } 84 | 85 | expected_job = 86 | Scheduler.new_job() 87 | |> Job.set_name(:newsletter) 88 | |> Job.set_state(:inactive) 89 | 90 | assert normalize(Scheduler.new_job(), job) == expected_job 91 | end 92 | 93 | test "expression tuple not extended" do 94 | job = { 95 | :newsletter, 96 | [ 97 | schedule: {:cron, "*"}, 98 | task: {MyModule, :my_method, [1, 2, 3]}, 99 | overlap: false 100 | ] 101 | } 102 | 103 | expected_job = 104 | Scheduler.new_job() 105 | |> Job.set_name(:newsletter) 106 | |> Job.set_schedule(~e[*]) 107 | |> Job.set_task({MyModule, :my_method, [1, 2, 3]}) 108 | |> Job.set_overlap(false) 109 | 110 | assert normalize(Scheduler.new_job(), job) == expected_job 111 | end 112 | 113 | test "named job with old schedule" do 114 | job = { 115 | :newsletter, 116 | [ 117 | schedule: "@weekly", 118 | task: {MyModule, :my_method, [1, 2, 3]}, 119 | overlap: false 120 | ] 121 | } 122 | 123 | expected_job = 124 | Scheduler.new_job() 125 | |> Job.set_name(:newsletter) 126 | |> Job.set_schedule(~e[@weekly]) 127 | |> Job.set_task({MyModule, :my_method, [1, 2, 3]}) 128 | |> Job.set_overlap(false) 129 | 130 | assert normalize(Scheduler.new_job(), job) == expected_job 131 | end 132 | 133 | test "unnamed job as tuple" do 134 | schedule = ~e[* * * * *] 135 | task = {MyModule, :my_method, [1, 2, 3]} 136 | 137 | assert %{schedule: ^schedule, task: ^task, name: name} = 138 | normalize(Scheduler.new_job(), {schedule, task}) 139 | 140 | assert is_reference(name) 141 | end 142 | 143 | test "unnamed job as tuple with arguments" do 144 | schedule = ~e[* * * * *] 145 | task = {MyModule, :my_method, [1, 2, 3]} 146 | opts = [state: :inactive] 147 | 148 | job = {"* * * * *", task, opts} 149 | 150 | assert %{schedule: ^schedule, task: ^task, name: name, state: :inactive} = 151 | normalize(Scheduler.new_job(), job) 152 | 153 | assert is_reference(name) 154 | end 155 | 156 | test "named job as a keyword" do 157 | job = [ 158 | name: :newsletter, 159 | schedule: "@weekly", 160 | task: {MyModule, :my_method, [1, 2, 3]}, 161 | overlap: false 162 | ] 163 | 164 | expected_job = 165 | Scheduler.new_job() 166 | |> Job.set_name(:newsletter) 167 | |> Job.set_schedule(~e[@weekly]) 168 | |> Job.set_task({MyModule, :my_method, [1, 2, 3]}) 169 | |> Job.set_overlap(false) 170 | 171 | assert normalize(Scheduler.new_job(), job) == expected_job 172 | end 173 | 174 | test "unnamed job as a keyword" do 175 | schedule = ~e[@weekly] 176 | 177 | job = [schedule: "@weekly", task: {MyModule, :my_method, [1, 2, 3]}, overlap: false] 178 | 179 | assert %{ 180 | schedule: ^schedule, 181 | task: {MyModule, :my_method, [1, 2, 3]}, 182 | overlap: false, 183 | name: name 184 | } = normalize(Scheduler.new_job(), job) 185 | 186 | assert is_reference(name) 187 | end 188 | 189 | test "named job with unknown option" do 190 | job = [ 191 | name: :newsletter, 192 | schedule: "@weekly", 193 | task: {MyModule, :my_method, [1, 2, 3]}, 194 | unknown_option: true 195 | ] 196 | 197 | expected_job = 198 | Scheduler.new_job() 199 | |> Job.set_name(:newsletter) 200 | |> Job.set_schedule(~e[@weekly]) 201 | |> Job.set_task({MyModule, :my_method, [1, 2, 3]}) 202 | 203 | assert normalize(Scheduler.new_job(), job) == expected_job 204 | end 205 | 206 | test "unnamed job with unknown option" do 207 | schedule = ~e[@weekly] 208 | task = {MyModule, :my_method, [1, 2, 3]} 209 | 210 | job = [schedule: "@weekly", task: task, unknown_option: true] 211 | 212 | assert %{schedule: ^schedule, task: ^task, name: name} = normalize(Scheduler.new_job(), job) 213 | assert is_reference(name) 214 | end 215 | end 216 | -------------------------------------------------------------------------------- /lib/quantum/job.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.Job do 2 | @moduledoc """ 3 | This Struct defines a Job. 4 | 5 | ## Usage 6 | 7 | The struct should never be defined by hand. Use `c:Quantum.new_job/1` to create a new job and use the setters mentioned 8 | below to mutate the job. 9 | 10 | This is to ensure type safety. 11 | 12 | """ 13 | 14 | alias Crontab.CronExpression 15 | 16 | @enforce_keys [:name, :run_strategy, :overlap, :timezone] 17 | 18 | defstruct [ 19 | :run_strategy, 20 | :overlap, 21 | :timezone, 22 | :name, 23 | schedule: nil, 24 | task: nil, 25 | state: :active 26 | ] 27 | 28 | @type name :: atom | reference() 29 | @type state :: :active | :inactive 30 | @type task :: {atom, atom, [any]} | (-> any) 31 | @type timezone :: :utc | String.t() 32 | @type schedule :: Crontab.CronExpression.t() 33 | 34 | @type t :: %__MODULE__{ 35 | name: name, 36 | schedule: schedule | nil, 37 | task: task | nil, 38 | state: state, 39 | run_strategy: Quantum.RunStrategy.NodeList, 40 | overlap: boolean, 41 | timezone: timezone 42 | } 43 | 44 | # Takes some config from a scheduler and returns a new job 45 | # Do not use directly, use `Scheduler.new_job/1` instead. 46 | @doc false 47 | @spec new(config :: Keyword.t()) :: t 48 | def new(config) do 49 | Enum.reduce( 50 | [ 51 | {&set_name/2, Keyword.get(config, :name, make_ref())}, 52 | {&set_overlap/2, Keyword.fetch!(config, :overlap)}, 53 | {&set_run_strategy/2, Keyword.fetch!(config, :run_strategy)}, 54 | {&set_schedule/2, Keyword.get(config, :schedule)}, 55 | {&set_state/2, Keyword.fetch!(config, :state)}, 56 | {&set_task/2, Keyword.get(config, :task)}, 57 | {&set_timezone/2, Keyword.fetch!(config, :timezone)} 58 | ], 59 | %__MODULE__{name: nil, run_strategy: nil, overlap: nil, timezone: nil}, 60 | fn 61 | {_fun, nil}, acc -> acc 62 | {fun, value}, acc -> fun.(acc, value) 63 | end 64 | ) 65 | end 66 | 67 | @doc """ 68 | Sets a job's name. 69 | 70 | ### Parameters 71 | 72 | 1. `job` - The job struct to modify 73 | 2. `name` - The name to set 74 | 75 | ### Examples 76 | 77 | iex> Acme.Scheduler.new_job() 78 | ...> |> Quantum.Job.set_name(:name) 79 | ...> |> Map.get(:name) 80 | :name 81 | 82 | """ 83 | @spec set_name(t, atom) :: t 84 | def set_name(%__MODULE__{} = job, name) when is_atom(name), do: Map.put(job, :name, name) 85 | def set_name(%__MODULE__{} = job, name) when is_reference(name), do: Map.put(job, :name, name) 86 | 87 | @doc """ 88 | Sets a job's schedule. 89 | 90 | ### Parameters 91 | 92 | 1. `job` - The job struct to modify 93 | 2. `schedule` - The schedule to set. May only be of type `%Crontab.CronExpression{}` 94 | 95 | ### Examples 96 | 97 | iex> Acme.Scheduler.new_job() 98 | ...> |> Quantum.Job.set_schedule(Crontab.CronExpression.Parser.parse!("*/7")) 99 | ...> |> Map.get(:schedule) 100 | Crontab.CronExpression.Parser.parse!("*/7") 101 | 102 | """ 103 | @spec set_schedule(t, CronExpression.t()) :: t 104 | def set_schedule(%__MODULE__{} = job, %CronExpression{} = schedule), 105 | do: %{job | schedule: schedule} 106 | 107 | @doc """ 108 | Sets a job's task. 109 | 110 | ### Parameters 111 | 112 | 1. `job` - The job struct to modify 113 | 2. `task` - The function to be performed, ex: `{Heartbeat, :send, []}` or `fn -> :something end` 114 | 115 | ### Examples 116 | 117 | iex> Acme.Scheduler.new_job() 118 | ...> |> Quantum.Job.set_task({Backup, :backup, []}) 119 | ...> |> Map.get(:task) 120 | {Backup, :backup, []} 121 | 122 | """ 123 | @spec set_task(t, task) :: t 124 | def set_task(%__MODULE__{} = job, {module, function, args} = task) 125 | when is_atom(module) and is_atom(function) and is_list(args), 126 | do: Map.put(job, :task, task) 127 | 128 | def set_task(%__MODULE__{} = job, task) when is_function(task, 0), do: Map.put(job, :task, task) 129 | 130 | @doc """ 131 | Sets a job's state. 132 | 133 | ### Parameters 134 | 135 | 1. `job` - The job struct to modify 136 | 2. `state` - The state to set 137 | 138 | ### Examples 139 | 140 | iex> Acme.Scheduler.new_job() 141 | ...> |> Quantum.Job.set_state(:active) 142 | ...> |> Map.get(:state) 143 | :active 144 | 145 | """ 146 | @spec set_state(t, state) :: t 147 | def set_state(%__MODULE__{} = job, :active), do: Map.put(job, :state, :active) 148 | def set_state(%__MODULE__{} = job, :inactive), do: Map.put(job, :state, :inactive) 149 | 150 | @doc """ 151 | Sets a job's run strategy. 152 | 153 | ### Parameters 154 | 155 | 1. `job` - The job struct to modify 156 | 2. `run_strategy` - The run strategy to set 157 | 158 | ### Examples 159 | 160 | iex> Acme.Scheduler.new_job() 161 | ...> |> Quantum.Job.run_strategy(%Quantum.RunStrategy.All{nodes: [:one, :two]}) 162 | ...> |> Map.get(:run_strategy) 163 | [:one, :two] 164 | 165 | """ 166 | @spec set_run_strategy(t, Quantum.RunStrategy.NodeList) :: t 167 | def set_run_strategy(%__MODULE__{} = job, run_strategy), 168 | do: Map.put(job, :run_strategy, run_strategy) 169 | 170 | @doc """ 171 | Sets a job's overlap. 172 | 173 | ### Parameters 174 | 175 | 1. `job` - The job struct to modify 176 | 2. `overlap` - Enable / Disable Overlap 177 | 178 | ### Examples 179 | 180 | iex> Acme.Scheduler.new_job() 181 | ...> |> Quantum.Job.set_overlap(false) 182 | ...> |> Map.get(:overlap) 183 | false 184 | 185 | """ 186 | @spec set_overlap(t, boolean) :: t 187 | def set_overlap(%__MODULE__{} = job, overlap?) when is_boolean(overlap?), 188 | do: Map.put(job, :overlap, overlap?) 189 | 190 | @doc """ 191 | Sets a job's timezone. 192 | 193 | ### Parameters 194 | 195 | 1. `job` - The job struct to modify 196 | 2. `timezone` - The timezone to set. 197 | 198 | ### Examples 199 | 200 | iex> Acme.Scheduler.new_job() 201 | ...> |> Quantum.Job.set_timezone("Europe/Zurich") 202 | ...> |> Map.get(:timezone) 203 | "Europe/Zurich" 204 | 205 | """ 206 | @spec set_timezone(t, String.t() | :utc) :: t 207 | def set_timezone(%__MODULE__{} = job, timezone), do: Map.put(job, :timezone, timezone) 208 | end 209 | -------------------------------------------------------------------------------- /test/quantum/scheduler_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.SchedulerTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | alias Quantum.Job 7 | alias Quantum.RunStrategy.Random 8 | 9 | import ExUnit.CaptureLog 10 | 11 | import Crontab.CronExpression 12 | 13 | defmodule Scheduler do 14 | @moduledoc false 15 | 16 | use Quantum, otp_app: :scheduler_test 17 | end 18 | 19 | @defaults %{ 20 | schedule: "*/7", 21 | overlap: false, 22 | timezone: "Europe/Zurich" 23 | } 24 | 25 | defmodule DefaultConfigScheduler do 26 | @moduledoc false 27 | 28 | use Quantum, otp_app: :scheduler_test 29 | end 30 | 31 | defmodule ZeroTimeoutScheduler do 32 | @moduledoc false 33 | 34 | use Quantum, otp_app: :scheduler_test 35 | end 36 | 37 | setup_all do 38 | Application.put_env(:scheduler_test, Scheduler, jobs: []) 39 | 40 | Application.put_env( 41 | :scheduler_test, 42 | DefaultConfigScheduler, 43 | jobs: [], 44 | schedule: @defaults.schedule, 45 | overlap: @defaults.overlap, 46 | timezone: @defaults.timezone 47 | ) 48 | 49 | Application.put_env(:scheduler_test, ZeroTimeoutScheduler, timeout: 0, jobs: []) 50 | end 51 | 52 | setup context do 53 | schedulers = Map.get(context, :schedulers, []) 54 | 55 | for scheduler <- schedulers do 56 | {:ok, _pid} = start_supervised(scheduler) 57 | end 58 | 59 | :ok 60 | end 61 | 62 | describe "new_job/0" do 63 | test "returns Quantum.Job struct" do 64 | %Job{schedule: schedule, overlap: overlap, timezone: timezone} = Scheduler.new_job() 65 | 66 | assert schedule == nil 67 | assert overlap == true 68 | assert timezone == :utc 69 | end 70 | 71 | test "has defaults set" do 72 | %Job{schedule: schedule, overlap: overlap, timezone: timezone} = 73 | DefaultConfigScheduler.new_job() 74 | 75 | assert schedule == ~e[#{@defaults.schedule}] 76 | assert overlap == @defaults.overlap 77 | assert timezone == @defaults.timezone 78 | end 79 | end 80 | 81 | describe "add_job/2" do 82 | @tag schedulers: [Scheduler] 83 | test "adding a job at run time" do 84 | spec = ~e[1 * * * *] 85 | fun = fn -> :ok end 86 | 87 | capture_log(fn -> 88 | :ok = Scheduler.add_job({spec, fun}) 89 | 90 | assert Enum.any?(Scheduler.jobs(), fn {_, %Job{schedule: schedule, task: task}} -> 91 | schedule == spec && task == fun 92 | end) 93 | end) 94 | end 95 | 96 | @tag schedulers: [Scheduler] 97 | test "adding a named job struct at run time" do 98 | spec = ~e[1 * * * *] 99 | fun = fn -> :ok end 100 | 101 | job = 102 | Scheduler.new_job() 103 | |> Job.set_name(:test_job) 104 | |> Job.set_schedule(spec) 105 | |> Job.set_task(fun) 106 | 107 | capture_log(fn -> 108 | :ok = Scheduler.add_job(job) 109 | 110 | assert Enum.member?(Scheduler.jobs(), { 111 | :test_job, 112 | %{job | run_strategy: %Random{nodes: :cluster}} 113 | }) 114 | end) 115 | end 116 | 117 | @tag schedulers: [Scheduler] 118 | test "adding a named {m, f, a} job at run time" do 119 | spec = ~e[1 * * * *] 120 | task = {IO, :puts, ["Tick"]} 121 | 122 | job = 123 | Scheduler.new_job() 124 | |> Job.set_name(:ticker) 125 | |> Job.set_schedule(spec) 126 | |> Job.set_task(task) 127 | 128 | capture_log(fn -> 129 | :ok = Scheduler.add_job(job) 130 | 131 | assert Enum.member?(Scheduler.jobs(), { 132 | :ticker, 133 | %{job | run_strategy: %Random{nodes: :cluster}} 134 | }) 135 | end) 136 | end 137 | 138 | @tag schedulers: [Scheduler] 139 | test "adding a unnamed job at run time" do 140 | spec = ~e[1 * * * *] 141 | fun = fn -> :ok end 142 | 143 | job = 144 | Scheduler.new_job() 145 | |> Job.set_schedule(spec) 146 | |> Job.set_task(fun) 147 | 148 | capture_log(fn -> 149 | :ok = Scheduler.add_job(job) 150 | assert Enum.member?(Scheduler.jobs(), {job.name, job}) 151 | end) 152 | end 153 | end 154 | 155 | @tag schedulers: [Scheduler] 156 | test "finding a named job" do 157 | spec = ~e[* * * * *] 158 | fun = fn -> :ok end 159 | 160 | job = 161 | Scheduler.new_job() 162 | |> Job.set_name(:test_job) 163 | |> Job.set_schedule(spec) 164 | |> Job.set_task(fun) 165 | 166 | capture_log(fn -> 167 | :ok = Scheduler.add_job(job) 168 | fjob = Scheduler.find_job(:test_job) 169 | assert fjob.name == :test_job 170 | assert fjob.schedule == spec 171 | assert fjob.run_strategy == %Random{nodes: :cluster} 172 | end) 173 | end 174 | 175 | @tag schedulers: [Scheduler] 176 | test "deactivating a named job" do 177 | spec = ~e[* * * * *] 178 | fun = fn -> :ok end 179 | 180 | job = 181 | Scheduler.new_job() 182 | |> Job.set_name(:test_job) 183 | |> Job.set_schedule(spec) 184 | |> Job.set_task(fun) 185 | 186 | capture_log(fn -> 187 | :ok = Scheduler.add_job(job) 188 | :ok = Scheduler.deactivate_job(:test_job) 189 | sjob = Scheduler.find_job(:test_job) 190 | assert sjob == %{job | state: :inactive} 191 | end) 192 | end 193 | 194 | @tag schedulers: [Scheduler] 195 | test "activating a named job" do 196 | spec = ~e[* * * * *] 197 | fun = fn -> :ok end 198 | 199 | job = 200 | Scheduler.new_job() 201 | |> Job.set_name(:test_job) 202 | |> Job.set_state(:inactive) 203 | |> Job.set_schedule(spec) 204 | |> Job.set_task(fun) 205 | 206 | capture_log(fn -> 207 | :ok = Scheduler.add_job(job) 208 | :ok = Scheduler.activate_job(:test_job) 209 | ajob = Scheduler.find_job(:test_job) 210 | assert ajob == %{job | state: :active} 211 | end) 212 | end 213 | 214 | @tag schedulers: [Scheduler] 215 | test "deleting a named job at run time" do 216 | spec = ~e[* * * * *] 217 | fun = fn -> :ok end 218 | 219 | job = 220 | Scheduler.new_job() 221 | |> Job.set_name(:test_job) 222 | |> Job.set_schedule(spec) 223 | |> Job.set_task(fun) 224 | 225 | capture_log(fn -> 226 | :ok = Scheduler.add_job(job) 227 | :ok = Scheduler.delete_job(:test_job) 228 | assert !Enum.member?(Scheduler.jobs(), {:test_job, job}) 229 | end) 230 | end 231 | 232 | @tag schedulers: [Scheduler] 233 | test "deleting all jobs" do 234 | capture_log(fn -> 235 | for i <- 1..3 do 236 | name = String.to_atom("test_job_" <> Integer.to_string(i)) 237 | spec = ~e[* * * * *] 238 | fun = fn -> :ok end 239 | 240 | job = 241 | Scheduler.new_job() 242 | |> Job.set_name(name) 243 | |> Job.set_schedule(spec) 244 | |> Job.set_task(fun) 245 | 246 | :ok = Scheduler.add_job(job) 247 | end 248 | 249 | assert Enum.count(Scheduler.jobs()) == 3 250 | Scheduler.delete_all_jobs() 251 | assert Scheduler.jobs() == [] 252 | end) 253 | end 254 | 255 | @tag schedulers: [ZeroTimeoutScheduler] 256 | test "timeout can be configured for genserver correctly" do 257 | job = 258 | ZeroTimeoutScheduler.new_job() 259 | |> Job.set_name(:tmpjob) 260 | |> Job.set_schedule(~e[* */5 * * *]) 261 | |> Job.set_task(fn -> :ok end) 262 | 263 | capture_log(fn -> 264 | ZeroTimeoutScheduler.add_job(job) 265 | 266 | assert {:timeout, _} = catch_exit(ZeroTimeoutScheduler.find_job(:tmpjob)) 267 | 268 | # Ensure that log message is contained 269 | Process.sleep(100) 270 | end) 271 | end 272 | end 273 | -------------------------------------------------------------------------------- /.github/workflows/part_test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: {} 3 | 4 | name: "Test" 5 | 6 | env: 7 | BUILD_EMBEDDED: true 8 | 9 | jobs: 10 | detectToolVersions: 11 | name: "Detect Tool Versions" 12 | 13 | runs-on: ubuntu-latest 14 | 15 | outputs: 16 | otpVersion: "${{ steps.toolVersions.outputs.OTP_VERSION }}" 17 | elixirVersion: "${{ steps.toolVersions.outputs.ELIXIR_VERSION }}" 18 | 19 | steps: 20 | - name: Harden Runner 21 | uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0 22 | with: 23 | egress-policy: audit 24 | 25 | - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 26 | - name: "Read .tool-versions" 27 | id: toolVersions 28 | run: | 29 | OTP_VERSION="$(cat .tool-versions | grep erlang | cut -d' ' -f2-)" 30 | echo OTP: $OTP_VERSION 31 | echo "OTP_VERSION=${OTP_VERSION}" >> $GITHUB_OUTPUT 32 | 33 | ELIXIR_VERSION="$(cat .tool-versions | grep elixir | cut -d' ' -f2-)" 34 | echo Rebar: $ELIXIR_VERSION 35 | echo "ELIXIR_VERSION=${ELIXIR_VERSION}" >> $GITHUB_OUTPUT 36 | 37 | format: 38 | name: Check Formatting 39 | 40 | runs-on: ubuntu-latest 41 | 42 | env: 43 | MIX_ENV: dev 44 | 45 | steps: 46 | - uses: actions/checkout@v6 47 | - uses: erlef/setup-elixir@v1 48 | id: setupBEAM 49 | with: 50 | version-file: '.tool-versions' 51 | version-type: strict 52 | - uses: actions/cache@v5 53 | with: 54 | path: deps 55 | key: deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 56 | restore-keys: | 57 | deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.elixir-version }}- 58 | - run: mix deps.get 59 | - uses: actions/cache@v5 60 | with: 61 | path: _build/test 62 | key: compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 63 | restore-keys: | 64 | compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.elixir-version }}- 65 | - run: mix deps.compile 66 | - run: mix format --check-formatted 67 | 68 | test: 69 | name: Run Tests & Submit Coverage (${{ matrix.name }}) 70 | 71 | needs: ["detectToolVersions"] 72 | 73 | runs-on: ${{ matrix.runs-on }} 74 | 75 | strategy: 76 | fail-fast: false 77 | matrix: 78 | include: 79 | # Lowest Supported 80 | - otp: "24.2" 81 | elixir: "1.15" 82 | runs-on: ubuntu-22.04 83 | name: "lowest" 84 | # Latest Supported 85 | - otp: "${{ needs.detectToolVersions.outputs.otpVersion }}" 86 | elixir: "${{ needs.detectToolVersions.outputs.elixirVersion }}" 87 | runs-on: ubuntu-24.04 88 | name: "latest" 89 | enable_coverage_export: "true" 90 | # Test Main 91 | - otp: "${{ needs.detectToolVersions.outputs.otpVersion }}" 92 | elixir: "main" 93 | runs-on: ubuntu-24.04 94 | name: "main" 95 | 96 | env: 97 | MIX_ENV: test 98 | 99 | steps: 100 | - uses: actions/checkout@v6 101 | - uses: erlef/setup-elixir@v1 102 | id: setupBEAM 103 | with: 104 | otp-version: ${{ matrix.otp }} 105 | elixir-version: ${{ matrix.elixir }} 106 | - uses: actions/cache@v5 107 | with: 108 | path: deps 109 | key: deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 110 | restore-keys: | 111 | deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 112 | - run: mix deps.get 113 | - uses: actions/cache@v5 114 | with: 115 | path: _build/test 116 | key: compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 117 | restore-keys: | 118 | compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 119 | - run: mix deps.compile 120 | - run: mix compile --warning-as-errors 121 | - run: mix coveralls.github 122 | if: ${{ matrix.enable_coverage_export == 'true' }} 123 | env: 124 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 125 | - run: mix test 126 | if: ${{ !matrix.enable_coverage_export }} 127 | env: 128 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 129 | 130 | credo: 131 | name: Check Credo 132 | 133 | runs-on: ubuntu-latest 134 | 135 | env: 136 | MIX_ENV: dev 137 | 138 | steps: 139 | - uses: actions/checkout@v6 140 | - uses: erlef/setup-elixir@v1 141 | id: setupBEAM 142 | with: 143 | version-file: '.tool-versions' 144 | version-type: strict 145 | - uses: actions/cache@v5 146 | with: 147 | path: deps 148 | key: deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 149 | restore-keys: | 150 | deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 151 | - run: mix deps.get 152 | - uses: actions/cache@v5 153 | with: 154 | path: _build/dev 155 | key: compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 156 | restore-keys: | 157 | compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 158 | - run: mix deps.compile 159 | - run: mix compile --warning-as-errors 160 | - run: mix credo --strict 161 | 162 | dialyzer_plt: 163 | name: Generate Dialyzer PLT 164 | 165 | runs-on: ubuntu-latest 166 | 167 | env: 168 | MIX_ENV: dev 169 | DIALYZER_PLT_PRIV: true 170 | 171 | steps: 172 | - uses: actions/checkout@v6 173 | - uses: erlef/setup-elixir@v1 174 | id: setupBEAM 175 | with: 176 | version-file: '.tool-versions' 177 | version-type: strict 178 | - uses: actions/cache@v5 179 | with: 180 | path: deps 181 | key: deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 182 | restore-keys: | 183 | deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 184 | - run: mix deps.get 185 | - uses: actions/cache@v5 186 | with: 187 | path: _build/dev 188 | key: compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 189 | restore-keys: | 190 | compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 191 | - run: mix deps.compile 192 | - run: mix compile --warning-as-errors 193 | - uses: actions/cache@v5 194 | with: 195 | path: priv/plts/ 196 | key: dialyzer_plt_dev-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 197 | restore-keys: | 198 | dialyzer_plt_dev-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 199 | - run: mix dialyzer --plt 200 | - uses: actions/upload-artifact@v6 201 | with: 202 | name: dialyzer_plt_dev 203 | path: priv/plts/ 204 | 205 | dialyzer_test: 206 | name: "Check Dialyzer" 207 | 208 | runs-on: ubuntu-latest 209 | 210 | needs: ['dialyzer_plt'] 211 | 212 | env: 213 | MIX_ENV: dev 214 | DIALYZER_PLT_PRIV: true 215 | 216 | steps: 217 | - uses: actions/checkout@v6 218 | - uses: erlef/setup-elixir@v1 219 | id: setupBEAM 220 | with: 221 | version-file: '.tool-versions' 222 | version-type: strict 223 | - uses: actions/cache@v5 224 | with: 225 | path: deps 226 | key: deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 227 | restore-keys: | 228 | deps-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 229 | - run: mix deps.get 230 | - uses: actions/cache@v5 231 | with: 232 | path: _build/dev 233 | key: compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}-${{ hashFiles('mix.lock') }} 234 | restore-keys: | 235 | compile-${{ env.MIX_ENV }}-${{ runner.os }}-${{ steps.setupBEAM.outputs.otp-version }}-${{ steps.setupBEAM.outputs.elixir-version }}- 236 | - run: mix deps.compile 237 | - run: mix compile --warning-as-errors 238 | - uses: actions/download-artifact@v7 239 | with: 240 | name: dialyzer_plt_dev 241 | path: priv/plts/ 242 | - run: mix dialyzer 243 | -------------------------------------------------------------------------------- /lib/quantum/execution_broadcaster.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutionBroadcaster do 2 | @moduledoc false 3 | 4 | # Receives Added / Removed Jobs, Broadcasts Executions of Jobs 5 | 6 | use GenStage 7 | 8 | require Logger 9 | 10 | alias Crontab.CronExpression 11 | alias Crontab.Scheduler, as: CrontabScheduler 12 | 13 | alias Quantum.ClockBroadcaster.Event, as: ClockEvent 14 | 15 | alias Quantum.ExecutionBroadcaster.Event, as: ExecuteEvent 16 | alias Quantum.ExecutionBroadcaster.InitOpts 17 | alias Quantum.ExecutionBroadcaster.State 18 | alias Quantum.Job 19 | 20 | alias __MODULE__.{InitOpts, StartOpts, State} 21 | 22 | @type event :: {:add, Job.t()} | {:execute, Job.t()} 23 | 24 | defmodule JobInPastError do 25 | @moduledoc false 26 | defexception message: 27 | "The job was scheduled in the past. This must not happen to prevent infinite loops!" 28 | end 29 | 30 | # Start Stage 31 | @spec start_link(StartOpts.t()) :: GenServer.on_start() 32 | def start_link(%StartOpts{name: name} = opts) do 33 | __MODULE__ 34 | |> GenStage.start_link( 35 | struct!( 36 | InitOpts, 37 | Map.take(opts, [ 38 | :job_broadcaster_reference, 39 | :clock_broadcaster_reference, 40 | :storage, 41 | :scheduler, 42 | :debug_logging 43 | ]) 44 | ), 45 | name: name 46 | ) 47 | |> case do 48 | {:ok, pid} -> 49 | {:ok, pid} 50 | 51 | {:error, {:already_started, pid}} -> 52 | Process.monitor(pid) 53 | {:ok, pid} 54 | 55 | {:error, _reason} = error -> 56 | error 57 | end 58 | end 59 | 60 | @impl GenStage 61 | def init(%InitOpts{ 62 | job_broadcaster_reference: job_broadcaster, 63 | clock_broadcaster_reference: clock_broadcaster, 64 | storage: storage, 65 | scheduler: scheduler, 66 | debug_logging: debug_logging 67 | }) do 68 | storage_pid = GenServer.whereis(Module.concat(scheduler, Storage)) 69 | 70 | {:producer_consumer, 71 | %State{ 72 | uninitialized_jobs: [], 73 | execution_timeline: [], 74 | storage: storage, 75 | storage_pid: storage_pid, 76 | scheduler: scheduler, 77 | debug_logging: debug_logging 78 | }, subscribe_to: [job_broadcaster, clock_broadcaster]} 79 | end 80 | 81 | @impl GenStage 82 | def handle_events(events, _, state) do 83 | {events, state} = 84 | Enum.reduce(events, {[], state}, fn event, {list, state} -> 85 | {new_events, state} = handle_event(event, state) 86 | {list ++ new_events, state} 87 | end) 88 | 89 | {:noreply, events, state} 90 | end 91 | 92 | def handle_event( 93 | {:add, %Job{schedule: %CronExpression{reboot: true}, name: name} = job}, 94 | %State{uninitialized_jobs: uninitialized_jobs, debug_logging: debug_logging} = state 95 | ) do 96 | debug_logging && 97 | Logger.debug(fn -> 98 | {"Scheduling job for single reboot execution", node: Node.self(), name: name} 99 | end) 100 | 101 | {[%ExecuteEvent{job: job}], %{state | uninitialized_jobs: [job | uninitialized_jobs]}} 102 | end 103 | 104 | def handle_event( 105 | {:add, %Job{name: name} = job}, 106 | %State{uninitialized_jobs: uninitialized_jobs, debug_logging: debug_logging} = state 107 | ) do 108 | debug_logging && 109 | Logger.debug(fn -> 110 | {"Adding job", node: Node.self(), name: name} 111 | end) 112 | 113 | {[], %{state | uninitialized_jobs: [job | uninitialized_jobs]}} 114 | end 115 | 116 | def handle_event( 117 | {:run, %Job{name: name} = job}, 118 | %State{debug_logging: debug_logging} = state 119 | ) do 120 | debug_logging && 121 | Logger.debug(fn -> 122 | {"Running job once", node: Node.self(), name: name} 123 | end) 124 | 125 | {[%ExecuteEvent{job: job}], state} 126 | end 127 | 128 | def handle_event( 129 | {:remove, name}, 130 | %State{ 131 | uninitialized_jobs: uninitialized_jobs, 132 | execution_timeline: execution_timeline, 133 | debug_logging: debug_logging 134 | } = state 135 | ) do 136 | debug_logging && 137 | Logger.debug(fn -> 138 | {"Removing job", node: Node.self(), name: name} 139 | end) 140 | 141 | uninitialized_jobs = Enum.reject(uninitialized_jobs, &(&1.name == name)) 142 | 143 | execution_timeline = 144 | execution_timeline 145 | |> Enum.map(fn {date, job_list} -> 146 | {date, Enum.reject(job_list, &match?(%Job{name: ^name}, &1))} 147 | end) 148 | |> Enum.reject(fn 149 | {_, []} -> true 150 | {_, _} -> false 151 | end) 152 | 153 | {[], 154 | %{state | uninitialized_jobs: uninitialized_jobs, execution_timeline: execution_timeline}} 155 | end 156 | 157 | def handle_event( 158 | %ClockEvent{time: time}, 159 | state 160 | ) do 161 | state 162 | |> initialize_jobs(time) 163 | |> execute_events_to_fire(time) 164 | end 165 | 166 | @impl GenStage 167 | def handle_info(_message, state) do 168 | {:noreply, [], state} 169 | end 170 | 171 | defp initialize_jobs(%State{uninitialized_jobs: uninitialized_jobs} = state, time) do 172 | uninitialized_jobs 173 | |> Enum.reject(&match?(%Job{schedule: %CronExpression{reboot: true}}, &1)) 174 | |> Enum.reduce( 175 | %{ 176 | state 177 | | uninitialized_jobs: 178 | Enum.filter( 179 | uninitialized_jobs, 180 | &match?(%Job{schedule: %CronExpression{reboot: true}}, &1) 181 | ) 182 | }, 183 | &add_job_to_state(&1, &2, time) 184 | ) 185 | |> sort_state 186 | end 187 | 188 | defp execute_events_to_fire(%State{execution_timeline: []} = state, _time), do: {[], state} 189 | 190 | defp execute_events_to_fire( 191 | %State{ 192 | storage: storage, 193 | storage_pid: storage_pid, 194 | debug_logging: debug_logging, 195 | execution_timeline: [{time_to_execute, jobs} | tail] 196 | } = state, 197 | time 198 | ) do 199 | case DateTime.compare(time, time_to_execute) do 200 | :gt -> 201 | raise "Jobs were skipped" 202 | 203 | :lt -> 204 | {[], state} 205 | 206 | :eq -> 207 | :ok = storage.update_last_execution_date(storage_pid, time_to_execute) 208 | 209 | events = 210 | for %Job{name: job_name} = job <- jobs do 211 | debug_logging && 212 | Logger.debug(fn -> 213 | {"Scheduling job for execution", node: Node.self(), name: job_name} 214 | end) 215 | 216 | %ExecuteEvent{job: job} 217 | end 218 | 219 | {next_events, new_state} = 220 | jobs 221 | |> Enum.reduce( 222 | %{state | execution_timeline: tail}, 223 | &add_job_to_state(&1, &2, DateTime.add(time, 1, :second)) 224 | ) 225 | |> sort_state 226 | |> execute_events_to_fire(time) 227 | 228 | {events ++ next_events, new_state} 229 | end 230 | end 231 | 232 | defp add_job_to_state( 233 | %Job{schedule: schedule, timezone: timezone, name: name} = job, 234 | state, 235 | time 236 | ) do 237 | with {:ok, execution_date} <- get_next_execution_time(job, time) do 238 | add_to_state(state, time, execution_date, job) 239 | else 240 | {:error, :time_zone_not_found} -> 241 | Logger.error( 242 | "Invalid Timezone #{inspect(timezone)} provided for job #{inspect(name)}.", 243 | job: job, 244 | error: :time_zone_not_found 245 | ) 246 | 247 | state 248 | 249 | {:error, _} -> 250 | Logger.warning(fn -> 251 | """ 252 | Invalid Schedule #{inspect(schedule)} provided for job #{inspect(name)}. 253 | No matching dates found. The job was removed. 254 | """ 255 | end) 256 | 257 | state 258 | end 259 | end 260 | 261 | defp get_next_execution_time( 262 | %Job{schedule: schedule, timezone: :utc}, 263 | time 264 | ) do 265 | CrontabScheduler.get_next_run_date(schedule, time) 266 | end 267 | 268 | defp get_next_execution_time( 269 | %Job{schedule: schedule, timezone: timezone}, 270 | time 271 | ) do 272 | with {:ok, localized_time} <- DateTime.shift_zone(time, timezone), 273 | {:ok, localized_execution_time} <- 274 | CrontabScheduler.get_next_run_date(schedule, localized_time) do 275 | DateTime.shift_zone(localized_execution_time, "Etc/UTC") 276 | end 277 | end 278 | 279 | defp sort_state(%State{execution_timeline: execution_timeline} = state) do 280 | %{state | execution_timeline: Enum.sort_by(execution_timeline, &elem(&1, 0), DateTime)} 281 | end 282 | 283 | defp add_to_state(%State{execution_timeline: execution_timeline} = state, time, date, job) do 284 | unless DateTime.compare(time, date) in [:lt, :eq] do 285 | raise Quantum.ExecutionBroadcaster.JobInPastError 286 | end 287 | 288 | %{state | execution_timeline: add_job_at_date(execution_timeline, date, job)} 289 | end 290 | 291 | defp add_job_at_date(execution_timeline, date, job) do 292 | case find_date_and_put_job(execution_timeline, date, job) do 293 | {:found, list} -> list 294 | {:not_found, list} -> [{date, [job]} | list] 295 | end 296 | end 297 | 298 | defp find_date_and_put_job([{date, jobs} | rest], date, job) do 299 | {:found, [{date, [job | jobs]} | rest]} 300 | end 301 | 302 | defp find_date_and_put_job([], _, _) do 303 | {:not_found, []} 304 | end 305 | 306 | defp find_date_and_put_job([head | rest], date, job) do 307 | {state, new_rest} = find_date_and_put_job(rest, date, job) 308 | {state, [head | new_rest]} 309 | end 310 | end 311 | -------------------------------------------------------------------------------- /test/quantum/execution_broadcaster_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutionBroadcasterTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Crontab.CronExpression 7 | import ExUnit.CaptureLog 8 | import Quantum.CaptureLogExtend 9 | 10 | alias Quantum.ClockBroadcaster.Event, as: ClockEvent 11 | alias Quantum.ExecutionBroadcaster 12 | alias Quantum.ExecutionBroadcaster.Event, as: ExecuteEvent 13 | alias Quantum.ExecutionBroadcaster.StartOpts 14 | alias Quantum.Job 15 | alias Quantum.Storage.Test, as: TestStorage 16 | alias Quantum.{TestConsumer, TestProducer} 17 | 18 | # Allow max 10% Latency 19 | @max_timeout 1_100 20 | 21 | doctest ExecutionBroadcaster 22 | 23 | defmodule TestScheduler do 24 | @moduledoc false 25 | 26 | use Quantum, otp_app: :execution_broadcaster_test 27 | end 28 | 29 | setup tags do 30 | if tags[:listen_storage] do 31 | Process.put(:test_pid, self()) 32 | end 33 | 34 | if tags[:manual_dispatch] do 35 | :ok 36 | else 37 | producer = start_supervised!({TestProducer, []}) 38 | 39 | {broadcaster, _} = 40 | capture_log_with_return(fn -> 41 | start_supervised!( 42 | {ExecutionBroadcaster, 43 | %StartOpts{ 44 | name: __MODULE__, 45 | job_broadcaster_reference: producer, 46 | clock_broadcaster_reference: producer, 47 | storage: TestStorage, 48 | scheduler: TestScheduler, 49 | debug_logging: true 50 | }} 51 | ) 52 | end) 53 | 54 | start_supervised!({TestConsumer, [broadcaster, self()]}) 55 | 56 | {:ok, %{producer: producer, broadcaster: broadcaster, debug_logging: true}} 57 | end 58 | end 59 | 60 | describe "add" do 61 | test "reboot triggers", %{producer: producer} do 62 | reboot_job = 63 | TestScheduler.new_job() 64 | |> Job.set_schedule(~e[@reboot]) 65 | 66 | # Some schedule that is valid but will not trigger the next 10 years 67 | non_reboot_job = 68 | TestScheduler.new_job() 69 | |> Job.set_schedule(~e[* * * * * #{DateTime.utc_now().year + 1}]) 70 | 71 | capture_log(fn -> 72 | TestProducer.send(producer, {:add, reboot_job}) 73 | TestProducer.send(producer, {:add, non_reboot_job}) 74 | 75 | assert_receive {:received, %ExecuteEvent{job: ^reboot_job}}, @max_timeout 76 | refute_receive {:received, %ExecuteEvent{job: ^non_reboot_job}}, @max_timeout 77 | end) 78 | end 79 | 80 | test "run_job triggers job to run once", %{producer: producer} do 81 | job = TestScheduler.new_job() 82 | 83 | TestProducer.send(producer, {:run, job}) 84 | 85 | assert_receive {:received, %ExecuteEvent{job: ^job}} 86 | end 87 | 88 | test "normal schedule triggers once per second", %{producer: producer} do 89 | job = 90 | TestScheduler.new_job() 91 | |> Job.set_schedule(~e[*]e) 92 | 93 | capture_log(fn -> 94 | TestProducer.send(producer, {:add, job}) 95 | 96 | spawn(fn -> 97 | now = DateTime.truncate(DateTime.utc_now(), :second) 98 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 99 | 100 | Process.sleep(1_000) 101 | 102 | TestProducer.send(producer, %ClockEvent{ 103 | time: DateTime.add(now, 1, :second), 104 | catch_up: false 105 | }) 106 | end) 107 | 108 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 109 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 110 | end) 111 | end 112 | 113 | @tag listen_storage: true 114 | test "saves new last execution time in storage", %{producer: producer} do 115 | job = 116 | TestScheduler.new_job() 117 | |> Job.set_schedule(~e[*]e) 118 | 119 | capture_log(fn -> 120 | TestProducer.send(producer, {:add, job}) 121 | now = DateTime.truncate(DateTime.utc_now(), :second) 122 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 123 | 124 | assert_receive {:update_last_execution_date, %DateTime{}, _}, @max_timeout 125 | 126 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 127 | end) 128 | end 129 | 130 | test "normal schedule in other timezone triggers once per second", %{producer: producer} do 131 | job = 132 | TestScheduler.new_job() 133 | |> Job.set_schedule(~e[*]e) 134 | |> Job.set_timezone("Europe/Zurich") 135 | 136 | capture_log(fn -> 137 | TestProducer.send(producer, {:add, job}) 138 | 139 | spawn(fn -> 140 | now = DateTime.truncate(DateTime.utc_now(), :second) 141 | add1 = DateTime.add(now, 1, :second) 142 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 143 | Process.sleep(1_000) 144 | TestProducer.send(producer, %ClockEvent{time: add1, catch_up: false}) 145 | end) 146 | 147 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 148 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 149 | end) 150 | end 151 | 152 | test "impossible schedule will not create a crash", %{producer: producer} do 153 | # Some schedule that will never trigger 154 | job = 155 | TestScheduler.new_job() 156 | |> Job.set_schedule(~e[1 1 1 1 1 2000]) 157 | 158 | assert capture_log(fn -> 159 | TestProducer.send(producer, {:add, job}) 160 | 161 | now = DateTime.truncate(DateTime.utc_now(), :second) 162 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 163 | 164 | refute_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 165 | end) =~ """ 166 | Invalid Schedule #{inspect(job.schedule)} provided for job #{inspect(job.name)}. 167 | No matching dates found. The job was removed. 168 | """ 169 | end 170 | 171 | test "invalid timezone will not create a crash", %{producer: producer} do 172 | job = 173 | TestScheduler.new_job() 174 | |> Job.set_schedule(~e[*]e) 175 | |> Job.set_timezone("Foobar") 176 | 177 | assert capture_log(fn -> 178 | TestProducer.send(producer, {:add, job}) 179 | 180 | now = DateTime.truncate(DateTime.utc_now(), :second) 181 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 182 | 183 | refute_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 184 | end) =~ 185 | "Invalid Timezone #{inspect(job.timezone)} provided for job #{inspect(job.name)}." 186 | end 187 | 188 | test "will continue to send after new job is added", %{producer: producer} do 189 | job = 190 | TestScheduler.new_job() 191 | |> Job.set_schedule(~e[*]e) 192 | 193 | job_new = 194 | TestScheduler.new_job() 195 | |> Job.set_schedule(~e[*]) 196 | 197 | capture_log(fn -> 198 | TestProducer.send(producer, {:add, job}) 199 | 200 | now = DateTime.truncate(DateTime.utc_now(), :second) 201 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 202 | 203 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 204 | 205 | TestProducer.send(producer, {:add, job_new}) 206 | 207 | TestProducer.send(producer, %ClockEvent{ 208 | time: DateTime.add(now, 1, :second), 209 | catch_up: false 210 | }) 211 | 212 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 213 | end) 214 | end 215 | 216 | test "will recalculate execution timer when a new job is added", %{producer: producer} do 217 | job = 218 | TestScheduler.new_job() 219 | |> Job.set_schedule(~e[1 1 1 1 1]) 220 | 221 | job_new = 222 | TestScheduler.new_job() 223 | |> Job.set_schedule(~e[*]e) 224 | 225 | capture_log(fn -> 226 | TestProducer.send(producer, {:add, job}) 227 | TestProducer.send(producer, {:add, job_new}) 228 | 229 | now = DateTime.truncate(DateTime.utc_now(), :second) 230 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 231 | 232 | assert_receive {:received, %ExecuteEvent{job: ^job_new}}, @max_timeout 233 | end) 234 | end 235 | end 236 | 237 | describe "remove" do 238 | test "stops triggering after remove", %{producer: producer} do 239 | job = 240 | TestScheduler.new_job() 241 | |> Job.set_schedule(~e[*]e) 242 | 243 | capture_log(fn -> 244 | TestProducer.send(producer, {:add, job}) 245 | now = DateTime.truncate(DateTime.utc_now(), :second) 246 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 247 | 248 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 249 | 250 | TestProducer.send(producer, {:remove, job.name}) 251 | 252 | TestProducer.send(producer, %ClockEvent{ 253 | time: DateTime.add(now, 1, :second), 254 | catch_up: false 255 | }) 256 | 257 | refute_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 258 | end) 259 | end 260 | 261 | test "remove inexistent will not crash", %{producer: producer} do 262 | job = 263 | TestScheduler.new_job() 264 | |> Job.set_schedule(~e[*]e) 265 | 266 | capture_log(fn -> 267 | TestProducer.send(producer, {:add, job}) 268 | 269 | now = DateTime.truncate(DateTime.utc_now(), :second) 270 | TestProducer.send(producer, %ClockEvent{time: now, catch_up: false}) 271 | 272 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 273 | 274 | TestProducer.send(producer, {:remove, make_ref()}) 275 | 276 | TestProducer.send(producer, %ClockEvent{ 277 | time: DateTime.add(now, 1, :second), 278 | catch_up: false 279 | }) 280 | 281 | assert_receive {:received, %ExecuteEvent{job: ^job}}, @max_timeout 282 | end) 283 | end 284 | end 285 | end 286 | -------------------------------------------------------------------------------- /lib/quantum/job_broadcaster.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum.JobBroadcaster do 2 | @moduledoc false 3 | 4 | # This Module is here to broadcast added / removed tabs into the execution pipeline. 5 | 6 | use GenStage 7 | 8 | require Logger 9 | 10 | alias Quantum.Job 11 | alias __MODULE__.{InitOpts, StartOpts, State} 12 | 13 | @type event :: {:add, Job.t()} | {:remove, Job.t()} 14 | 15 | # Start Job Broadcaster 16 | @spec start_link(StartOpts.t()) :: GenServer.on_start() 17 | def start_link(%StartOpts{name: name} = opts) do 18 | __MODULE__ 19 | |> GenStage.start_link( 20 | struct!(InitOpts, Map.take(opts, [:jobs, :storage, :scheduler, :debug_logging])), 21 | name: name 22 | ) 23 | |> case do 24 | {:ok, pid} -> 25 | {:ok, pid} 26 | 27 | {:error, {:already_started, pid}} -> 28 | Process.monitor(pid) 29 | {:ok, pid} 30 | 31 | {:error, _reason} = error -> 32 | error 33 | end 34 | end 35 | 36 | @impl GenStage 37 | def init(%InitOpts{ 38 | jobs: jobs, 39 | storage: storage, 40 | scheduler: scheduler, 41 | debug_logging: debug_logging 42 | }) do 43 | storage_pid = GenServer.whereis(Module.concat(scheduler, Storage)) 44 | 45 | effective_jobs = 46 | storage_pid 47 | |> storage.jobs() 48 | |> case do 49 | :not_applicable -> 50 | debug_logging && 51 | Logger.debug(fn -> 52 | {"Loading Initial Jobs from Config", node: Node.self()} 53 | end) 54 | 55 | jobs 56 | 57 | storage_jobs when is_list(storage_jobs) -> 58 | debug_logging && 59 | Logger.debug(fn -> 60 | {"Loading Initial Jobs from Storage, skipping config", node: Node.self()} 61 | end) 62 | 63 | for %Job{state: :active} = job <- storage_jobs do 64 | # Send event to telemetry in case the end user wants to monitor events 65 | :telemetry.execute([:quantum, :job, :add], %{}, %{ 66 | job: job, 67 | scheduler: scheduler 68 | }) 69 | end 70 | 71 | storage_jobs 72 | end 73 | 74 | {:producer, 75 | %State{ 76 | jobs: effective_jobs |> Enum.map(&{&1.name, &1}) |> Map.new(), 77 | buffer: for(%{state: :active} = job <- effective_jobs, do: {:add, job}), 78 | storage: storage, 79 | storage_pid: storage_pid, 80 | scheduler: scheduler, 81 | debug_logging: debug_logging 82 | }} 83 | end 84 | 85 | @impl GenStage 86 | def handle_demand(demand, %State{buffer: buffer} = state) do 87 | {to_send, remaining} = Enum.split(buffer, demand) 88 | 89 | {:noreply, to_send, %{state | buffer: remaining}} 90 | end 91 | 92 | @impl GenStage 93 | 94 | def handle_cast( 95 | {:add, %Job{state: :active, name: job_name} = job}, 96 | %State{ 97 | jobs: jobs, 98 | storage: storage, 99 | storage_pid: storage_pid, 100 | debug_logging: debug_logging 101 | } = state 102 | ) do 103 | case jobs do 104 | %{^job_name => %Job{state: :active} = old_job} -> 105 | debug_logging && 106 | Logger.debug(fn -> 107 | {"Replacing job", node: Node.self(), name: job_name} 108 | end) 109 | 110 | :ok = update_job(storage, storage_pid, job, state.scheduler) 111 | 112 | {:noreply, [{:remove, old_job}, {:add, job}], 113 | %{state | jobs: Map.put(jobs, job_name, job)}} 114 | 115 | %{^job_name => %Job{state: :inactive}} -> 116 | debug_logging && 117 | Logger.debug(fn -> 118 | {"Replacing job", node: Node.self(), name: job_name} 119 | end) 120 | 121 | :ok = update_job(storage, storage_pid, job, state.scheduler) 122 | 123 | {:noreply, [{:add, job}], %{state | jobs: Map.put(jobs, job_name, job)}} 124 | 125 | _ -> 126 | debug_logging && 127 | Logger.debug(fn -> 128 | {"Adding job", node: Node.self(), name: job_name} 129 | end) 130 | 131 | # Send event to telemetry in case the end user wants to monitor events 132 | :telemetry.execute([:quantum, :job, :add], %{}, %{ 133 | job: job, 134 | scheduler: state.scheduler 135 | }) 136 | 137 | :ok = storage.add_job(storage_pid, job) 138 | 139 | {:noreply, [{:add, job}], %{state | jobs: Map.put(jobs, job_name, job)}} 140 | end 141 | end 142 | 143 | def handle_cast( 144 | {:add, %Job{state: :inactive, name: job_name} = job}, 145 | %State{ 146 | jobs: jobs, 147 | storage: storage, 148 | storage_pid: storage_pid, 149 | debug_logging: debug_logging 150 | } = state 151 | ) do 152 | case jobs do 153 | %{^job_name => %Job{state: :active} = old_job} -> 154 | debug_logging && 155 | Logger.debug(fn -> 156 | {"Replacing job", node: Node.self(), name: job_name} 157 | end) 158 | 159 | :ok = update_job(storage, storage_pid, job, state.scheduler) 160 | 161 | {:noreply, [{:remove, old_job}], %{state | jobs: Map.put(jobs, job_name, job)}} 162 | 163 | %{^job_name => %Job{state: :inactive}} -> 164 | debug_logging && 165 | Logger.debug(fn -> 166 | {"Replacing job", node: Node.self(), name: job_name} 167 | end) 168 | 169 | :ok = update_job(storage, storage_pid, job, state.scheduler) 170 | 171 | {:noreply, [], %{state | jobs: Map.put(jobs, job_name, job)}} 172 | 173 | _ -> 174 | debug_logging && 175 | Logger.debug(fn -> 176 | {"Adding job", node: Node.self(), name: job_name} 177 | end) 178 | 179 | # Send event to telemetry in case the end user wants to monitor events 180 | :telemetry.execute([:quantum, :job, :add], %{}, %{ 181 | job: job, 182 | scheduler: state.scheduler 183 | }) 184 | 185 | :ok = storage.add_job(storage_pid, job) 186 | 187 | {:noreply, [], %{state | jobs: Map.put(jobs, job_name, job)}} 188 | end 189 | end 190 | 191 | def handle_cast( 192 | {:delete, name}, 193 | %State{ 194 | jobs: jobs, 195 | storage: storage, 196 | storage_pid: storage_pid, 197 | debug_logging: debug_logging 198 | } = state 199 | ) do 200 | debug_logging && 201 | Logger.debug(fn -> 202 | {"Deleting job", node: Node.self(), name: name} 203 | end) 204 | 205 | case Map.fetch(jobs, name) do 206 | {:ok, %{state: :active, name: name} = job} -> 207 | # Send event to telemetry in case the end user wants to monitor events 208 | :telemetry.execute([:quantum, :job, :delete], %{}, %{ 209 | job: job, 210 | scheduler: state.scheduler 211 | }) 212 | 213 | :ok = storage.delete_job(storage_pid, name) 214 | 215 | {:noreply, [{:remove, name}], %{state | jobs: Map.delete(jobs, name)}} 216 | 217 | {:ok, %{state: :inactive, name: name} = job} -> 218 | # Send event to telemetry in case the end user wants to monitor events 219 | :telemetry.execute([:quantum, :job, :delete], %{}, %{ 220 | job: job, 221 | scheduler: state.scheduler 222 | }) 223 | 224 | :ok = storage.delete_job(storage_pid, name) 225 | 226 | {:noreply, [], %{state | jobs: Map.delete(jobs, name)}} 227 | 228 | :error -> 229 | {:noreply, [], state} 230 | end 231 | end 232 | 233 | def handle_cast( 234 | {:change_state, name, new_state}, 235 | %State{ 236 | jobs: jobs, 237 | storage: storage, 238 | storage_pid: storage_pid, 239 | debug_logging: debug_logging 240 | } = state 241 | ) do 242 | debug_logging && 243 | Logger.debug(fn -> 244 | {"Change job state", node: Node.self(), name: name} 245 | end) 246 | 247 | case Map.fetch(jobs, name) do 248 | :error -> 249 | {:noreply, [], state} 250 | 251 | {:ok, %{state: ^new_state}} -> 252 | {:noreply, [], state} 253 | 254 | {:ok, job} -> 255 | # Send event to telemetry in case the end user wants to monitor events 256 | :telemetry.execute([:quantum, :job, :update], %{}, %{ 257 | job: job, 258 | scheduler: state.scheduler 259 | }) 260 | 261 | jobs = Map.update!(jobs, name, &Job.set_state(&1, new_state)) 262 | 263 | :ok = storage.update_job_state(storage_pid, job.name, new_state) 264 | 265 | case new_state do 266 | :active -> 267 | {:noreply, [{:add, %{job | state: new_state}}], %{state | jobs: jobs}} 268 | 269 | :inactive -> 270 | {:noreply, [{:remove, name}], %{state | jobs: jobs}} 271 | end 272 | end 273 | end 274 | 275 | def handle_cast( 276 | {:run_job, name}, 277 | %State{ 278 | jobs: jobs, 279 | debug_logging: debug_logging 280 | } = state 281 | ) do 282 | debug_logging && 283 | Logger.debug(fn -> 284 | {"Running job once", node: Node.self(), name: name} 285 | end) 286 | 287 | case Map.fetch(jobs, name) do 288 | :error -> 289 | {:noreply, [], state} 290 | 291 | {:ok, job} -> 292 | {:noreply, [{:run, job}], state} 293 | end 294 | end 295 | 296 | def handle_cast( 297 | :delete_all, 298 | %State{ 299 | jobs: jobs, 300 | storage: storage, 301 | storage_pid: storage_pid, 302 | debug_logging: debug_logging 303 | } = state 304 | ) do 305 | debug_logging && 306 | Logger.debug(fn -> 307 | {"Deleting all jobs", node: Node.self()} 308 | end) 309 | 310 | for {_name, %Job{} = job} <- jobs do 311 | # Send event to telemetry in case the end user wants to monitor events 312 | :telemetry.execute([:quantum, :job, :delete], %{}, %{ 313 | job: job, 314 | scheduler: state.scheduler 315 | }) 316 | end 317 | 318 | messages = for {name, %Job{state: :active}} <- jobs, do: {:remove, name} 319 | 320 | :ok = storage.purge(storage_pid) 321 | 322 | {:noreply, messages, %{state | jobs: %{}}} 323 | end 324 | 325 | @impl GenStage 326 | def handle_call(:jobs, _, %State{jobs: jobs} = state), 327 | do: {:reply, Map.to_list(jobs), [], state} 328 | 329 | def handle_call({:find_job, name}, _, %State{jobs: jobs} = state), 330 | do: {:reply, Map.get(jobs, name), [], state} 331 | 332 | @impl GenStage 333 | def handle_info(_message, state) do 334 | {:noreply, [], state} 335 | end 336 | 337 | defp update_job(storage, storage_pid, %Job{name: job_name} = job, scheduler) do 338 | # Send event to telemetry in case the end user wants to monitor events 339 | :telemetry.execute([:quantum, :job, :update], %{}, %{ 340 | job: job, 341 | scheduler: scheduler 342 | }) 343 | 344 | if function_exported?(storage, :update_job, 2) do 345 | :ok = storage.update_job(storage_pid, job) 346 | else 347 | :ok = storage.delete_job(storage_pid, job_name) 348 | :ok = storage.add_job(storage_pid, job) 349 | end 350 | end 351 | end 352 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /lib/quantum.ex: -------------------------------------------------------------------------------- 1 | defmodule Quantum do 2 | use TelemetryRegistry 3 | 4 | telemetry_event(%{ 5 | event: [:quantum, :job, :add], 6 | description: "dispatched when a job is added", 7 | measurements: "%{}", 8 | metadata: "%{job: Quantum.Job.t(), scheduler: atom()}" 9 | }) 10 | 11 | telemetry_event(%{ 12 | event: [:quantum, :job, :update], 13 | description: "dispatched when a job is updated", 14 | measurements: "%{}", 15 | metadata: "%{job: Quantum.Job.t(), scheduler: atom()}" 16 | }) 17 | 18 | telemetry_event(%{ 19 | event: [:quantum, :job, :delete], 20 | description: "dispatched when a job is deleted", 21 | measurements: "%{}", 22 | metadata: "%{job: Quantum.Job.t(), scheduler: atom()}" 23 | }) 24 | 25 | telemetry_event(%{ 26 | event: [:quantum, :job, :start], 27 | description: "dispatched on job execution start", 28 | measurements: "%{system_time: integer()}", 29 | metadata: 30 | "%{telemetry_span_context: term(), job: Quantum.Job.t(), node: Node.t(), scheduler: atom()}" 31 | }) 32 | 33 | telemetry_event(%{ 34 | event: [:quantum, :job, :stop], 35 | description: "dispatched on job execution end", 36 | measurements: "%{duration: integer()}", 37 | metadata: 38 | "%{telemetry_span_context: term(), job: Quantum.Job.t(), node: Node.t(), scheduler: atom(), result: term()}" 39 | }) 40 | 41 | telemetry_event(%{ 42 | event: [:quantum, :job, :exception], 43 | description: "dispatched on job execution fail", 44 | measurements: "%{duration: integer()}", 45 | metadata: 46 | "%{telemetry_span_context: term(), job: Quantum.Job.t(), node: Node.t(), scheduler: atom(), kind: :throw | :error | :exit, reason: term(), stacktrace: list()}" 47 | }) 48 | 49 | @moduledoc """ 50 | Defines a quantum Scheduler. 51 | 52 | When used, the quantum scheduler expects the `:otp_app` as option. 53 | The `:otp_app` should point to an OTP application that has 54 | the quantum runner configuration. For example, the quantum scheduler: 55 | 56 | defmodule MyApp.Scheduler do 57 | use Quantum, otp_app: :my_app 58 | end 59 | 60 | Could be configured with: 61 | 62 | config :my_app, MyApp.Scheduler, 63 | jobs: [ 64 | {"@daily", {Backup, :backup, []}}, 65 | ] 66 | 67 | ## Configuration: 68 | 69 | * `:clock_broadcaster_name` - GenServer name of clock broadcaster \\ 70 | *(unstable, may break without major release until declared stable)* 71 | 72 | * `:execution_broadcaster_name` - GenServer name of execution broadcaster \\ 73 | *(unstable, may break without major release until declared stable)* 74 | 75 | * `:executor_supervisor_name` - GenServer name of execution supervisor \\ 76 | *(unstable, may break without major release until declared stable)* 77 | 78 | * `:debug_logging` - Turn on debug logging 79 | 80 | * `:jobs` - list of cron jobs to execute 81 | 82 | * `:job_broadcaster_name` - GenServer name of job broadcaster \\ 83 | *(unstable, may break without major release until declared stable)* 84 | 85 | * `:name` - GenServer name of scheduler \\ 86 | *(unstable, may break without major release until declared stable)* 87 | 88 | * `:node_selector_broadcaster_name` - GenServer name of node selector broadcaster \\ 89 | *(unstable, may break without major release until declared stable)* 90 | 91 | * `:overlap` - Default overlap of new Job 92 | 93 | * `:otp_app` - Application where scheduler runs 94 | 95 | * `:run_strategy` - Default Run Strategy of new Job 96 | 97 | * `:schedule` - Default schedule of new Job 98 | 99 | * `:storage` - Storage to use for persistence 100 | 101 | * `:storage_name` - GenServer name of storage \\ 102 | *(unstable, may break without major release until declared stable)* 103 | 104 | * `:supervisor_module` - Module to supervise scheduler \\ 105 | Can be overwritten to supervise processes differently (for example for clustering) \\ 106 | *(unstable, may break without major release until declared stable)* 107 | 108 | * `:task_registry_name` - GenServer name of task registry \\ 109 | *(unstable, may break without major release until declared stable)* 110 | 111 | * `:task_supervisor_name` - GenServer name of task supervisor \\ 112 | *(unstable, may break without major release until declared stable)* 113 | 114 | * `:timeout` - Sometimes, you may come across GenServer timeout errors 115 | esp. when you have too many jobs or high load. The default `GenServer.call/3` 116 | timeout is `5_000`. 117 | 118 | * `:timezone` - Default timezone of new Job 119 | 120 | ## Telemetry 121 | 122 | #{telemetry_docs()} 123 | 124 | ### Examples 125 | 126 | iex(1)> :telemetry_registry.discover_all(:quantum) 127 | :ok 128 | iex(2)> :telemetry_registry.spannable_events() 129 | [{[:quantum, :job], [:start, :stop, :exception]}] 130 | iex(3)> :telemetry_registry.list_events 131 | [ 132 | {[:quantum, :job, :add], Quantum, 133 | %{ 134 | description: "dispatched when a job is added", 135 | measurements: "%{}", 136 | metadata: "%{job: Quantum.Job.t(), scheduler: atom()}" 137 | }}, 138 | {[:quantum, :job, :delete], Quantum, 139 | %{ 140 | description: "dispatched when a job is deleted", 141 | measurements: "%{}", 142 | metadata: "%{job: Quantum.Job.t(), scheduler: atom()}" 143 | }}, 144 | {[:quantum, :job, :exception], Quantum, 145 | %{ 146 | description: "dispatched on job execution fail", 147 | measurements: "%{duration: integer()}", 148 | metadata: "%{telemetry_span_context: term(), job: Quantum.Job.t(), node: Node.t(), scheduler: atom(), kind: :throw | :error | :exit, reason: term(), stacktrace: list()}" 149 | }}, 150 | {[:quantum, :job, :start], Quantum, 151 | %{ 152 | description: "dispatched on job execution start", 153 | measurements: "%{system_time: integer()}", 154 | metadata: "%{telemetry_span_context: term(), job: Quantum.Job.t(), node: Node.t(), scheduler: atom()}" 155 | }}, 156 | {[:quantum, :job, :stop], Quantum, 157 | %{ 158 | description: "dispatched on job execution end", 159 | measurements: "%{duration: integer()}", 160 | metadata: "%{telemetry_span_context: term(), job: Quantum.Job.t(), node: Node.t(), scheduler: atom(), result: term()}" 161 | }}, 162 | {[:quantum, :job, :update], Quantum, 163 | %{ 164 | description: "dispatched when a job is updated", 165 | measurements: "%{}", 166 | metadata: "%{job: Quantum.Job.t(), scheduler: atom()}" 167 | }} 168 | ] 169 | """ 170 | 171 | require Logger 172 | 173 | alias Quantum.{Job, Normalizer, RunStrategy.Random, Storage.Noop} 174 | 175 | @typedoc """ 176 | Quantum Scheduler Implementation 177 | """ 178 | @type t :: module 179 | 180 | @defaults [ 181 | timeout: 5_000, 182 | schedule: nil, 183 | overlap: true, 184 | state: :active, 185 | timezone: :utc, 186 | run_strategy: {Random, :cluster}, 187 | debug_logging: true, 188 | storage: Noop 189 | ] 190 | 191 | # Returns the configuration stored in the `:otp_app` environment. 192 | @doc false 193 | @callback config(Keyword.t()) :: Keyword.t() 194 | 195 | @doc """ 196 | Starts supervision and return `{:ok, pid}` 197 | or just `:ok` if nothing needs to be done. 198 | 199 | Returns `{:error, {:already_started, pid}}` if the scheduler is already 200 | started or `{:error, term}` in case anything else goes wrong. 201 | 202 | ## Options 203 | 204 | See the configuration in the moduledoc for options. 205 | """ 206 | @callback start_link(opts :: Keyword.t()) :: 207 | {:ok, pid} 208 | | {:error, {:already_started, pid}} 209 | | {:error, term} 210 | 211 | @doc """ 212 | A callback executed when the quantum starts. 213 | 214 | It takes the quantum configuration that is stored in the application 215 | environment, and may change it to suit the application business. 216 | 217 | It must return the updated list of configuration 218 | """ 219 | @callback init(config :: Keyword.t()) :: Keyword.t() 220 | 221 | @doc """ 222 | Shuts down the quantum represented by the given pid. 223 | """ 224 | @callback stop(server :: GenServer.server(), timeout) :: :ok 225 | 226 | @doc """ 227 | Creates a new Job. The job can be added by calling `add_job/1`. 228 | 229 | ## Supported options 230 | 231 | * `name` - see `Quantum.Job.set_name/2` 232 | * `overlap` - see `Quantum.Job.set_overlap/2` 233 | * `run_strategy` - see `Quantum.Job.set_run_strategy/2` 234 | * `schedule` - see `Quantum.Job.set_schedule/2` 235 | * `state` - see `Quantum.Job.set_state/2` 236 | * `task` - see `Quantum.Job.set_task/2` 237 | * `timezone` - see `Quantum.Job.set_timezone/2` 238 | """ 239 | @callback new_job(opts :: Keyword.t()) :: Quantum.Job.t() 240 | 241 | @doc """ 242 | Adds a new job 243 | """ 244 | @callback add_job(GenStage.stage(), Quantum.Job.t() | {Crontab.CronExpression.t(), Job.task()}) :: 245 | :ok 246 | 247 | @doc """ 248 | Deactivates a job by name 249 | """ 250 | @callback deactivate_job(GenStage.stage(), atom) :: :ok 251 | 252 | @doc """ 253 | Activates a job by name 254 | """ 255 | @callback activate_job(GenStage.stage(), atom) :: :ok 256 | 257 | @doc """ 258 | Runs a job by name once 259 | """ 260 | @callback run_job(GenStage.stage(), atom) :: :ok 261 | 262 | @doc """ 263 | Resolves a job by name 264 | """ 265 | @callback find_job(GenStage.stage(), atom) :: Quantum.Job.t() | nil 266 | 267 | @doc """ 268 | Deletes a job by name 269 | """ 270 | @callback delete_job(GenStage.stage(), atom) :: :ok 271 | 272 | @doc """ 273 | Deletes all jobs 274 | """ 275 | @callback delete_all_jobs(GenStage.stage()) :: :ok 276 | 277 | @doc """ 278 | Returns the list of currently defined jobs 279 | """ 280 | @callback jobs(GenStage.stage()) :: [Quantum.Job.t()] 281 | 282 | @doc false 283 | # Retrieves only scheduler related configuration. 284 | def scheduler_config(opts, scheduler, otp_app) do 285 | @defaults 286 | |> Keyword.merge(Application.get_env(otp_app, scheduler, [])) 287 | |> Keyword.merge(opts) 288 | |> Keyword.put_new(:otp_app, otp_app) 289 | |> Keyword.put_new(:scheduler, scheduler) 290 | |> Keyword.put_new(:name, scheduler) 291 | |> update_in([:schedule], &Normalizer.normalize_schedule/1) 292 | |> Keyword.put_new(:task_supervisor_name, Module.concat(scheduler, TaskSupervisor)) 293 | |> Keyword.put_new(:storage_name, Module.concat(scheduler, Storage)) 294 | |> Keyword.put_new(:task_registry_name, Module.concat(scheduler, TaskRegistry)) 295 | |> Keyword.put_new(:clock_broadcaster_name, Module.concat(scheduler, ClockBroadcaster)) 296 | |> Keyword.put_new(:job_broadcaster_name, Module.concat(scheduler, JobBroadcaster)) 297 | |> Keyword.put_new( 298 | :execution_broadcaster_name, 299 | Module.concat(scheduler, ExecutionBroadcaster) 300 | ) 301 | |> Keyword.put_new( 302 | :node_selector_broadcaster_name, 303 | Module.concat(scheduler, NodeSelectorBroadcaster) 304 | ) 305 | |> Keyword.put_new(:executor_supervisor_name, Module.concat(scheduler, ExecutorSupervisor)) 306 | |> Kernel.then(fn config -> 307 | Keyword.update(config, :jobs, [], fn jobs -> 308 | jobs 309 | |> Enum.map(&Normalizer.normalize(scheduler.__new_job__([], config), &1)) 310 | |> remove_invalid_jobs(scheduler) 311 | end) 312 | end) 313 | |> Keyword.put_new(:supervisor_module, Quantum.Supervisor) 314 | |> Keyword.put_new(:name, Quantum.Supervisor) 315 | end 316 | 317 | defp remove_invalid_jobs(job_list, scheduler) do 318 | job_list 319 | |> Enum.reduce(%{}, fn %Job{name: name} = job, acc -> 320 | cond do 321 | duplicate_job?(Map.keys(acc), job) -> 322 | Logger.warning( 323 | "Job with name #{inspect(name)} of scheduler #{inspect(scheduler)} not started: duplicate job name" 324 | ) 325 | 326 | acc 327 | 328 | invalid_job_task?(job) -> 329 | Logger.warning( 330 | "Job with name #{inspect(name)} of scheduler #{inspect(scheduler)} not started: invalid task function" 331 | ) 332 | 333 | acc 334 | 335 | :else -> 336 | Map.put_new(acc, name, job) 337 | end 338 | end) 339 | |> Map.values() 340 | end 341 | 342 | defp duplicate_job?(existent_jobs, %Job{name: name}), do: Enum.member?(existent_jobs, name) 343 | 344 | defp invalid_job_task?(%Job{task: {m, f, args}}) 345 | when is_atom(m) and is_atom(f) and is_list(args) do 346 | if Code.ensure_loaded?(m), 347 | do: not function_exported?(m, f, length(args)), 348 | else: true 349 | end 350 | 351 | defp invalid_job_task?(_), do: false 352 | 353 | defmacro __using__(opts) do 354 | quote bind_quoted: [behaviour: __MODULE__, opts: opts, moduledoc: @moduledoc], 355 | location: :keep do 356 | @otp_app Keyword.fetch!(opts, :otp_app) 357 | @moduledoc moduledoc 358 | |> String.replace(~r/MyApp\.Scheduler/, Enum.join(Module.split(__MODULE__), ".")) 359 | |> String.replace(~r/:my_app/, ":" <> Atom.to_string(@otp_app)) 360 | 361 | @behaviour behaviour 362 | 363 | @doc false 364 | @impl behaviour 365 | def config(opts \\ []) do 366 | Quantum.scheduler_config(opts, __MODULE__, @otp_app) 367 | end 368 | 369 | defp __job_broadcaster__ do 370 | config() |> Keyword.fetch!(:job_broadcaster_name) 371 | end 372 | 373 | defp __timeout__, do: Keyword.fetch!(config(), :timeout) 374 | 375 | @impl behaviour 376 | def start_link(opts \\ []) do 377 | opts = config(opts) 378 | Keyword.fetch!(opts, :supervisor_module).start_link(__MODULE__, opts) 379 | end 380 | 381 | @impl behaviour 382 | def init(opts) do 383 | opts 384 | end 385 | 386 | @impl behaviour 387 | def stop(server \\ __MODULE__, timeout \\ 5000) do 388 | Supervisor.stop(server, :normal, timeout) 389 | end 390 | 391 | @impl behaviour 392 | def add_job(server \\ __job_broadcaster__(), job) 393 | 394 | def add_job(server, %Job{name: name} = job) do 395 | GenStage.cast(server, {:add, job}) 396 | end 397 | 398 | def add_job(server, {%Crontab.CronExpression{} = schedule, task}) 399 | when is_tuple(task) or is_function(task, 0) do 400 | job = 401 | new_job() 402 | |> Job.set_schedule(schedule) 403 | |> Job.set_task(task) 404 | 405 | add_job(server, job) 406 | end 407 | 408 | @impl behaviour 409 | def new_job(opts \\ []), do: __new_job__(opts, config()) 410 | 411 | @doc false 412 | def __new_job__(opts, config) do 413 | config 414 | |> Keyword.take([:overlap, :schedule, :state, :timezone, :run_strategy]) 415 | |> Keyword.merge(opts) 416 | |> Keyword.update!(:run_strategy, fn 417 | {module, options} when is_atom(module) -> module.normalize_config!(options) 418 | module when is_atom(module) -> module.normalize_config!(nil) 419 | %_struct{} = run_strategy -> run_strategy 420 | end) 421 | |> Job.new() 422 | end 423 | 424 | @impl behaviour 425 | def deactivate_job(server \\ __job_broadcaster__(), name) 426 | when is_atom(name) or is_reference(name) do 427 | GenStage.cast(server, {:change_state, name, :inactive}) 428 | end 429 | 430 | @impl behaviour 431 | def activate_job(server \\ __job_broadcaster__(), name) 432 | when is_atom(name) or is_reference(name) do 433 | GenStage.cast(server, {:change_state, name, :active}) 434 | end 435 | 436 | @impl behaviour 437 | def run_job(server \\ __job_broadcaster__(), name) 438 | when is_atom(name) or is_reference(name) do 439 | GenStage.cast(server, {:run_job, name}) 440 | end 441 | 442 | @impl behaviour 443 | def find_job(server \\ __job_broadcaster__(), name) 444 | when is_atom(name) or is_reference(name) do 445 | GenStage.call(server, {:find_job, name}, __timeout__()) 446 | end 447 | 448 | @impl behaviour 449 | def delete_job(server \\ __job_broadcaster__(), name) 450 | when is_atom(name) or is_reference(name) do 451 | GenStage.cast(server, {:delete, name}) 452 | end 453 | 454 | @impl behaviour 455 | def delete_all_jobs(server \\ __job_broadcaster__()) do 456 | GenStage.cast(server, :delete_all) 457 | end 458 | 459 | @impl behaviour 460 | def jobs(server \\ __job_broadcaster__()) do 461 | GenStage.call(server, :jobs, __timeout__()) 462 | end 463 | 464 | spec = [ 465 | id: opts[:id] || __MODULE__, 466 | start: Macro.escape(opts[:start]) || quote(do: {__MODULE__, :start_link, [opts]}), 467 | restart: opts[:restart] || :permanent, 468 | type: :worker 469 | ] 470 | 471 | @spec child_spec(Keyword.t()) :: Supervisor.child_spec() 472 | def child_spec(opts) do 473 | %{unquote_splicing(spec)} 474 | end 475 | 476 | defoverridable child_spec: 1, config: 0, config: 1, init: 1 477 | end 478 | end 479 | end 480 | -------------------------------------------------------------------------------- /test/quantum/executor_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.ExecutorTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import ExUnit.CaptureLog 7 | 8 | alias Quantum.{Executor, Executor.StartOpts, Job, NodeSelectorBroadcaster.Event} 9 | alias Quantum.TaskRegistry 10 | alias Quantum.TaskRegistry.StartOpts, as: TaskRegistryStartOpts 11 | 12 | doctest Executor 13 | 14 | defmodule TestScheduler do 15 | @moduledoc false 16 | 17 | use Quantum, otp_app: :job_broadcaster_test 18 | end 19 | 20 | defmodule TelemetryTestHandler do 21 | @moduledoc false 22 | 23 | def handle_event( 24 | [:quantum, :job, :start], 25 | %{system_time: _system_time} = _measurements, 26 | %{ 27 | job: %Job{name: job_name}, 28 | node: _node, 29 | scheduler: _scheduler, 30 | telemetry_span_context: telemetry_span_context 31 | } = _metadata, 32 | %{parent_thread: parent_thread, test_id: test_id} 33 | ) do 34 | send(parent_thread, %{ 35 | test_id: test_id, 36 | job_name: job_name, 37 | type: :start, 38 | telemetry_span_context: telemetry_span_context 39 | }) 40 | end 41 | 42 | def handle_event( 43 | [:quantum, :job, :stop], 44 | %{duration: _duration} = _measurements, 45 | %{ 46 | job: %Job{name: job_name}, 47 | node: _node, 48 | scheduler: _scheduler, 49 | result: _result, 50 | telemetry_span_context: telemetry_span_context 51 | } = _metadata, 52 | %{parent_thread: parent_thread, test_id: test_id} 53 | ) do 54 | send(parent_thread, %{ 55 | test_id: test_id, 56 | job_name: job_name, 57 | telemetry_span_context: telemetry_span_context, 58 | type: :stop 59 | }) 60 | end 61 | 62 | def handle_event( 63 | [:quantum, :job, :exception], 64 | %{duration: _duration} = _measurements, 65 | %{ 66 | job: %Job{name: job_name}, 67 | node: _node, 68 | scheduler: _scheduler, 69 | kind: kind, 70 | reason: reason, 71 | stacktrace: stacktrace, 72 | telemetry_span_context: telemetry_span_context 73 | } = _metadata, 74 | %{parent_thread: parent_thread, test_id: test_id} 75 | ) do 76 | send(parent_thread, %{ 77 | test_id: test_id, 78 | job_name: job_name, 79 | telemetry_span_context: telemetry_span_context, 80 | type: :exception, 81 | kind: kind, 82 | reason: reason, 83 | stacktrace: stacktrace 84 | }) 85 | end 86 | end 87 | 88 | defp attach_telemetry(test_id, parent_thread) do 89 | :telemetry.attach_many( 90 | test_id, 91 | [ 92 | [:quantum, :job, :start], 93 | [:quantum, :job, :stop], 94 | [:quantum, :job, :exception] 95 | ], 96 | &TelemetryTestHandler.handle_event/4, 97 | %{ 98 | parent_thread: parent_thread, 99 | test_id: test_id 100 | } 101 | ) 102 | end 103 | 104 | setup tags do 105 | {:ok, _task_supervisor} = 106 | start_supervised({Task.Supervisor, [name: Module.concat(__MODULE__, TaskSupervisor)]}) 107 | 108 | process_name = Module.concat(__MODULE__, tags.test) 109 | 110 | Process.register(self(), process_name) 111 | 112 | {:ok, _task_registry} = 113 | start_supervised( 114 | {TaskRegistry, 115 | %TaskRegistryStartOpts{ 116 | name: Module.concat(__MODULE__, TaskRegistry), 117 | listeners: [process_name] 118 | }} 119 | ) 120 | 121 | { 122 | :ok, 123 | %{ 124 | task_supervisor: Module.concat(__MODULE__, TaskSupervisor), 125 | task_registry: Module.concat(__MODULE__, TaskRegistry), 126 | debug_logging: true, 127 | scheduler: TestScheduler 128 | } 129 | } 130 | end 131 | 132 | describe "start_link/3" do 133 | test "executes given task using anonymous function", %{ 134 | task_supervisor: task_supervisor, 135 | task_registry: task_registry, 136 | debug_logging: debug_logging, 137 | scheduler: scheduler 138 | } do 139 | caller = self() 140 | 141 | test_id = "log-anonymous-job-handler" 142 | 143 | :ok = attach_telemetry(test_id, self()) 144 | 145 | job = 146 | TestScheduler.new_job() 147 | |> Job.set_task(fn -> send(caller, :executed) end) 148 | 149 | capture_log(fn -> 150 | Executor.start_link( 151 | %StartOpts{ 152 | task_supervisor_reference: task_supervisor, 153 | task_registry_reference: task_registry, 154 | debug_logging: debug_logging, 155 | scheduler: scheduler 156 | }, 157 | %Event{job: job, node: Node.self()} 158 | ) 159 | 160 | assert_receive :executed 161 | 162 | assert_receive %{test_id: ^test_id, type: :start} 163 | assert_receive %{test_id: ^test_id, type: :stop}, 2000 164 | end) 165 | end 166 | 167 | test "executes given task using function tuple", %{ 168 | task_supervisor: task_supervisor, 169 | task_registry: task_registry, 170 | debug_logging: debug_logging, 171 | scheduler: scheduler 172 | } do 173 | caller = self() 174 | 175 | test_id = "log-function-tuple-job-handler" 176 | 177 | :ok = attach_telemetry(test_id, self()) 178 | 179 | job = 180 | TestScheduler.new_job() 181 | |> Job.set_task({__MODULE__, :send, [caller]}) 182 | 183 | capture_log(fn -> 184 | Executor.start_link( 185 | %StartOpts{ 186 | task_supervisor_reference: task_supervisor, 187 | task_registry_reference: task_registry, 188 | debug_logging: debug_logging, 189 | scheduler: scheduler 190 | }, 191 | %Event{job: job, node: Node.self()} 192 | ) 193 | 194 | assert_receive :executed 195 | end) 196 | 197 | assert_receive %{test_id: ^test_id, type: :start} 198 | assert_receive %{test_id: ^test_id, type: :stop}, 2000 199 | end 200 | 201 | test "executes given task without overlap", %{ 202 | task_supervisor: task_supervisor, 203 | task_registry: task_registry, 204 | debug_logging: debug_logging, 205 | scheduler: scheduler 206 | } do 207 | caller = self() 208 | test_id = "log-task-no-overlap-handler" 209 | 210 | :ok = attach_telemetry(test_id, self()) 211 | 212 | job = 213 | TestScheduler.new_job() 214 | |> Job.set_task(fn -> 215 | send(caller, :executed) 216 | Process.sleep(500) 217 | end) 218 | |> Job.set_overlap(false) 219 | 220 | capture_log(fn -> 221 | Executor.start_link( 222 | %StartOpts{ 223 | task_supervisor_reference: task_supervisor, 224 | task_registry_reference: task_registry, 225 | debug_logging: debug_logging, 226 | scheduler: scheduler 227 | }, 228 | %Event{job: job, node: Node.self()} 229 | ) 230 | 231 | Executor.start_link( 232 | %StartOpts{ 233 | task_supervisor_reference: task_supervisor, 234 | task_registry_reference: task_registry, 235 | debug_logging: debug_logging, 236 | scheduler: scheduler 237 | }, 238 | %Event{job: job, node: Node.self()} 239 | ) 240 | 241 | assert_receive :executed 242 | refute_receive :executed 243 | end) 244 | 245 | assert_receive %{test_id: ^test_id, type: :start} 246 | assert_receive %{test_id: ^test_id, type: :stop}, 2000 247 | end 248 | 249 | test "releases lock on success", %{ 250 | task_supervisor: task_supervisor, 251 | task_registry: task_registry, 252 | debug_logging: debug_logging, 253 | scheduler: scheduler 254 | } do 255 | caller = self() 256 | test_id = "release-lock-on-success-handler" 257 | 258 | :ok = attach_telemetry(test_id, self()) 259 | 260 | job = 261 | TestScheduler.new_job() 262 | |> Job.set_task(fn -> 263 | send(caller, {:executing, self()}) 264 | 265 | receive do 266 | :continue -> nil 267 | end 268 | 269 | send(caller, :execution_end) 270 | end) 271 | |> Job.set_overlap(false) 272 | 273 | job_name = job.name 274 | node = Node.self() 275 | 276 | capture_log(fn -> 277 | Executor.start_link( 278 | %StartOpts{ 279 | task_supervisor_reference: task_supervisor, 280 | task_registry_reference: task_registry, 281 | debug_logging: debug_logging, 282 | scheduler: scheduler 283 | }, 284 | %Event{job: job, node: node} 285 | ) 286 | 287 | assert_receive {:executing, job_pid} 288 | 289 | assert :already_running = TaskRegistry.mark_running(task_registry, job.name, Node.self()) 290 | 291 | send(job_pid, :continue) 292 | 293 | assert_receive :execution_end 294 | 295 | assert_receive {:unregister, _, {^job_name, ^node}, _pid} 296 | 297 | assert :marked_running = TaskRegistry.mark_running(task_registry, job.name, Node.self()) 298 | end) 299 | 300 | assert_receive %{test_id: ^test_id, type: :start} 301 | assert_receive %{test_id: ^test_id, type: :stop}, 2000 302 | end 303 | 304 | test "releases lock on error", %{ 305 | task_supervisor: task_supervisor, 306 | task_registry: task_registry, 307 | debug_logging: debug_logging, 308 | scheduler: scheduler 309 | } do 310 | test_id = "release-lock-on-error-handler" 311 | 312 | :ok = attach_telemetry(test_id, self()) 313 | 314 | job = 315 | TestScheduler.new_job() 316 | |> Job.set_task(fn -> raise "failed" end) 317 | |> Job.set_overlap(false) 318 | 319 | job_name = job.name 320 | node = Node.self() 321 | 322 | # Mute Error 323 | capture_log(fn -> 324 | Executor.start_link( 325 | %StartOpts{ 326 | task_supervisor_reference: task_supervisor, 327 | task_registry_reference: task_registry, 328 | debug_logging: debug_logging, 329 | scheduler: scheduler 330 | }, 331 | %Event{job: job, node: Node.self()} 332 | ) 333 | 334 | assert_receive {:unregister, _, {^job_name, ^node}, _pid} 335 | end) 336 | 337 | assert :marked_running = TaskRegistry.mark_running(task_registry, job.name, Node.self()) 338 | assert_receive %{test_id: ^test_id, type: :start} 339 | 340 | assert_receive %{ 341 | test_id: ^test_id, 342 | type: :exception, 343 | kind: :error, 344 | reason: %RuntimeError{message: "failed"}, 345 | stacktrace: [ 346 | {Quantum.ExecutorTest, _, _, _}, 347 | {Quantum.Executor, _, _, _}, 348 | {:telemetry, _, _, _}, 349 | {Quantum.Executor, _, _, _}, 350 | {Task.Supervised, _, _, _}, 351 | {Task.Supervised, _, _, _} | _rest 352 | ] 353 | }, 354 | 2000 355 | end 356 | 357 | test "logs error", %{ 358 | task_supervisor: task_supervisor, 359 | task_registry: task_registry, 360 | debug_logging: debug_logging, 361 | scheduler: scheduler 362 | } do 363 | test_id = "logs-error-handler" 364 | 365 | :ok = attach_telemetry(test_id, self()) 366 | 367 | job = 368 | TestScheduler.new_job() 369 | |> Job.set_task(fn -> raise "failed" end) 370 | |> Job.set_overlap(false) 371 | 372 | logs = 373 | capture_log(fn -> 374 | {:ok, task} = 375 | Executor.start_link( 376 | %StartOpts{ 377 | task_supervisor_reference: task_supervisor, 378 | task_registry_reference: task_registry, 379 | debug_logging: debug_logging, 380 | scheduler: scheduler 381 | }, 382 | %Event{job: job, node: Node.self()} 383 | ) 384 | 385 | assert :ok == wait_for_termination(task) 386 | end) 387 | 388 | assert logs =~ ~r/[error]/ 389 | assert logs =~ ~r/Execution failed for job/ 390 | assert_receive %{test_id: ^test_id, type: :start} 391 | 392 | assert_receive %{ 393 | test_id: ^test_id, 394 | type: :exception, 395 | kind: :error, 396 | reason: %RuntimeError{message: "failed"}, 397 | stacktrace: [ 398 | {Quantum.ExecutorTest, _, _, _}, 399 | {Quantum.Executor, _, _, _}, 400 | {:telemetry, _, _, _}, 401 | {Quantum.Executor, _, _, _}, 402 | {Task.Supervised, _, _, _}, 403 | {Task.Supervised, _, _, _} | _rest 404 | ] 405 | }, 406 | 2000 407 | end 408 | 409 | test "logs exit", %{ 410 | task_supervisor: task_supervisor, 411 | task_registry: task_registry, 412 | debug_logging: debug_logging, 413 | scheduler: scheduler 414 | } do 415 | test_id = "logs-exit-handler" 416 | 417 | :ok = attach_telemetry(test_id, self()) 418 | 419 | job = 420 | TestScheduler.new_job() 421 | |> Job.set_task(fn -> exit(:failure) end) 422 | |> Job.set_overlap(false) 423 | 424 | logs = 425 | capture_log(fn -> 426 | {:ok, task} = 427 | Executor.start_link( 428 | %StartOpts{ 429 | task_supervisor_reference: task_supervisor, 430 | task_registry_reference: task_registry, 431 | debug_logging: debug_logging, 432 | scheduler: scheduler 433 | }, 434 | %Event{job: job, node: Node.self()} 435 | ) 436 | 437 | assert :ok == wait_for_termination(task) 438 | end) 439 | 440 | assert logs =~ "[error] ** (exit) :failure" 441 | assert_receive %{test_id: ^test_id, type: :start} 442 | 443 | assert_receive %{ 444 | test_id: ^test_id, 445 | type: :exception, 446 | kind: :exit, 447 | reason: :failure, 448 | stacktrace: [ 449 | {Quantum.ExecutorTest, _, _, _}, 450 | {Quantum.Executor, _, _, _}, 451 | {:telemetry, _, _, _}, 452 | {Quantum.Executor, _, _, _}, 453 | {Task.Supervised, _, _, _}, 454 | {Task.Supervised, _, _, _} | _rest 455 | ] 456 | }, 457 | 2000 458 | end 459 | 460 | test "logs throw", %{ 461 | task_supervisor: task_supervisor, 462 | task_registry: task_registry, 463 | debug_logging: debug_logging, 464 | scheduler: scheduler 465 | } do 466 | test_id = "logs-throw-handler" 467 | 468 | :ok = attach_telemetry(test_id, self()) 469 | 470 | ref = make_ref() 471 | 472 | job = 473 | TestScheduler.new_job() 474 | |> Job.set_task(fn -> throw(ref) end) 475 | |> Job.set_overlap(false) 476 | 477 | logs = 478 | capture_log(fn -> 479 | {:ok, task} = 480 | Executor.start_link( 481 | %StartOpts{ 482 | task_supervisor_reference: task_supervisor, 483 | task_registry_reference: task_registry, 484 | debug_logging: debug_logging, 485 | scheduler: scheduler 486 | }, 487 | %Event{job: job, node: Node.self()} 488 | ) 489 | 490 | assert :ok == wait_for_termination(task) 491 | end) 492 | 493 | ~c"#Ref" ++ rest = :erlang.ref_to_list(ref) 494 | assert logs =~ "[error] ** (throw)" 495 | assert logs =~ "#{rest}" 496 | assert_receive %{test_id: ^test_id, type: :start} 497 | 498 | assert_receive %{ 499 | test_id: ^test_id, 500 | type: :exception, 501 | kind: :throw, 502 | reason: ^ref, 503 | stacktrace: [ 504 | {Quantum.ExecutorTest, _, _, _}, 505 | {Quantum.Executor, _, _, _}, 506 | {:telemetry, _, _, _}, 507 | {Quantum.Executor, _, _, _}, 508 | {Task.Supervised, _, _, _}, 509 | {Task.Supervised, _, _, _} | _rest 510 | ] 511 | }, 512 | 2000 513 | end 514 | end 515 | 516 | def send(caller) do 517 | send(caller, :executed) 518 | end 519 | 520 | def wait_for_termination(pid, timeout \\ 5000) do 521 | ref = Process.monitor(pid) 522 | 523 | receive do 524 | {:DOWN, ^ref, :process, _pid, _reason} -> 525 | :ok 526 | after 527 | timeout -> 528 | :error 529 | end 530 | end 531 | end 532 | -------------------------------------------------------------------------------- /test/quantum/job_broadcaster_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Quantum.JobBroadcasterTest do 2 | @moduledoc false 3 | 4 | # , async: true causes random failures due to race conditions with telemetry tests 5 | use ExUnit.Case 6 | 7 | alias Quantum.{Job, JobBroadcaster, JobBroadcaster.StartOpts} 8 | alias Quantum.Storage.Test, as: TestStorage 9 | alias Quantum.Storage.TestWithUpdate, as: TestStorageWithUpdate 10 | alias Quantum.TestConsumer 11 | 12 | import ExUnit.CaptureLog 13 | import Quantum.CaptureLogExtend 14 | 15 | import Crontab.CronExpression 16 | 17 | doctest JobBroadcaster 18 | 19 | defmodule TestScheduler do 20 | @moduledoc false 21 | 22 | use Quantum, otp_app: :job_broadcaster_test 23 | end 24 | 25 | defmodule TelemetryTestHandler do 26 | @moduledoc false 27 | 28 | def handle_event( 29 | [:quantum, :job, :add], 30 | _measurements, 31 | %{job: %Job{name: job_name}, scheduler: _scheduler} = _metadata, 32 | %{parent_thread: parent_thread, test_id: test_id} 33 | ) do 34 | send(parent_thread, %{test_id: test_id, job_name: job_name, type: :add}) 35 | end 36 | 37 | def handle_event( 38 | [:quantum, :job, :delete], 39 | _measurements, 40 | %{job: %Job{name: job_name}, scheduler: _scheduler} = _metadata, 41 | %{parent_thread: parent_thread, test_id: test_id} 42 | ) do 43 | send(parent_thread, %{test_id: test_id, job_name: job_name, type: :delete}) 44 | end 45 | 46 | def handle_event( 47 | [:quantum, :job, :update], 48 | _measurements, 49 | %{job: %Job{name: job_name}, scheduler: _scheduler} = _metadata, 50 | %{parent_thread: parent_thread, test_id: test_id} 51 | ) do 52 | send(parent_thread, %{test_id: test_id, job_name: job_name, type: :update}) 53 | end 54 | end 55 | 56 | defp attach_telemetry(last_atom, test_id, parent_thread) do 57 | :telemetry.attach( 58 | test_id, 59 | [:quantum, :job, last_atom], 60 | &TelemetryTestHandler.handle_event/4, 61 | %{ 62 | parent_thread: parent_thread, 63 | test_id: test_id 64 | } 65 | ) 66 | end 67 | 68 | setup tags do 69 | if tags[:listen_storage] do 70 | Process.put(:test_pid, self()) 71 | end 72 | 73 | active_job = TestScheduler.new_job() 74 | inactive_job = Job.set_state(TestScheduler.new_job(), :inactive) 75 | 76 | init_jobs = 77 | case tags[:jobs] do 78 | :both -> 79 | [active_job, inactive_job] 80 | 81 | :active -> 82 | [active_job] 83 | 84 | :inactive -> 85 | [inactive_job] 86 | 87 | _ -> 88 | [] 89 | end 90 | 91 | storage = 92 | case tags[:storage] do 93 | :with_update -> 94 | TestStorageWithUpdate 95 | 96 | _ -> 97 | TestStorage 98 | end 99 | 100 | broadcaster = 101 | if tags[:manual_dispatch] do 102 | nil 103 | else 104 | {{:ok, broadcaster}, _} = 105 | capture_log_with_return(fn -> 106 | start_supervised( 107 | {JobBroadcaster, 108 | %StartOpts{ 109 | name: __MODULE__, 110 | jobs: init_jobs, 111 | storage: storage, 112 | scheduler: TestScheduler, 113 | debug_logging: true 114 | }} 115 | ) 116 | end) 117 | 118 | {:ok, _consumer} = start_supervised({TestConsumer, [broadcaster, self()]}) 119 | 120 | broadcaster 121 | end 122 | 123 | { 124 | :ok, 125 | broadcaster: broadcaster, 126 | active_job: active_job, 127 | inactive_job: inactive_job, 128 | init_jobs: init_jobs 129 | } 130 | end 131 | 132 | describe "init" do 133 | @tag jobs: :both 134 | test "config jobs", %{active_job: active_job, inactive_job: inactive_job} do 135 | refute_receive {:received, {:add, ^inactive_job}} 136 | assert_receive {:received, {:add, ^active_job}} 137 | end 138 | 139 | @tag manual_dispatch: true 140 | test "storage jobs", %{active_job: active_job, inactive_job: inactive_job} do 141 | test_id = "init-storage-jobs-handler" 142 | 143 | :ok = attach_telemetry(:add, test_id, self()) 144 | 145 | capture_log(fn -> 146 | defmodule FullStorage do 147 | @moduledoc false 148 | 149 | use Quantum.Storage.Test 150 | 151 | def jobs(_), 152 | do: [ 153 | TestScheduler.new_job(), 154 | Job.set_state(TestScheduler.new_job(), :inactive) 155 | ] 156 | end 157 | 158 | {:ok, broadcaster} = 159 | start_supervised( 160 | {JobBroadcaster, 161 | %StartOpts{ 162 | name: __MODULE__, 163 | jobs: [], 164 | storage: FullStorage, 165 | scheduler: TestScheduler, 166 | debug_logging: true 167 | }} 168 | ) 169 | 170 | {:ok, _consumer} = start_supervised({TestConsumer, [broadcaster, self()]}) 171 | 172 | assert_receive {:received, {:add, _}} 173 | refute_receive {:received, {:add, _}} 174 | end) 175 | 176 | # Ensure exactly one :add telemetry notifation because only one job is active 177 | assert_receive %{test_id: ^test_id, type: :add} 178 | refute_receive %{test_id: ^test_id, type: :add} 179 | end 180 | end 181 | 182 | describe "add" do 183 | @tag listen_storage: true 184 | test "active", %{broadcaster: broadcaster, active_job: active_job} do 185 | test_id = "add-active-job-handler" 186 | 187 | :ok = attach_telemetry(:add, test_id, self()) 188 | 189 | assert capture_log(fn -> 190 | TestScheduler.add_job(broadcaster, active_job) 191 | 192 | assert_receive {:received, {:add, ^active_job}} 193 | 194 | assert_receive {:add_job, ^active_job, _} 195 | end) =~ "Adding job" 196 | 197 | assert_receive %{test_id: ^test_id} 198 | end 199 | 200 | test "active (without debug-logging)", %{init_jobs: init_jobs, active_job: active_job} do 201 | refute capture_log(fn -> 202 | # Restart JobBroadcaster with debug-logging false 203 | :ok = stop_supervised(JobBroadcaster) 204 | :ok = stop_supervised(TestConsumer) 205 | 206 | {:ok, broadcaster} = 207 | start_supervised( 208 | {JobBroadcaster, 209 | %StartOpts{ 210 | name: __MODULE__, 211 | jobs: init_jobs, 212 | storage: TestStorage, 213 | scheduler: TestScheduler, 214 | debug_logging: false 215 | }} 216 | ) 217 | 218 | {:ok, _consumer} = start_supervised({TestConsumer, [broadcaster, self()]}) 219 | 220 | TestScheduler.add_job(broadcaster, active_job) 221 | 222 | assert_receive {:received, {:add, ^active_job}} 223 | end) =~ "Adding job #Reference" 224 | end 225 | 226 | @tag listen_storage: true 227 | test "run", %{broadcaster: broadcaster, active_job: active_job} do 228 | test_id = "run-job-handler" 229 | 230 | :ok = attach_telemetry(:run, test_id, self()) 231 | 232 | TestScheduler.add_job(broadcaster, active_job) 233 | TestScheduler.run_job(broadcaster, active_job.name) 234 | 235 | assert_receive {:received, {:add, ^active_job}} 236 | assert_receive {:received, {:run, ^active_job}} 237 | end 238 | 239 | @tag listen_storage: true 240 | test "inactive", %{broadcaster: broadcaster, inactive_job: inactive_job} do 241 | test_id = "add-inactive-job-handler" 242 | 243 | :ok = attach_telemetry(:add, test_id, self()) 244 | 245 | capture_log(fn -> 246 | TestScheduler.add_job(broadcaster, inactive_job) 247 | 248 | refute_receive {:received, {:add, _}} 249 | 250 | assert_receive {:add_job, ^inactive_job, _} 251 | end) 252 | 253 | assert_receive %{test_id: ^test_id} 254 | end 255 | 256 | @tag listen_storage: true 257 | test "override active with active", %{broadcaster: broadcaster, test: test_name} do 258 | job_1 = 259 | TestScheduler.new_job() 260 | |> Quantum.Job.set_name(test_name) 261 | |> Quantum.Job.set_schedule(~e[*/5 * * * * *]e) 262 | 263 | TestScheduler.add_job(broadcaster, job_1) 264 | 265 | assert_receive {:received, {:add, ^job_1}} 266 | 267 | job_2 = 268 | TestScheduler.new_job() 269 | |> Quantum.Job.set_name(test_name) 270 | |> Quantum.Job.set_schedule(~e[*/10 * * * * *]e) 271 | 272 | TestScheduler.add_job(broadcaster, job_2) 273 | 274 | assert_receive {:received, {:remove, ^job_1}} 275 | assert_receive {:delete_job, ^test_name, _} 276 | assert_receive {:received, {:add, ^job_2}} 277 | assert_receive {:add_job, ^job_2, _} 278 | end 279 | 280 | @tag listen_storage: true, storage: :with_update 281 | test "storage with update does not add and delete job", %{ 282 | broadcaster: broadcaster, 283 | test: test_name 284 | } do 285 | job_1 = 286 | TestScheduler.new_job() 287 | |> Quantum.Job.set_name(test_name) 288 | |> Quantum.Job.set_schedule(~e[*/5 * * * * *]e) 289 | 290 | TestScheduler.add_job(broadcaster, job_1) 291 | 292 | assert_receive {:received, {:add, ^job_1}} 293 | assert_receive {:add_job, ^job_1, _} 294 | 295 | job_2 = 296 | TestScheduler.new_job() 297 | |> Quantum.Job.set_name(test_name) 298 | |> Quantum.Job.set_schedule(~e[*/10 * * * * *]e) 299 | 300 | TestScheduler.add_job(broadcaster, job_2) 301 | 302 | assert_receive {:received, {:remove, ^job_1}} 303 | refute_receive {:delete_job, ^test_name, _} 304 | assert_receive {:received, {:add, ^job_2}} 305 | refute_receive {:add_job, ^job_2, _} 306 | assert_receive {:update_job, ^job_2, _} 307 | end 308 | 309 | @tag listen_storage: true 310 | test "override active with inactive", %{broadcaster: broadcaster, test: test_name} do 311 | job_1 = 312 | TestScheduler.new_job() 313 | |> Quantum.Job.set_name(test_name) 314 | |> Quantum.Job.set_schedule(~e[*/5 * * * * *]e) 315 | 316 | TestScheduler.add_job(broadcaster, job_1) 317 | 318 | assert_receive {:received, {:add, ^job_1}} 319 | 320 | job_2 = 321 | TestScheduler.new_job() 322 | |> Quantum.Job.set_name(test_name) 323 | |> Quantum.Job.set_schedule(~e[*/10 * * * * *]e) 324 | |> Quantum.Job.set_state(:inactive) 325 | 326 | TestScheduler.add_job(broadcaster, job_2) 327 | 328 | assert_receive {:received, {:remove, ^job_1}} 329 | refute_receive {:received, {:add, _}} 330 | end 331 | 332 | @tag listen_storage: true 333 | test "override inactive with active", %{broadcaster: broadcaster, test: test_name} do 334 | job_1 = 335 | TestScheduler.new_job() 336 | |> Quantum.Job.set_name(test_name) 337 | |> Quantum.Job.set_schedule(~e[*/5 * * * * *]e) 338 | |> Quantum.Job.set_state(:inactive) 339 | 340 | TestScheduler.add_job(broadcaster, job_1) 341 | 342 | refute_receive {:received, {:add, _}} 343 | 344 | job_2 = 345 | TestScheduler.new_job() 346 | |> Quantum.Job.set_name(test_name) 347 | |> Quantum.Job.set_schedule(~e[*/10 * * * * *]e) 348 | 349 | TestScheduler.add_job(broadcaster, job_2) 350 | 351 | refute_receive {:received, {:remove, _}} 352 | assert_receive {:received, {:add, ^job_2}} 353 | end 354 | 355 | @tag listen_storage: true 356 | test "override inactive with inactive", %{broadcaster: broadcaster, test: test_name} do 357 | job_1 = 358 | TestScheduler.new_job() 359 | |> Quantum.Job.set_name(test_name) 360 | |> Quantum.Job.set_schedule(~e[*/5 * * * * *]e) 361 | |> Quantum.Job.set_state(:inactive) 362 | 363 | TestScheduler.add_job(broadcaster, job_1) 364 | 365 | refute_receive {:received, {:add, _}} 366 | 367 | job_2 = 368 | TestScheduler.new_job() 369 | |> Quantum.Job.set_name(test_name) 370 | |> Quantum.Job.set_schedule(~e[*/10 * * * * *]e) 371 | |> Quantum.Job.set_state(:inactive) 372 | 373 | TestScheduler.add_job(broadcaster, job_2) 374 | 375 | refute_receive {:received, {:remove, _}} 376 | refute_receive {:received, {:add, _}} 377 | end 378 | end 379 | 380 | describe "delete" do 381 | @tag jobs: :active, listen_storage: true 382 | test "active", %{broadcaster: broadcaster, active_job: active_job} do 383 | active_job_name = active_job.name 384 | 385 | test_id = "log-delete-active-job-handler" 386 | 387 | :ok = attach_telemetry(:delete, test_id, self()) 388 | 389 | capture_log(fn -> 390 | TestScheduler.delete_job(broadcaster, active_job.name) 391 | 392 | assert_receive {:received, {:remove, ^active_job_name}} 393 | 394 | assert_receive {:delete_job, ^active_job_name, _} 395 | 396 | refute Enum.any?(TestScheduler.jobs(broadcaster), fn {key, _} -> 397 | key == active_job_name 398 | end) 399 | end) 400 | 401 | assert_receive %{test_id: ^test_id, type: :delete} 402 | end 403 | 404 | @tag listen_storage: true 405 | test "missing", %{broadcaster: broadcaster} do 406 | capture_log(fn -> 407 | TestScheduler.delete_job(broadcaster, make_ref()) 408 | 409 | refute_receive {:received, {:remove, _}} 410 | 411 | refute_receive {:delete_job, {TestScheduler, _}, _} 412 | end) 413 | end 414 | 415 | @tag jobs: :inactive, listen_storage: true 416 | test "inactive", %{broadcaster: broadcaster, inactive_job: inactive_job} do 417 | test_id = "delete-inactive-job-handler" 418 | 419 | :ok = attach_telemetry(:delete, test_id, self()) 420 | 421 | capture_log(fn -> 422 | inactive_job_name = inactive_job.name 423 | 424 | TestScheduler.delete_job(broadcaster, inactive_job.name) 425 | 426 | refute_receive {:received, {:remove, _}} 427 | 428 | assert_receive {:delete_job, ^inactive_job_name, _} 429 | 430 | refute Enum.any?(TestScheduler.jobs(broadcaster), fn {key, _} -> 431 | key == inactive_job.name 432 | end) 433 | end) 434 | 435 | assert_receive %{test_id: ^test_id, type: :delete} 436 | end 437 | end 438 | 439 | describe "change_state" do 440 | @tag jobs: :active, listen_storage: true 441 | test "active => inactive", %{broadcaster: broadcaster, active_job: active_job} do 442 | active_job_name = active_job.name 443 | 444 | test_id = "update-active-to-inactive-job-handler" 445 | 446 | :ok = attach_telemetry(:update, test_id, self()) 447 | 448 | capture_log(fn -> 449 | TestScheduler.deactivate_job(broadcaster, active_job.name) 450 | 451 | assert_receive {:received, {:remove, ^active_job_name}} 452 | 453 | assert_receive {:update_job_state, {_, _}, _} 454 | end) 455 | 456 | assert_receive %{test_id: ^test_id} 457 | end 458 | 459 | @tag jobs: :inactive, listen_storage: true 460 | test "inactive => active", %{broadcaster: broadcaster, inactive_job: inactive_job} do 461 | test_id = "update-inactive-to-active-job-handler" 462 | 463 | :ok = attach_telemetry(:update, test_id, self()) 464 | 465 | capture_log(fn -> 466 | TestScheduler.activate_job(broadcaster, inactive_job.name) 467 | 468 | active_job = Job.set_state(inactive_job, :active) 469 | 470 | assert_receive {:received, {:add, ^active_job}} 471 | 472 | assert_receive {:update_job_state, {_, _}, _} 473 | end) 474 | 475 | assert_receive %{test_id: ^test_id} 476 | end 477 | 478 | @tag jobs: :active, listen_storage: true 479 | test "active => active", %{broadcaster: broadcaster, active_job: active_job} do 480 | test_id = "update-active-to-active-job-handler" 481 | 482 | :ok = attach_telemetry(:update, test_id, self()) 483 | 484 | # Initial 485 | assert_receive {:received, {:add, ^active_job}} 486 | name = active_job.name 487 | 488 | capture_log(fn -> 489 | TestScheduler.activate_job(broadcaster, name) 490 | 491 | refute_receive {:received, {:add, ^active_job}} 492 | 493 | refute_receive {:update_job_state, {TestScheduler, _, _}, _} 494 | end) 495 | 496 | refute_receive %{test_id: ^test_id, job_name: ^name} 497 | end 498 | 499 | @tag jobs: :inactive, listen_storage: true 500 | test "inactive => inactive", %{broadcaster: broadcaster, inactive_job: inactive_job} do 501 | test_id = "update-inactive-to-inactive-job-handler" 502 | 503 | :ok = attach_telemetry(:update, test_id, self()) 504 | 505 | inactive_job_name = inactive_job.name 506 | 507 | capture_log(fn -> 508 | TestScheduler.deactivate_job(broadcaster, inactive_job_name) 509 | 510 | refute_receive {:received, {:remove, ^inactive_job_name}} 511 | 512 | refute_receive {:update_job_state, {TestScheduler, _, _}, _} 513 | end) 514 | 515 | refute_receive %{test_id: ^test_id, job_name: ^inactive_job_name} 516 | end 517 | 518 | @tag listen_storage: true 519 | test "missing", %{broadcaster: broadcaster} do 520 | test_id = "update-missing-job-handler" 521 | 522 | :ok = attach_telemetry(:update, test_id, self()) 523 | 524 | ref1 = make_ref() 525 | ref2 = make_ref() 526 | 527 | capture_log(fn -> 528 | TestScheduler.deactivate_job(broadcaster, ref1) 529 | TestScheduler.activate_job(broadcaster, ref2) 530 | 531 | refute_receive {:received, {:remove, _}} 532 | refute_receive {:received, {:add, _}} 533 | refute_receive {:update_job_state, {TestScheduler, _, _}, _} 534 | end) 535 | 536 | refute_receive %{test_id: ^test_id, job_name: ^ref1} 537 | refute_receive %{test_id: ^test_id, job_name: ^ref2} 538 | end 539 | end 540 | 541 | describe "delete_all" do 542 | @tag jobs: :both, listen_storage: true 543 | test "only active jobs", %{ 544 | broadcaster: broadcaster, 545 | active_job: active_job, 546 | inactive_job: inactive_job 547 | } do 548 | test_id = "delete-all-active-jobs-handler" 549 | 550 | :ok = attach_telemetry(:delete, test_id, self()) 551 | 552 | active_job_name = active_job.name 553 | inactive_job_name = inactive_job.name 554 | 555 | capture_log(fn -> 556 | TestScheduler.delete_all_jobs(broadcaster) 557 | 558 | refute_receive {:received, {:remove, ^inactive_job_name}} 559 | assert_receive {:received, {:remove, ^active_job_name}} 560 | 561 | assert_receive {:purge, _, _} 562 | end) 563 | 564 | assert_receive %{test_id: ^test_id, job_name: ^inactive_job_name, type: :delete} 565 | assert_receive %{test_id: ^test_id, job_name: ^active_job_name, type: :delete} 566 | end 567 | end 568 | 569 | describe "jobs" do 570 | @tag jobs: :both 571 | test "gets all jobs", %{ 572 | broadcaster: broadcaster, 573 | active_job: active_job, 574 | inactive_job: inactive_job 575 | } do 576 | active_job_name = active_job.name 577 | inactive_job_name = inactive_job.name 578 | 579 | assert [{^active_job_name, %Job{}}, {^inactive_job_name, %Job{}}] = 580 | TestScheduler.jobs(broadcaster) 581 | end 582 | end 583 | 584 | describe "find_job" do 585 | @tag jobs: :active 586 | test "finds correct one", %{broadcaster: broadcaster, active_job: active_job} do 587 | active_job_name = active_job.name 588 | 589 | assert active_job == TestScheduler.find_job(broadcaster, active_job_name) 590 | end 591 | end 592 | end 593 | --------------------------------------------------------------------------------