├── .tool-versions ├── apps ├── gust │ ├── README.md │ ├── lib │ │ ├── gust │ │ │ ├── mailer.ex │ │ │ ├── dag │ │ │ │ ├── cron.ex │ │ │ │ ├── graph │ │ │ │ │ └── cycle_dectection.ex │ │ │ │ ├── error_parser.ex │ │ │ │ ├── task_delayer.ex │ │ │ │ ├── scheduler.ex │ │ │ │ ├── logger.ex │ │ │ │ ├── task_delayer │ │ │ │ │ └── calculator.ex │ │ │ │ ├── runner_supervisor.ex │ │ │ │ ├── compiler.ex │ │ │ │ ├── stage_runner_supervisor.ex │ │ │ │ ├── loader.ex │ │ │ │ ├── task_runner_supervisor.ex │ │ │ │ ├── terminator.ex │ │ │ │ ├── parser.ex │ │ │ │ ├── terminator │ │ │ │ │ └── process.ex │ │ │ │ ├── runner_supervisor │ │ │ │ │ └── dynamic_supervisor.ex │ │ │ │ ├── stage_runner_supervisor │ │ │ │ │ └── dynamic_supervisor.ex │ │ │ │ ├── task_runner_supervisor │ │ │ │ │ └── dynamic_supervisor.ex │ │ │ │ ├── run_restarter.ex │ │ │ │ ├── definition.ex │ │ │ │ ├── compiler │ │ │ │ │ └── code.ex │ │ │ │ ├── scheduler │ │ │ │ │ └── worker.ex │ │ │ │ ├── stage_coordinator.ex │ │ │ │ ├── logger │ │ │ │ │ └── database.ex │ │ │ │ ├── runner │ │ │ │ │ ├── task_worker.ex │ │ │ │ │ └── dag_worker.ex │ │ │ │ ├── graph.ex │ │ │ │ ├── stage_coordinator │ │ │ │ │ └── retrying_runner.ex │ │ │ │ ├── loader │ │ │ │ │ └── worker.ex │ │ │ │ ├── parser │ │ │ │ │ └── file.ex │ │ │ │ └── run_restarter │ │ │ │ │ └── worker.ex │ │ │ ├── repo.ex │ │ │ ├── encrypted │ │ │ │ └── binary.ex │ │ │ ├── file_monitor.ex │ │ │ ├── vault.ex │ │ │ ├── flows │ │ │ │ ├── log.ex │ │ │ │ ├── dag.ex │ │ │ │ ├── run.ex │ │ │ │ ├── secret.ex │ │ │ │ └── task.ex │ │ │ ├── release.ex │ │ │ ├── file_monitor │ │ │ │ ├── system_fs.ex │ │ │ │ └── worker.ex │ │ │ ├── pub_sub.ex │ │ │ └── dsl.ex │ │ └── gust.ex │ ├── priv │ │ └── repo │ │ │ ├── migrations │ │ │ ├── .formatter.exs │ │ │ ├── 20251031173208_add_task_error.exs │ │ │ ├── 20250815190135_create_runs.exs │ │ │ ├── 20250806203011_create_dags.exs │ │ │ ├── 20251026200057_create_secrets.exs │ │ │ ├── 20250930125654_create_logs.exs │ │ │ └── 20250815205059_create_tasks.exs │ │ │ └── seeds.exs │ ├── .formatter.exs │ ├── coveralls.json │ ├── test │ │ ├── flows │ │ │ ├── run_test.exs │ │ │ ├── dag_test.exs │ │ │ └── secret_test.exs │ │ ├── support │ │ │ ├── runner │ │ │ │ └── empty.ex │ │ │ ├── fs_helpers.ex │ │ │ ├── fixtures │ │ │ │ └── flows_fixtures.ex │ │ │ └── data_case.ex │ │ ├── dag │ │ │ ├── graph │ │ │ │ └── cycle_dectection_test.exs │ │ │ ├── task_delayer │ │ │ │ └── calculator_test.exs │ │ │ ├── error_parser_test.exs │ │ │ ├── definition_test.exs │ │ │ ├── runner_supervisor │ │ │ │ └── dynamic_supervisor_test.exs │ │ │ ├── stage_runner_supervisor │ │ │ │ └── dynamic_supervisor_test.exs │ │ │ ├── task_runner_supervisor │ │ │ │ └── dynamic_supervisor_test.exs │ │ │ ├── compiler │ │ │ │ └── code_test.exs │ │ │ ├── terminator │ │ │ │ └── process_test.exs │ │ │ ├── graph_test.exs │ │ │ ├── scheduler │ │ │ │ └── worker_test.exs │ │ │ ├── logger │ │ │ │ └── database_test.exs │ │ │ └── runner │ │ │ │ └── task_worker_test.exs │ │ ├── pub_sub_test.exs │ │ ├── test_helper.exs │ │ └── dsl_test.exs │ ├── .gitignore │ └── mix.exs └── gust_web │ ├── priv │ ├── static │ │ ├── favicon.ico │ │ ├── images │ │ │ └── gust-logo.png │ │ └── robots.txt │ └── gettext │ │ ├── en │ │ └── LC_MESSAGES │ │ │ └── errors.po │ │ └── errors.pot │ ├── .formatter.exs │ ├── lib │ ├── gust_web │ │ ├── controllers │ │ │ ├── page_controller.ex │ │ │ ├── error_json.ex │ │ │ └── error_html.ex │ │ ├── mix │ │ │ └── tasks │ │ │ │ ├── gust.start.ex │ │ │ │ ├── gust.dags.boot.ex │ │ │ │ └── gust.ecto.migrate.ex │ │ ├── mermaid.ex │ │ ├── components │ │ │ ├── layouts │ │ │ │ └── root.html.heex │ │ │ ├── dag_run_components.ex │ │ │ └── layouts.ex │ │ ├── live │ │ │ ├── dag_live │ │ │ │ ├── index.html.heex │ │ │ │ └── index.ex │ │ │ ├── breadcrumbs_component.ex │ │ │ ├── run_live │ │ │ │ ├── index.ex │ │ │ │ └── index.html.heex │ │ │ ├── secret_live │ │ │ │ ├── index.ex │ │ │ │ └── index.html.heex │ │ │ └── dag_summary_component.ex │ │ ├── gettext.ex │ │ ├── application.ex │ │ ├── endpoint.ex │ │ ├── router.ex │ │ └── telemetry.ex │ └── gust_web.ex │ ├── test │ ├── gust_web │ │ ├── controllers │ │ │ ├── page_controller_test.exs │ │ │ ├── error_json_test.exs │ │ │ └── error_html_test.exs │ │ ├── mermaid_test.exs │ │ └── live │ │ │ ├── breadcrumbs_live_component_test.exs │ │ │ ├── run_live_test.exs │ │ │ ├── dag_summary_live_component_test.exs │ │ │ ├── secret_live_test.exs │ │ │ └── dag_live_test.exs │ ├── test_helper.exs │ └── support │ │ ├── conn_case.ex │ │ └── live_component_tests.ex │ ├── coveralls.json │ ├── assets │ ├── package.json │ ├── tsconfig.json │ ├── vendor │ │ └── heroicons.js │ └── js │ │ └── app.js │ ├── README.md │ ├── .gitignore │ └── mix.exs ├── .formatter.exs ├── Makefile ├── config ├── prod.exs ├── test.exs ├── config.exs ├── dev.exs └── runtime.exs ├── .gitignore ├── LICENSE ├── .github └── workflows │ ├── publish.yml │ └── test.yml ├── mix.exs └── README.md /.tool-versions: -------------------------------------------------------------------------------- 1 | elixir 1.18.4 2 | -------------------------------------------------------------------------------- /apps/gust/README.md: -------------------------------------------------------------------------------- 1 | # Gust 2 | 3 | **TODO: Add description** 4 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/mailer.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Mailer do 2 | use Swoosh.Mailer, otp_app: :gust 3 | end 4 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [:ecto_sql], 3 | inputs: ["*.exs"] 4 | ] 5 | -------------------------------------------------------------------------------- /apps/gust_web/priv/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marciok/gust/HEAD/apps/gust_web/priv/static/favicon.ico -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/cron.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Cron do 2 | @moduledoc false 3 | use Quantum, otp_app: :gust 4 | end 5 | -------------------------------------------------------------------------------- /apps/gust_web/priv/static/images/gust-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/marciok/gust/HEAD/apps/gust_web/priv/static/images/gust-logo.png -------------------------------------------------------------------------------- /apps/gust/lib/gust/repo.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo do 2 | use Ecto.Repo, 3 | otp_app: :gust, 4 | adapter: Ecto.Adapters.Postgres 5 | end 6 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | plugins: [Phoenix.LiveView.HTMLFormatter], 3 | inputs: ["mix.exs", "config/*.exs"], 4 | subdirectories: ["apps/*"] 5 | ] 6 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/graph/cycle_dectection.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Graph.CycleDection do 2 | defexception message: "Possible cycle detected" 3 | end 4 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/encrypted/binary.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Encrypted.Binary do 2 | @moduledoc false 3 | use Cloak.Ecto.Binary, vault: Gust.Vault 4 | end 5 | -------------------------------------------------------------------------------- /apps/gust_web/.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [:phoenix], 3 | plugins: [Phoenix.LiveView.HTMLFormatter], 4 | inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}"] 5 | ] 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | dev: 3 | mix gust.start 4 | 5 | test: 6 | mix test 7 | 8 | test-cover: 9 | MIX_ENV=test mix coveralls.html --umbrella 10 | 11 | lint: 12 | mix lint 13 | 14 | console: 15 | iex -S mix 16 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/controllers/page_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.PageController do 2 | @moduledoc false 3 | use GustWeb, :controller 4 | 5 | def home(conn, _params) do 6 | redirect(conn, to: ~p"/dags") 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /apps/gust_web/priv/static/robots.txt: -------------------------------------------------------------------------------- 1 | # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | User-agent: * 5 | Disallow: / 6 | -------------------------------------------------------------------------------- /apps/gust/.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [:ecto, :ecto_sql], 3 | subdirectories: ["priv/*/migrations"], 4 | plugins: [Phoenix.LiveView.HTMLFormatter], 5 | inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"] 6 | ] 7 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/error_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.ErrorParser do 2 | @moduledoc false 3 | def parse(error) do 4 | %{ 5 | type: inspect(error.__struct__), 6 | message: Exception.message(error) 7 | } 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/controllers/page_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.PageControllerTest do 2 | use GustWeb.ConnCase 3 | 4 | test "GET /", %{conn: conn} do 5 | conn = get(conn, ~p"/") 6 | assert redirected_to(conn) == ~p"/dags" 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /apps/gust/coveralls.json: -------------------------------------------------------------------------------- 1 | { 2 | "coverage_options": { 3 | "minimum_coverage": 98 4 | }, 5 | "skip_files": [ 6 | "test/support", 7 | "lib/gust.ex", 8 | "lib/gust/mailer.ex", 9 | "lib/gust/release.ex", 10 | "lib/gust/repo.ex" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/20251031173208_add_task_error.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo.Migrations.AddTaskError do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tasks) do 6 | add :error, :jsonb, default: "{}", null: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust/test/flows/run_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.RunTest do 2 | use ExUnit.Case, async: true 3 | alias Gust.Flows.Run 4 | 5 | test "rejects creation without dag_id" do 6 | changeset = Run.changeset(%Run{}, %{}) 7 | refute changeset.valid? 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust/test/support/runner/empty.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Runner.Empty do 2 | @moduledoc false 3 | use GenServer 4 | 5 | def init(init_arg) do 6 | {:ok, init_arg} 7 | end 8 | 9 | def start_link(arg) do 10 | GenServer.start_link(__MODULE__, arg) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust/test/dag/graph/cycle_dectection_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.Graph.CycleDectectionTest do 2 | alias Gust.DAG.Graph.CycleDection 3 | use Gust.DataCase 4 | 5 | test "error message" do 6 | assert_raise(CycleDection, "Possible cycle detected", fn -> raise CycleDection end) 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /apps/gust/test/dag/task_delayer/calculator_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.TaskDelayer.CalculatorTest do 2 | alias Gust.DAG.TaskDelayer.Calculator 3 | use Gust.DataCase 4 | 5 | test "calc_delay/1" do 6 | attempt = 3 7 | assert 20_000 = Calculator.calc_delay(attempt) 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust/lib/gust.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust do 2 | @moduledoc """ 3 | Gust keeps the contexts that define your domain 4 | and business logic. 5 | 6 | Contexts are also responsible for managing your data, regardless 7 | if it comes from the database, an external API or others. 8 | """ 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/task_delayer.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.TaskDelayer do 2 | @moduledoc false 3 | @callback calc_delay(attempt :: integer()) :: integer() 4 | 5 | def calc_delay(attempt), do: impl().calc_delay(attempt) 6 | def impl, do: Application.get_env(:gust, :dag_task_delayer, Gust.DAG.TaskDelayer.Calculator) 7 | end 8 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/scheduler.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Scheduler do 2 | @moduledoc false 3 | 4 | @callback schedule(%{integer() => {:ok, Gust.DAG.Definition.t()} | {:error, term()}}) :: :ok 5 | 6 | def schedule(dag_defs), do: impl().schedule(dag_defs) 7 | 8 | defp impl, do: Application.get_env(:gust, :dag_scheduler) 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/mix/tasks/gust.start.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Gust.Start do 2 | @moduledoc false 3 | use Mix.Task 4 | 5 | # coveralls-ignore-start 6 | @impl Mix.Task 7 | def run(_args) do 8 | Mix.Task.run("gust.dags.boot") 9 | Mix.Task.run("phx.server") 10 | end 11 | 12 | # coveralls-ignore-stop 13 | end 14 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/20250815190135_create_runs.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo.Migrations.CreateRuns do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:runs) do 6 | add :dag_id, references(:dags, on_delete: :delete_all), null: false 7 | add :status, :string 8 | 9 | timestamps(type: :utc_datetime) 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/logger.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Logger do 2 | @moduledoc false 3 | @callback set_task(task_id :: integer(), attempt :: integer()) :: nil 4 | @callback unset() :: nil 5 | 6 | def set_task(task_id, attempt), do: impl().set_task(task_id, attempt) 7 | def unset, do: impl().unset() 8 | 9 | def impl, do: Application.get_env(:gust, :dag_logger) 10 | end 11 | -------------------------------------------------------------------------------- /apps/gust/test/support/fs_helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.FSHelpers do 2 | @moduledoc false 3 | def make_rand_dir!(prefix) do 4 | base = System.tmp_dir!() 5 | 6 | uniq = 7 | "#{prefix}_#{System.monotonic_time()}_#{System.unique_integer([:positive, :monotonic])}" 8 | 9 | path = Path.join(base, uniq) 10 | File.mkdir_p!(path) 11 | path 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/mix/tasks/gust.dags.boot.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Gust.Dags.Boot do 2 | @moduledoc false 3 | use Mix.Task 4 | require Logger 5 | 6 | # coveralls-ignore-start 7 | @impl Mix.Task 8 | def run(_args) do 9 | Logger.info("Booting DAGs") 10 | Application.put_env(:gust, :boot_dag, true) 11 | end 12 | 13 | # coveralls-ignore-stop 14 | end 15 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/task_delayer/calculator.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.TaskDelayer.Calculator do 2 | @moduledoc false 3 | @behaviour Gust.DAG.TaskDelayer 4 | 5 | def calc_delay(attempt) do 6 | delay = 5_000 7 | exp_backoff(delay, attempt) 8 | end 9 | 10 | defp exp_backoff(delay, attempt) do 11 | (delay * :math.pow(2, attempt - 1)) |> round() 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /apps/gust_web/coveralls.json: -------------------------------------------------------------------------------- 1 | { 2 | "coverage_options": { 3 | "minimum_coverage": 98 4 | }, 5 | "skip_files": [ 6 | "test/support", 7 | "lib/gust_web/telemetry.ex", 8 | "lib/gust_web.ex", 9 | "lib/gust_web/gettext.ex", 10 | "lib/gust_web/endpoint.ex", 11 | "lib/gust_web/components/core_components.ex", 12 | "lib/gust_web/application.ex" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /apps/gust_web/assets/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "assets", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "scripts": { 6 | "test": "echo \"Error: no test specified\" && exit 1" 7 | }, 8 | "keywords": [], 9 | "author": "", 10 | "license": "ISC", 11 | "description": "", 12 | "dependencies": { 13 | "mermaid": "^11.12.0", 14 | "prismjs": "^1.30.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/mix/tasks/gust.ecto.migrate.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Gust.Ecto.Migrate do 2 | @moduledoc false 3 | require Logger 4 | use Mix.Task 5 | 6 | # coveralls-ignore-start 7 | @impl Mix.Task 8 | def run(_args) do 9 | Logger.info("Migrating Gust") 10 | Mix.Task.run("mix ecto.migrate --repo Gust.Repo") 11 | end 12 | 13 | # coveralls-ignore-stop 14 | end 15 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/runner_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.RunnerSupervisor do 2 | @moduledoc false 3 | 4 | @callback start_child(Gust.Flows.Run.t(), Gust.DAG.Definition.t()) :: 5 | Supervisor.on_start_child() 6 | 7 | def start_child(run, dag_def), 8 | do: impl().start_child(run, dag_def) 9 | 10 | defp impl, 11 | do: Application.get_env(:gust, :dag_runner_supervisor) 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/file_monitor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.FileMonitor do 2 | @moduledoc false 3 | @callback start_link(keyword()) :: GenServer.on_start() 4 | @callback watch(GenServer.server()) :: :ok 5 | 6 | def watch(server_pid), do: impl().watch(server_pid) 7 | def start_link(args), do: impl().start_link(args) 8 | defp impl, do: Application.get_env(:gust, :file_monitor, Gust.FileMonitor.SystemFs) 9 | end 10 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/seeds.exs: -------------------------------------------------------------------------------- 1 | # Script for populating the database. You can run it as: 2 | # 3 | # mix run priv/repo/seeds.exs 4 | # 5 | # Inside the script, you can read and write to any of your 6 | # repositories directly: 7 | # 8 | # Gust.Repo.insert!(%Gust.SomeSchema{}) 9 | # 10 | # We recommend using the bang functions (`insert!`, `update!` 11 | # and so on) as they will fail if something goes wrong. 12 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/20250806203011_create_dags.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo.Migrations.CreateDags do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:dags) do 6 | add :name, :string, null: false 7 | add :enabled, :boolean, default: true, null: false 8 | 9 | timestamps(type: :utc_datetime) 10 | end 11 | 12 | create unique_index(:dags, [:name]) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/compiler.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Compiler do 2 | @moduledoc false 3 | @type dag_def :: Gust.DAG.Definition.t() 4 | 5 | @callback compile(dag_def) :: module() 6 | @callback purge(module()) :: nil 7 | 8 | def compile(dag_def), do: impl().compile(dag_def) 9 | def purge(mod), do: impl().purge(mod) 10 | 11 | defp impl, do: Application.get_env(:gust, :dag_compiler, Gust.DAG.Compiler.Code) 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/controllers/error_json_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.ErrorJSONTest do 2 | use GustWeb.ConnCase, async: true 3 | 4 | test "renders 404" do 5 | assert GustWeb.ErrorJSON.render("404.json", %{}) == %{errors: %{detail: "Not Found"}} 6 | end 7 | 8 | test "renders 500" do 9 | assert GustWeb.ErrorJSON.render("500.json", %{}) == 10 | %{errors: %{detail: "Internal Server Error"}} 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/stage_runner_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.StageRunnerSupervisor do 2 | @moduledoc false 3 | 4 | @callback start_child(Gust.DAG.Definition.t(), [term()], term()) :: 5 | Supervisor.on_start_child() 6 | 7 | def start_child(dag_def, stage, run_id), 8 | do: impl().start_child(dag_def, stage, run_id) 9 | 10 | defp impl, 11 | do: Application.get_env(:gust, :dag_stage_runner_supervisor) 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust/test/dag/error_parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Dag.ErrorParser do 2 | use Gust.DataCase 3 | import Gust.DAG.ErrorParser 4 | 5 | test "parse/1 when result does not have a reason" do 6 | error_msg = "ops.. something went wrong." 7 | error = %Ecto.Query.CastError{message: error_msg} 8 | 9 | assert parse(error) == %{ 10 | type: "Ecto.Query.CastError", 11 | message: error_msg 12 | } 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/loader.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Loader do 2 | @moduledoc false 3 | 4 | @callback get_definitions() :: %{term() => Gust.DAG.Definition.t()} 5 | @callback get_definition(term()) :: Gust.DAG.Definition.t() 6 | 7 | # coveralls-ignore-start 8 | def get_definitions, do: impl().get_definitions() 9 | def get_definition(dag_id), do: impl().get_definition(dag_id) 10 | defp impl, do: Application.get_env(:gust, :dag_loader) 11 | # coveralls-ignore-stop 12 | end 13 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/vault.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Vault do 2 | @moduledoc false 3 | use Cloak.Vault, otp_app: :gust 4 | 5 | @impl GenServer 6 | def init(config) do 7 | config = 8 | Keyword.put(config, :ciphers, 9 | default: {Cloak.Ciphers.AES.GCM, tag: "AES.GCM.V1", key: decode_env!()} 10 | ) 11 | 12 | {:ok, config} 13 | end 14 | 15 | defp decode_env! do 16 | Application.get_env(:gust, :b64_secrets_cloak_key) |> Base.decode64!() 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/20251026200057_create_secrets.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo.Migrations.CreateSecrets do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:secrets, primary_key: false) do 6 | add :id, :binary_id, primary_key: true 7 | add :name, :string, null: false 8 | add :value, :binary, null: false 9 | add :value_type, :string, null: false 10 | 11 | timestamps() 12 | end 13 | 14 | create unique_index(:secrets, [:name]) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/20250930125654_create_logs.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo.Migrations.CreateLogs do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:logs) do 6 | add :attempt, :integer 7 | add :level, :string 8 | add :content, :text 9 | add :timestamp, :utc_datetime_usec 10 | add :task_id, references(:tasks, on_delete: :delete_all) 11 | 12 | timestamps(type: :utc_datetime) 13 | end 14 | 15 | create index(:logs, [:task_id]) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/controllers/error_html_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.ErrorHTMLTest do 2 | use GustWeb.ConnCase, async: true 3 | 4 | # Bring render_to_string/4 for testing custom views 5 | import Phoenix.Template, only: [render_to_string: 4] 6 | 7 | test "renders 404.html" do 8 | assert render_to_string(GustWeb.ErrorHTML, "404", "html", []) == "Not Found" 9 | end 10 | 11 | test "renders 500.html" do 12 | assert render_to_string(GustWeb.ErrorHTML, "500", "html", []) == "Internal Server Error" 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/task_runner_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.TaskRunnerSupervisor do 2 | @moduledoc false 3 | 4 | @callback start_child(Gust.Flows.Task.t(), module(), pid(), term()) :: 5 | Supervisor.on_start_child() 6 | 7 | def start_child(task, mod, stage_pid, opts), do: impl().start_child(task, mod, stage_pid, opts) 8 | 9 | defp impl, 10 | do: 11 | Application.get_env( 12 | :gust, 13 | :dag_task_runner_supervisor, 14 | Gust.DAG.TaskRunnerSupervisor.DynamicSupervisor 15 | ) 16 | end 17 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/terminator.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Terminator do 2 | @moduledoc false 3 | @callback kill_task(task :: Gust.Flows.Task.t(), status :: atom()) :: any() 4 | @callback cancel_timer(task :: Gust.Flows.Task.t(), status :: atom()) :: any() 5 | 6 | # coveralls-ignore-start 7 | def kill_task(task, status), do: impl().kill_task(task, status) 8 | def cancel_timer(task, status), do: impl().cancel_timer(task, status) 9 | def impl, do: Application.get_env(:gust, :dag_terminator, Gust.DAG.Terminator.Process) 10 | # coveralls-ignore-stop 11 | end 12 | -------------------------------------------------------------------------------- /apps/gust/priv/repo/migrations/20250815205059_create_tasks.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Repo.Migrations.CreateTasks do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:tasks) do 6 | add :name, :string 7 | add :status, :string 8 | add :result, :jsonb, default: "{}", null: false 9 | add :run_id, references(:runs, on_delete: :delete_all), null: false 10 | add :attempt, :integer, default: 1 11 | 12 | timestamps(type: :utc_datetime) 13 | end 14 | 15 | create index(:tasks, [:run_id]) 16 | create index(:runs, [:dag_id]) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/flows/log.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.Log do 2 | @moduledoc false 3 | 4 | use Ecto.Schema 5 | import Ecto.Changeset 6 | 7 | schema "logs" do 8 | field :level, :string 9 | field :attempt, :integer 10 | field :content, :string 11 | field :timestamp, :utc_datetime_usec 12 | belongs_to :task, Gust.Flows.Task 13 | 14 | timestamps(type: :utc_datetime) 15 | end 16 | 17 | @doc false 18 | def changeset(log, attrs) do 19 | log 20 | |> cast(attrs, [:task_id, :attempt, :level, :content]) 21 | |> validate_required([:task_id, :attempt, :level, :content]) 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/release.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Release do 2 | @moduledoc false 3 | @app :gust 4 | 5 | def migrate do 6 | load_app() 7 | 8 | for repo <- repos() do 9 | {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) 10 | end 11 | end 12 | 13 | def rollback(repo, version) do 14 | load_app() 15 | {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) 16 | end 17 | 18 | defp repos do 19 | Application.fetch_env!(@app, :ecto_repos) 20 | end 21 | 22 | defp load_app do 23 | Application.load(@app) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/mermaid.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Mermaid do 2 | @moduledoc false 3 | def chart(tasks) do 4 | tasks 5 | |> Enum.reduce("flowchart LR\n ", fn {name, %{upstream: upstream}}, flow_description -> 6 | upstream = upstream |> MapSet.to_list() 7 | lines = name |> build_lines(upstream) 8 | 9 | "#{flow_description}#{lines}" 10 | end) 11 | end 12 | 13 | defp build_lines(name, []) do 14 | "\n#{name}" 15 | end 16 | 17 | defp build_lines(name, upstream) do 18 | upstream 19 | |> Enum.reduce("", fn upstream_name, line -> 20 | "#{line}\n#{upstream_name} --> #{name}" 21 | end) 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Parser do 2 | @moduledoc false 3 | 4 | @callback parse(file_path :: String.t()) :: 5 | {:ok, module()} | {:error, term()} 6 | 7 | @callback parse_folder(folder :: String.t()) :: 8 | [{String.t(), {:ok, module()} | {:error, term()}}] 9 | 10 | @callback maybe_ex_file(path :: String.t()) :: String.t() | nil 11 | 12 | def parse(file_path), do: impl().parse(file_path) 13 | def parse_folder(folder), do: impl().parse_folder(folder) 14 | def maybe_ex_file(path), do: impl().maybe_ex_file(path) 15 | 16 | defp impl, do: Application.get_env(:gust, :dag_parser, Gust.DAG.Parser.File) 17 | end 18 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/components/layouts/root.html.heex: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | <.live_title default="Gust" suffix=" · GustFlow Framework"> 8 | {assigns[:page_title]} 9 | 10 | 11 | 13 | 14 | 15 | {@inner_content} 16 | 17 | 18 | -------------------------------------------------------------------------------- /apps/gust_web/README.md: -------------------------------------------------------------------------------- 1 | # GustWeb 2 | 3 | To start your Phoenix server: 4 | 5 | * Run `mix setup` to install and setup dependencies 6 | * Start Phoenix endpoint with `mix phx.server` 7 | 8 | Now you can visit [`localhost:4000`](http://localhost:4000) from your browser. 9 | 10 | Ready to run in production? Please [check our deployment guides](https://hexdocs.pm/phoenix/deployment.html). 11 | 12 | ## Learn more 13 | 14 | * Official website: https://www.phoenixframework.org/ 15 | * Guides: https://hexdocs.pm/phoenix/overview.html 16 | * Docs: https://hexdocs.pm/phoenix 17 | * Forum: https://elixirforum.com/c/phoenix-forum 18 | * Source: https://github.com/phoenixframework/phoenix 19 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/terminator/process.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Terminator.Process do 2 | @moduledoc false 3 | @behaviour Gust.DAG.Terminator 4 | 5 | alias Gust.Registry, as: GustReg 6 | 7 | def kill_task(task, status) do 8 | stage_pid = lookup("stage_run_#{task.run_id}") 9 | task_pid = lookup("task_#{task.id}") 10 | 11 | true = Process.exit(task_pid, :kill) 12 | 13 | send(stage_pid, {:task_result, nil, task.id, status}) 14 | end 15 | 16 | def cancel_timer(task, status) do 17 | stage_pid = lookup("stage_run_#{task.run_id}") 18 | send(stage_pid, {:cancel_timer, task.id, status}) 19 | end 20 | 21 | defp lookup(key) do 22 | [{pid, _val}] = Registry.lookup(GustReg, key) 23 | pid 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /apps/gust/.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Temporary files, for example, from tests. 23 | /tmp/ 24 | 25 | # Ignore package tarball (built via "mix hex.build"). 26 | gust-*.tar 27 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/controllers/error_json.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.ErrorJSON do 2 | @moduledoc """ 3 | This module is invoked by your endpoint in case of errors on JSON requests. 4 | 5 | See config/config.exs. 6 | """ 7 | 8 | # If you want to customize a particular status code, 9 | # you may add your own clauses, such as: 10 | # 11 | # def render("500.json", _assigns) do 12 | # %{errors: %{detail: "Internal Server Error"}} 13 | # end 14 | 15 | # By default, Phoenix returns the status message from 16 | # the template name. For example, "404.json" becomes 17 | # "Not Found". 18 | def render(template, _assigns) do 19 | %{errors: %{detail: Phoenix.Controller.status_message_from_template(template)}} 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/runner_supervisor/dynamic_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.RunnerSupervisor.DynamicSupervisor do 2 | @moduledoc false 3 | use DynamicSupervisor 4 | @behaviour Gust.DAG.RunnerSupervisor 5 | 6 | def start_link(init_arg) do 7 | DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) 8 | end 9 | 10 | @impl true 11 | def init(_init_arg) do 12 | DynamicSupervisor.init(strategy: :one_for_one) 13 | end 14 | 15 | @impl true 16 | def start_child(run, dag_def) do 17 | spec = {runner(), %{run: run, dag_def: dag_def}} 18 | DynamicSupervisor.start_child(__MODULE__, spec) 19 | end 20 | 21 | def runner do 22 | Application.get_env(:gust, :dag_runner, Gust.DAG.Runner.DAGWorker) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/dag_live/index.html.heex: -------------------------------------------------------------------------------- 1 | 2 |
3 |
8 | {dag.name} has syntax errors. 9 |
10 |
11 |
12 |
17 | <.live_component 18 | module={GustWeb.DagSummaryComponent} 19 | dag={dag} 20 | dag_def={dag_def} 21 | id={dag.id} 22 | /> 23 |
24 |
25 | 26 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/stage_runner_supervisor/dynamic_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.StageRunnerSupervisor.DynamicSupervisor do 2 | @moduledoc false 3 | alias Gust.DAG.StageRunnerSupervisor 4 | @behaviour StageRunnerSupervisor 5 | 6 | use DynamicSupervisor 7 | 8 | def start_link(init_arg) do 9 | DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) 10 | end 11 | 12 | @impl true 13 | def init(_init_arg) do 14 | DynamicSupervisor.init(strategy: :one_for_one) 15 | end 16 | 17 | @impl true 18 | def start_child(dag_def, stage, run_id) do 19 | spec = {runner(), %{stage: stage, dag_def: dag_def, run_id: run_id}} 20 | DynamicSupervisor.start_child(__MODULE__, spec) 21 | end 22 | 23 | def runner do 24 | Application.get_env(:gust, :dag_stage_runner) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/task_runner_supervisor/dynamic_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.TaskRunnerSupervisor.DynamicSupervisor do 2 | @moduledoc false 3 | 4 | @behaviour Gust.DAG.TaskRunnerSupervisor 5 | use DynamicSupervisor 6 | 7 | def start_link(init_arg) do 8 | DynamicSupervisor.start_link(__MODULE__, init_arg, name: __MODULE__) 9 | end 10 | 11 | @impl true 12 | def init(_init_arg) do 13 | DynamicSupervisor.init(strategy: :one_for_one) 14 | end 15 | 16 | @impl true 17 | def start_child(task, mod, stage_pid, opts) do 18 | spec = {runner(), %{task: task, mod: mod, stage_pid: stage_pid, opts: opts}} 19 | DynamicSupervisor.start_child(__MODULE__, spec) 20 | end 21 | 22 | def runner do 23 | Application.get_env(:gust, :dag_task_runner, Gust.DAG.Runner.TaskWorker) 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/controllers/error_html.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.ErrorHTML do 2 | @moduledoc """ 3 | This module is invoked by your endpoint in case of errors on HTML requests. 4 | 5 | See config/config.exs. 6 | """ 7 | use GustWeb, :html 8 | 9 | # If you want to customize your error pages, 10 | # uncomment the embed_templates/1 call below 11 | # and add pages to the error directory: 12 | # 13 | # * lib/gust_web/controllers/error_html/404.html.heex 14 | # * lib/gust_web/controllers/error_html/500.html.heex 15 | # 16 | # embed_templates "error_html/*" 17 | 18 | # The default is to render a plain text page based on 19 | # the template name. For example, "404.html" becomes 20 | # "Not Found". 21 | def render(template, _assigns) do 22 | Phoenix.Controller.status_message_from_template(template) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/file_monitor/system_fs.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.FileMonitor.SystemFs do 2 | @moduledoc false 3 | @behaviour Gust.FileMonitor 4 | 5 | # Note: To avoid compile-time warnings in environments where `:file_system` is not present 6 | # (e.g., `:prod`), we use `apply/3` instead of direct function calls. 7 | # 8 | # This ensures the compiler does not resolve `FileSystem.start_link/1` or 9 | # `FileSystem.subscribe/1` unless the module is actually loaded at runtime. 10 | 11 | # coveralls-ignore-start 12 | @impl true 13 | # credo:disable-for-next-line Credo.Check.Refactor.Apply 14 | def start_link(opts), do: apply(FileSystem, :start_link, [opts]) 15 | 16 | @impl true 17 | # credo:disable-for-next-line Credo.Check.Refactor.Apply 18 | def watch(server), do: apply(FileSystem, :subscribe, [server]) 19 | 20 | # coveralls-ignore-stop 21 | end 22 | -------------------------------------------------------------------------------- /apps/gust_web/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | Application.ensure_all_started(:mox) 2 | 3 | Mox.defmock(GustWeb.DAGLoaderMock, for: Gust.DAG.Loader) 4 | Mox.defmock(GustWeb.DAGParserMock, for: Gust.DAG.Parser) 5 | Mox.defmock(GustWeb.DAGRunnerSupervisorMock, for: Gust.DAG.RunnerSupervisor) 6 | Mox.defmock(GustWeb.DAGTerminatorMock, for: Gust.DAG.Terminator) 7 | Mox.defmock(GustWeb.DAGRunRestarterMock, for: Gust.DAG.RunRestarter) 8 | 9 | Application.put_env(:gust, :dag_parser, GustWeb.DAGParserMock) 10 | Application.put_env(:gust, :dag_runner_supervisor, GustWeb.DAGRunnerSupervisorMock) 11 | Application.put_env(:gust, :dag_loader, GustWeb.DAGLoaderMock) 12 | Application.put_env(:gust, :dag_run_restarter, GustWeb.DAGRunRestarterMock) 13 | Application.put_env(:gust, :dag_terminator, GustWeb.DAGTerminatorMock) 14 | 15 | ExUnit.start() 16 | Ecto.Adapters.SQL.Sandbox.mode(Gust.Repo, :manual) 17 | -------------------------------------------------------------------------------- /apps/gust/test/dag/definition_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.DefinitionTest do 2 | alias Gust.DAG.Definition 3 | use Gust.DataCase 4 | 5 | test "fields are present" do 6 | dfn = %Definition{} 7 | assert dfn.name == "" 8 | assert dfn.mod == nil 9 | assert dfn.task_list == [] 10 | assert dfn.stages == [] 11 | assert dfn.tasks == %{} 12 | assert dfn.error == %{} 13 | assert dfn.messages == [] 14 | assert dfn.file_path == "" 15 | assert dfn.options == [] 16 | 17 | assert Map.keys(dfn) |> Enum.sort() == 18 | [ 19 | :__struct__, 20 | :error, 21 | :file_path, 22 | :messages, 23 | :mod, 24 | :name, 25 | :options, 26 | :stages, 27 | :task_list, 28 | :tasks 29 | ] 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # Note we also include the path to a cache manifest 4 | # containing the digested version of static files. This 5 | # manifest is generated by the `mix phx.digest` task, 6 | # which you should run after static files are built and 7 | # before starting your production server. 8 | config :gust_web, GustWeb.Endpoint, 9 | url: [host: "example.com", port: 80], 10 | cache_static_manifest: "priv/static/cache_manifest.json" 11 | 12 | # Configures Swoosh API Client 13 | config :swoosh, :api_client, Swoosh.ApiClient.Req 14 | 15 | # Disable Swoosh Local Memory Storage 16 | config :swoosh, local: false 17 | 18 | # Do not print debug messages in production 19 | config :logger, level: :info 20 | 21 | config :gust_web, basic_auth: true 22 | 23 | # Runtime production configuration, including reading 24 | # of environment variables, is done on config/runtime.exs. 25 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/run_restarter.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.RunRestarter do 2 | @moduledoc false 3 | 4 | @callback start_dag(integer()) :: term() 5 | @callback restart_run(Gust.Flows.Run.t()) :: :ok 6 | @callback restart_task(map(), Gust.Flows.Task.t()) :: :ok 7 | @callback restart_dags(%{integer() => {:ok, Gust.DAG.Definition.t()} | {:error, term()}}) :: :ok 8 | @callback restart_enqueued(integer()) :: :ok 9 | 10 | # coveralls-ignore-start 11 | def start_dag(dag_id), do: impl().start_dag(dag_id) 12 | def restart_run(run), do: impl().restart_run(run) 13 | def restart_task(graph, task), do: impl().restart_task(graph, task) 14 | def restart_dags(dags), do: impl().restart_dags(dags) 15 | def restart_enqueued(dag), do: impl().restart_enqueued(dag) 16 | 17 | defp impl, do: Application.get_env(:gust, :dag_run_restarter, Gust.DAG.RunRestarter.Worker) 18 | # coveralls-ignore-stop 19 | end 20 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/definition.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Definition do 2 | @moduledoc false 3 | defstruct name: "", 4 | mod: nil, 5 | error: %{}, 6 | messages: [], 7 | task_list: [], 8 | stages: [], 9 | tasks: %{}, 10 | file_path: "", 11 | options: Keyword.new() 12 | 13 | @type t :: %__MODULE__{ 14 | name: String.t(), 15 | mod: module() | nil, 16 | error: map(), 17 | messages: list(), 18 | task_list: list(), 19 | stages: list(), 20 | tasks: map(), 21 | file_path: String.t(), 22 | options: keyword() 23 | } 24 | 25 | @doc """ 26 | Returns `true` if the given DAG definition has any errors, by checking that the `error` map is non-empty. 27 | """ 28 | def empty_errors?(%__MODULE__{error: error}), do: map_size(error) == 0 29 | end 30 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/gettext.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Gettext do 2 | @moduledoc """ 3 | A module providing Internationalization with a gettext-based API. 4 | 5 | By using [Gettext](https://hexdocs.pm/gettext), your module compiles translations 6 | that you can use in your application. To use this Gettext backend module, 7 | call `use Gettext` and pass it as an option: 8 | 9 | use Gettext, backend: GustWeb.Gettext 10 | 11 | # Simple translation 12 | gettext("Here is the string to translate") 13 | 14 | # Plural translation 15 | ngettext("Here is the string to translate", 16 | "Here are the strings to translate", 17 | 3) 18 | 19 | # Domain-based translation 20 | dgettext("errors", "Here is the error message to translate") 21 | 22 | See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage. 23 | """ 24 | use Gettext.Backend, otp_app: :gust_web 25 | end 26 | -------------------------------------------------------------------------------- /apps/gust/test/flows/dag_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.DagTest do 2 | use ExUnit.Case, async: true 3 | alias Gust.Flows.Dag 4 | 5 | describe "name format validation" do 6 | test "accepts valid names" do 7 | valid_names = ["some_name", "dag1", "abc_123"] 8 | 9 | for name <- valid_names do 10 | changeset = Dag.changeset(%Dag{}, %{name: name}) 11 | 12 | assert changeset.valid?, 13 | "Expected #{name} to be valid, but got errors: #{inspect(changeset.errors)}" 14 | end 15 | end 16 | 17 | test "rejects invalid names" do 18 | invalid_names = ["SomeName", "some name", "some-name", "some$name", ""] 19 | 20 | for name <- invalid_names do 21 | changeset = Dag.changeset(%Dag{}, %{name: name}) 22 | 23 | refute changeset.valid?, 24 | "Expected #{name} to be invalid" 25 | 26 | assert {:name, _} = hd(changeset.errors), "Expected error on :name for #{name}" 27 | end 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/flows/dag.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.Dag do 2 | @moduledoc false 3 | use Ecto.Schema 4 | import Ecto.Changeset 5 | 6 | schema "dags" do 7 | field :name, :string 8 | field :enabled, :boolean, default: true 9 | has_many :runs, Gust.Flows.Run 10 | 11 | timestamps(type: :utc_datetime) 12 | end 13 | 14 | @type t :: %__MODULE__{ 15 | id: integer() | nil, 16 | name: String.t() | nil, 17 | enabled: boolean(), 18 | runs: [Gust.Flows.Run.t()] | Ecto.Association.NotLoaded.t(), 19 | inserted_at: DateTime.t() | nil, 20 | updated_at: DateTime.t() | nil 21 | } 22 | 23 | @doc false 24 | def changeset(dag, attrs) do 25 | dag 26 | |> cast(attrs, [:name, :enabled]) 27 | |> validate_required([:name]) 28 | |> validate_format(:name, ~r/^[a-z0-9_]+$/, 29 | message: "must be lowercase, no spaces, only letters, numbers, and underscores" 30 | ) 31 | |> unique_constraint(:name) 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /apps/gust_web/.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Temporary files, for example, from tests. 23 | /tmp/ 24 | 25 | # Ignore package tarball (built via "mix hex.build"). 26 | gust_web-*.tar 27 | 28 | # Ignore assets that are produced by build tools. 29 | /priv/static/assets/ 30 | 31 | # Ignore digested assets cache. 32 | /priv/static/cache_manifest.json 33 | 34 | # In case you use Node.js/npm, you want to ignore these. 35 | npm-debug.log 36 | /assets/node_modules/ 37 | 38 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/application.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Application do 2 | # See https://hexdocs.pm/elixir/Application.html 3 | # for more information on OTP Applications 4 | @moduledoc false 5 | 6 | use Application 7 | 8 | @impl true 9 | def start(_type, _args) do 10 | children = [ 11 | GustWeb.Telemetry, 12 | # Start a worker by calling: GustWeb.Worker.start_link(arg) 13 | # {GustWeb.Worker, arg}, 14 | # Start to serve requests, typically the last entry 15 | GustWeb.Endpoint 16 | ] 17 | 18 | # See https://hexdocs.pm/elixir/Supervisor.html 19 | # for other strategies and supported options 20 | opts = [strategy: :one_for_one, name: GustWeb.Supervisor] 21 | Supervisor.start_link(children, opts) 22 | end 23 | 24 | # Tell Phoenix to update the endpoint configuration 25 | # whenever the application is updated. 26 | @impl true 27 | def config_change(changed, _new, removed) do 28 | GustWeb.Endpoint.config_change(changed, removed) 29 | :ok 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Temporary files, for example, from tests. 23 | /tmp/ 24 | 25 | .DS_Store 26 | 27 | dags/ 28 | 29 | .envrc 30 | .env 31 | .env.test 32 | .env.dev 33 | 34 | # Built assets (compiled/digested) 35 | priv/static/assets/ 36 | priv/static/cache_manifest.json 37 | apps/gust_web/priv/static/assets/ 38 | apps/gust_web/priv/static/cache_manifest.json 39 | 40 | robots-*.txt.gz 41 | robots-*.txt 42 | 43 | favicon-*.ico 44 | robots-*.txt 45 | gust-logo-*.png 46 | -------------------------------------------------------------------------------- /apps/gust/test/dag/runner_supervisor/dynamic_supervisor_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.RunnerSupervisor.DynamicSupervisorTest do 2 | use Gust.DataCase, async: true 3 | import Gust.FlowsFixtures 4 | alias Gust.DAG.Runner 5 | alias Gust.DAG.RunnerSupervisor.DynamicSupervisor, as: RunnerSupervisor 6 | 7 | test "start_child/3" do 8 | mod = MyPlainDag 9 | 10 | dag_def = %Gust.DAG.Definition{ 11 | mod: mod, 12 | stages: [["sublime"]] 13 | } 14 | 15 | dag = dag_fixture() 16 | run = run_fixture(%{dag_id: dag.id}) 17 | runner = Runner.Empty 18 | 19 | old = Application.get_env(:gust, :dag_runner) 20 | Application.put_env(:gust, :dag_runner, runner) 21 | on_exit(fn -> Application.put_env(:gust, :dag_runner, old) end) 22 | 23 | start_supervised!(RunnerSupervisor) 24 | 25 | {:ok, runner_pid} = 26 | RunnerSupervisor.start_child(run, dag_def) 27 | 28 | assert Process.alive?(runner_pid) 29 | 30 | assert [{_id, ^runner_pid, :worker, [^runner]}] = 31 | DynamicSupervisor.which_children(RunnerSupervisor) 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/compiler/code.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Compiler.Code do 2 | @moduledoc false 3 | @behaviour Gust.DAG.Compiler 4 | 5 | def compile(dag_def) do 6 | file_path = dag_def.file_path 7 | {:ok, ast} = Code.string_to_quoted(File.read!(file_path)) 8 | 9 | runtime_mod = Module.concat(["Gust", "Runner", "#{dag_def.mod}_#{random_udid()}"]) 10 | dag_ast = patch_module(ast, runtime_mod) 11 | 12 | {dag_module, _} = Code.compile_quoted(dag_ast, file_path) |> List.first() 13 | dag_module 14 | end 15 | 16 | def purge(mod) do 17 | :code.purge(mod) 18 | :code.delete(mod) 19 | end 20 | 21 | defp random_udid do 22 | timestamp = :os.system_time(:microsecond) 23 | random = :crypto.strong_rand_bytes(4) |> Base.encode16() 24 | "#{timestamp}-#{random}" 25 | end 26 | 27 | defp patch_module(ast, runtime_mod) do 28 | Macro.postwalk(ast, fn 29 | {:defmodule, meta, [_module_ast_node, block]} -> 30 | {:defmodule, meta, [Module.concat([runtime_mod]), block]} 31 | 32 | node -> 33 | node 34 | end) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /apps/gust/test/dag/stage_runner_supervisor/dynamic_supervisor_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.StageRunnerSupervisor.DynamicSupervisorTest do 2 | use Gust.DataCase, async: true 3 | alias Gust.DAG.Runner 4 | alias Gust.DAG.StageRunnerSupervisor.DynamicSupervisor, as: StageRunnerSupervisor 5 | 6 | test "start_child/3" do 7 | mod = MyPlainDag 8 | 9 | dag_def = %Gust.DAG.Definition{ 10 | mod: mod, 11 | stages: [["sublime"]] 12 | } 13 | 14 | runner = Runner.Empty 15 | old = Application.get_env(:gust, :dag_stage_runner) 16 | Application.put_env(:gust, :dag_stage_runner, runner) 17 | 18 | on_exit(fn -> Application.put_env(:gust, :dag_stage_runner, old) end) 19 | 20 | start_supervised!(StageRunnerSupervisor) 21 | task_id = "123" 22 | run_id = "321" 23 | 24 | {:ok, runner_pid} = 25 | StageRunnerSupervisor.start_child(dag_def, [task_id], run_id) 26 | 27 | assert Process.alive?(runner_pid) 28 | 29 | assert [{_id, ^runner_pid, :worker, [^runner]}] = 30 | DynamicSupervisor.which_children(StageRunnerSupervisor) 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /apps/gust_web/assets/tsconfig.json: -------------------------------------------------------------------------------- 1 | // This file is needed on most editors to enable the intelligent autocompletion 2 | // of LiveView's JavaScript API methods. You can safely delete it if you don't need it. 3 | // 4 | // Note: This file assumes a basic esbuild setup without node_modules. 5 | // We include a generic paths alias to deps to mimic how esbuild resolves 6 | // the Phoenix and LiveView JavaScript assets. 7 | // If you have a package.json in your project, you should remove the 8 | // paths configuration and instead add the phoenix dependencies to the 9 | // dependencies section of your package.json: 10 | // 11 | // { 12 | // ... 13 | // "dependencies": { 14 | // ..., 15 | // "phoenix": "../deps/phoenix", 16 | // "phoenix_html": "../deps/phoenix_html", 17 | // "phoenix_live_view": "../deps/phoenix_live_view" 18 | // } 19 | // } 20 | // 21 | // Feel free to adjust this configuration however you need. 22 | { 23 | "compilerOptions": { 24 | "baseUrl": ".", 25 | "paths": { 26 | "*": ["../deps/*"] 27 | }, 28 | "allowJs": true, 29 | "noEmit": true 30 | }, 31 | "include": ["js/**/*"] 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Marcio Klepacz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /apps/gust/test/support/fixtures/flows_fixtures.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.FlowsFixtures do 2 | @moduledoc """ 3 | This module defines test helpers for creating 4 | entities via the `Gust.Flows` context. 5 | """ 6 | 7 | def secret_fixture(attrs \\ %{}) do 8 | {:ok, secret} = 9 | attrs 10 | |> Enum.into(%{ 11 | name: "SOME_NAME", 12 | value: "some value", 13 | value_type: :string 14 | }) 15 | |> Gust.Flows.create_secret() 16 | 17 | secret 18 | end 19 | 20 | def dag_fixture(attrs \\ %{}) do 21 | {:ok, dag} = 22 | attrs 23 | |> Enum.into(%{ 24 | name: "some_name" 25 | }) 26 | |> Gust.Flows.create_dag() 27 | 28 | dag 29 | end 30 | 31 | def run_fixture(attrs \\ %{}) do 32 | {:ok, run} = 33 | Gust.Flows.create_test_run(attrs) 34 | 35 | run 36 | end 37 | 38 | def task_fixture(attrs \\ %{}) do 39 | {:ok, task} = 40 | Gust.Flows.create_test_task(attrs) 41 | 42 | task 43 | end 44 | 45 | def log_fixture(attrs \\ %{}) do 46 | {:ok, log} = 47 | Gust.Flows.create_log(attrs) 48 | 49 | log 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /apps/gust/test/dag/task_runner_supervisor/dynamic_supervisor_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.TaskRunnerSupervisor.DynamicSupervisorTest do 2 | use Gust.DataCase, async: true 3 | import Gust.FlowsFixtures 4 | alias Gust.DAG.Runner 5 | alias Gust.DAG.TaskRunnerSupervisor.DynamicSupervisor, as: TaskRunnerSupervisor 6 | 7 | test "start_child/4" do 8 | mod = MyPlainDag 9 | dag = dag_fixture() 10 | run = run_fixture(%{dag_id: dag.id}) 11 | task = task_fixture(%{run_id: run.id, name: "green_day"}) 12 | runner = Runner.Empty 13 | 14 | old = Application.get_env(:gust, :dag_task_runner) 15 | Application.put_env(:gust, :dag_task_runner, runner) 16 | on_exit(fn -> Application.put_env(:gust, :dag_task_runner, old) end) 17 | 18 | start_supervised!(TaskRunnerSupervisor) 19 | 20 | stage_pid = spawn(fn -> Process.sleep(100) end) 21 | 22 | {:ok, runner_pid} = 23 | TaskRunnerSupervisor.start_child(task, mod, stage_pid, %{}) 24 | 25 | assert Process.alive?(runner_pid) 26 | 27 | assert [{_id, ^runner_pid, :worker, [^runner]}] = 28 | DynamicSupervisor.which_children(TaskRunnerSupervisor) 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/breadcrumbs_component.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.BreadcrumbsComponent do 2 | @moduledoc false 3 | use GustWeb, :live_component 4 | 5 | def render(assigns) do 6 | ~H""" 7 |
8 | 33 |
34 | """ 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /apps/gust_web/test/support/conn_case.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.ConnCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | tests that require setting up a connection. 5 | 6 | Such tests rely on `Phoenix.ConnTest` and also 7 | import other functionality to make it easier 8 | to build common data structures and query the data layer. 9 | 10 | Finally, if the test case interacts with the database, 11 | we enable the SQL sandbox, so changes done to the database 12 | are reverted at the end of every test. If you are using 13 | PostgreSQL, you can even run database tests asynchronously 14 | by setting `use GustWeb.ConnCase, async: true`, although 15 | this option is not recommended for other databases. 16 | """ 17 | 18 | use ExUnit.CaseTemplate 19 | 20 | using do 21 | quote do 22 | # The default endpoint for testing 23 | @endpoint GustWeb.Endpoint 24 | 25 | use GustWeb, :verified_routes 26 | 27 | # Import conveniences for testing with connections 28 | import Plug.Conn 29 | import Phoenix.ConnTest 30 | import GustWeb.ConnCase 31 | end 32 | end 33 | 34 | setup tags do 35 | Gust.DataCase.setup_sandbox(tags) 36 | {:ok, conn: Phoenix.ConnTest.build_conn()} 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/scheduler/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Scheduler.Worker do 2 | @moduledoc false 3 | @behaviour Gust.DAG.Scheduler 4 | 5 | import Crontab.CronExpression 6 | alias Gust.DAG.Cron 7 | alias Gust.DAG.Definition 8 | use GenServer 9 | alias Quantum.Job, as: QJob 10 | 11 | def init(args) do 12 | {:ok, args} 13 | end 14 | 15 | def start_link(args) do 16 | GenServer.start_link(__MODULE__, args, name: __MODULE__) 17 | end 18 | 19 | def schedule(dag_defs) do 20 | GenServer.call(__MODULE__, {:load_dags, dag_defs}) 21 | end 22 | 23 | def handle_call({:load_dags, dag_defs}, _from, state) do 24 | jobs = 25 | for {dag_id, {:ok, dag_def}} <- dag_defs, 26 | schedule = dag_def.options[:schedule], 27 | schedule != nil, 28 | Definition.empty_errors?(dag_def) do 29 | add_dag_job(dag_def, dag_id) 30 | end 31 | 32 | {:reply, jobs, state} 33 | end 34 | 35 | def add_dag_job(dag_def, dag_id) do 36 | schedule = dag_def.options[:schedule] 37 | 38 | Cron.new_job() 39 | |> QJob.set_name(String.to_atom(dag_def.name)) 40 | |> QJob.set_schedule(~e[#{schedule}]) 41 | |> QJob.set_task({Gust.DAG.RunRestarter, :start_dag, [dag_id]}) 42 | |> Cron.add_job() 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/flows/run.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.Run do 2 | @moduledoc false 3 | 4 | use Ecto.Schema 5 | import Ecto.Changeset 6 | 7 | schema "runs" do 8 | belongs_to :dag, Gust.Flows.Dag 9 | 10 | field :status, Ecto.Enum, 11 | values: [:created, :running, :succeeded, :failed, :retrying, :enqueued], 12 | default: :created 13 | 14 | has_many :tasks, Gust.Flows.Task 15 | 16 | timestamps(type: :utc_datetime) 17 | end 18 | 19 | @type t :: %__MODULE__{ 20 | id: integer() | nil, 21 | dag_id: integer() | nil, 22 | status: :created | :running | :succeeded | :failed | :retrying | :enqueued, 23 | tasks: [Gust.Flows.Task.t()] | Ecto.Association.NotLoaded.t(), 24 | dag: Gust.Flows.Dag.t() | Ecto.Association.NotLoaded.t(), 25 | inserted_at: DateTime.t() | nil, 26 | updated_at: DateTime.t() | nil 27 | } 28 | 29 | @doc false 30 | def changeset(run, attrs) do 31 | run 32 | |> cast(attrs, [:dag_id, :status]) 33 | |> validate_required([:dag_id, :status]) 34 | end 35 | 36 | @doc false 37 | def test_changeset(run, attrs) do 38 | run 39 | |> cast(attrs, [:dag_id, :status, :inserted_at]) 40 | |> validate_required([:dag_id, :status]) 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /apps/gust/test/dag/compiler/code_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.Compiler.CodeTest do 2 | alias DAG.Compiler 3 | use Gust.DataCase 4 | alias Gust.DAG.Compiler 5 | alias Gust.DAG.Definition 6 | import Gust.FSHelpers 7 | 8 | @original_mod_name "TheOffspring" 9 | 10 | setup do 11 | content = """ 12 | defmodule #{@original_mod_name} do 13 | 14 | end 15 | """ 16 | 17 | dir = make_rand_dir!("dags") 18 | dag_name = "the_offspring" 19 | file_path = "#{dir}/#{dag_name}.ex" 20 | File.write!(file_path, content) 21 | 22 | dag_def = %Definition{ 23 | file_path: file_path, 24 | mod: Module.concat([@original_mod_name]) 25 | } 26 | 27 | updated_mod = Compiler.Code.compile(dag_def) 28 | %{updated_mod: updated_mod} 29 | end 30 | 31 | describe "compile/1" do 32 | test "module is compiled and available", %{updated_mod: updated_mod} do 33 | assert Code.ensure_loaded?(updated_mod) 34 | assert "Elixir.Gust.Runner." <> random_udid = to_string(updated_mod) 35 | assert [@original_mod_name, _run_udid] = String.split(random_udid, "_") 36 | end 37 | end 38 | 39 | describe "purge/1" do 40 | test "purge code", %{updated_mod: updated_mod} do 41 | Compiler.Code.purge(updated_mod) 42 | 43 | assert Code.ensure_loaded?(updated_mod) == false 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/stage_coordinator.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.StageCoordinator do 2 | @moduledoc false 3 | @type stage_spec :: map() 4 | @type task_id :: integer() 5 | @type run_id :: integer() 6 | @type task :: Gust.Flows.Task.t() 7 | @type ref :: reference() 8 | @type reason :: term() 9 | 10 | @callback new(list(task_id)) :: stage_spec 11 | @callback put_running(stage_spec, task_id) :: stage_spec 12 | @callback apply_task_result(stage_spec, task, atom()) :: 13 | {:continue, stage_spec} 14 | | {:finished, stage_spec} 15 | | {:reschedule, stage_spec, task, integer()} 16 | @callback update_restart_timer(stage_spec, task, ref) :: stage_spec 17 | @callback process_task(task, map()) :: 18 | :ok | :upstream_failed | :already_processed 19 | 20 | def put_running(stage_spec, task_id), do: impl().put_running(stage_spec, task_id) 21 | 22 | def new(pending_task_ids), 23 | do: impl().new(pending_task_ids) 24 | 25 | def process_task(task, tasks), 26 | do: impl().process_task(task, tasks) 27 | 28 | def apply_task_result(stage_spec, ref, reason), 29 | do: impl().apply_task_result(stage_spec, ref, reason) 30 | 31 | def update_restart_timer(coord, task, ref), 32 | do: impl().update_restart_timer(coord, task, ref) 33 | 34 | def impl, 35 | do: 36 | Application.get_env(:gust, :dag_stage_coordinator, Gust.DAG.StageCoordinator.RetryingRunner) 37 | end 38 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/logger/database.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Logger.Database do 2 | @moduledoc false 3 | alias Gust.Flows 4 | @behaviour Gust.DAG.Logger 5 | @behaviour :gen_event 6 | 7 | def init(__MODULE__) do 8 | {:ok, %{}} 9 | end 10 | 11 | def handle_event(:flush, state), do: {:ok, state} 12 | 13 | def handle_event({level, _gl, {Logger, msg, _ts, md}}, state) do 14 | if md[:task_id] do 15 | Task.start(fn -> handle_log(level, msg, md) end) 16 | end 17 | 18 | {:ok, state} 19 | end 20 | 21 | def handle_call({:configure, opts}, state) do 22 | {:ok, :ok, Map.merge(state, Map.new(opts))} 23 | end 24 | 25 | def set_task(task_id, attempt) do 26 | Logger.metadata(task_id: task_id, attempt: attempt) 27 | end 28 | 29 | def unset do 30 | Logger.reset_metadata() 31 | end 32 | 33 | defp handle_log(_level, "", md), 34 | do: create_log(:error, "nil or empty was logged!", md) 35 | 36 | defp handle_log(level, msg, md) when is_list(msg), 37 | do: create_log(level, Enum.join(msg, "; "), md) 38 | 39 | defp handle_log(level, msg, md) when is_binary(msg), 40 | do: create_log(level, msg, md) 41 | 42 | defp create_log(level, msg, md) do 43 | {:ok, log} = 44 | Flows.create_log(%{ 45 | task_id: md[:task_id], 46 | content: msg, 47 | attempt: md[:attempt], 48 | level: to_string(level) 49 | }) 50 | 51 | Gust.PubSub.broadcast_log(md[:task_id], log.id) 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/flows/secret.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.Secret do 2 | @moduledoc false 3 | use Ecto.Schema 4 | import Ecto.Changeset 5 | 6 | @primary_key {:id, :binary_id, autogenerate: true} 7 | @derive {Jason.Encoder, only: [:id, :name, :value_type, :inserted_at, :updated_at]} 8 | schema "secrets" do 9 | field :name, :string 10 | field :value, Gust.Encrypted.Binary, redact: true 11 | field :value_type, Ecto.Enum, values: [:string, :json] 12 | 13 | timestamps() 14 | end 15 | 16 | @doc false 17 | def changeset(secret, attrs) do 18 | secret 19 | |> cast(attrs, [:name, :value, :value_type]) 20 | |> validate_required([:name, :value, :value_type]) 21 | |> validate_format(:name, ~r/^[A-Z0-9_]+$/, message: "must be uppercase with underscores") 22 | |> unique_constraint(:name) 23 | |> validate_json_if_needed() 24 | end 25 | 26 | defp validate_json_if_needed(changeset) do 27 | case get_field(changeset, :value_type) do 28 | :json -> 29 | value = get_field(changeset, :value) 30 | validate_json(value, changeset) 31 | 32 | _ -> 33 | changeset 34 | end 35 | end 36 | 37 | defp validate_json(nil, changeset) do 38 | add_error(changeset, :value, "it cannot be empty") 39 | end 40 | 41 | defp validate_json(value, changeset) when is_bitstring(value) do 42 | case Jason.decode(value) do 43 | {:ok, _decoded} -> changeset 44 | {:error, _} -> add_error(changeset, :value, "must be valid JSON") 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /apps/gust_web/assets/vendor/heroicons.js: -------------------------------------------------------------------------------- 1 | const plugin = require("tailwindcss/plugin") 2 | const fs = require("fs") 3 | const path = require("path") 4 | 5 | module.exports = plugin(function({matchComponents, theme}) { 6 | let iconsDir = path.join(__dirname, "../../../../deps/heroicons/optimized") 7 | let values = {} 8 | let icons = [ 9 | ["", "/24/outline"], 10 | ["-solid", "/24/solid"], 11 | ["-mini", "/20/solid"], 12 | ["-micro", "/16/solid"] 13 | ] 14 | icons.forEach(([suffix, dir]) => { 15 | fs.readdirSync(path.join(iconsDir, dir)).forEach(file => { 16 | let name = path.basename(file, ".svg") + suffix 17 | values[name] = {name, fullPath: path.join(iconsDir, dir, file)} 18 | }) 19 | }) 20 | matchComponents({ 21 | "hero": ({name, fullPath}) => { 22 | let content = fs.readFileSync(fullPath).toString().replace(/\r?\n|\r/g, "") 23 | content = encodeURIComponent(content) 24 | let size = theme("spacing.6") 25 | if (name.endsWith("-mini")) { 26 | size = theme("spacing.5") 27 | } else if (name.endsWith("-micro")) { 28 | size = theme("spacing.4") 29 | } 30 | return { 31 | [`--hero-${name}`]: `url('data:image/svg+xml;utf8,${content}')`, 32 | "-webkit-mask": `var(--hero-${name})`, 33 | "mask": `var(--hero-${name})`, 34 | "mask-repeat": "no-repeat", 35 | "background-color": "currentColor", 36 | "vertical-align": "middle", 37 | "display": "inline-block", 38 | "width": size, 39 | "height": size 40 | } 41 | } 42 | }, {values}) 43 | }) 44 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/file_monitor/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.FileMonitor.Worker do 2 | @moduledoc false 3 | 4 | use GenServer 5 | alias Gust.DAG.Parser 6 | alias Gust.FileMonitor 7 | 8 | @impl true 9 | def init(%{dags_folder: folder, loader: loader}) do 10 | {:ok, watcher_pid} = FileMonitor.start_link(dirs: [folder], latency: 0) 11 | FileMonitor.watch(watcher_pid) 12 | events_queue = MapSet.new() 13 | 14 | {:ok, %{watcher_pid: watcher_pid, events_queue: events_queue, loader: loader}} 15 | end 16 | 17 | def start_link(args) do 18 | GenServer.start_link(__MODULE__, args) 19 | end 20 | 21 | @impl true 22 | def handle_info({:file_event, _watcher_pid, {path, _events}}, %{events_queue: queue} = state) do 23 | if MapSet.member?(queue, path) do 24 | {:noreply, state} 25 | else 26 | Process.send_after(self(), {:check_queue, path}, delay()) 27 | {:noreply, %{state | events_queue: MapSet.put(queue, path)}} 28 | end 29 | end 30 | 31 | def handle_info({:check_queue, path}, %{events_queue: queue, loader: loader} = state) do 32 | Parser.maybe_ex_file(path) |> broadcast_path(loader) 33 | 34 | {:noreply, %{state | events_queue: MapSet.delete(queue, path)}} 35 | end 36 | 37 | defp delay, do: Application.get_env(:gust, :file_reload_delay) 38 | 39 | defp broadcast_path(nil, _loader), do: nil 40 | 41 | defp broadcast_path(path, loader) do 42 | action = if File.exists?(path), do: "reload", else: "removed" 43 | dag_name = path |> Path.basename() |> Path.rootname() 44 | 45 | send(loader, {dag_name, Parser.parse(path), action}) 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /apps/gust/test/pub_sub_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.PubSubTest do 2 | use ExUnit.Case, async: true 3 | 4 | describe "subscribe_run/1 & broadcast_run_status/2" do 5 | test "delivers the expected message to the right topic" do 6 | run_id = "r-123" 7 | status = :running 8 | 9 | assert :ok == Gust.PubSub.subscribe_run(run_id) 10 | assert :ok == Gust.PubSub.broadcast_run_status(run_id, status) 11 | 12 | assert_receive {:dag, :run_status, %{run_id: ^run_id, status: ^status}} 13 | end 14 | 15 | test "does not leak messages to other run topics" do 16 | subscribed = "r-aaa" 17 | other = "r-bbb" 18 | 19 | assert :ok == Gust.PubSub.subscribe_run(subscribed) 20 | assert :ok == Gust.PubSub.broadcast_run_status(other, :finished) 21 | 22 | refute_receive _any, 50 23 | end 24 | end 25 | 26 | describe "subscribe_runs_for_dag/1 & broadcast_run_started/2" do 27 | test "delivers the expected message to the dag:scheduled topic" do 28 | dag_id = "d-42" 29 | run_id = "r-999" 30 | 31 | assert :ok == Gust.PubSub.subscribe_runs_for_dag(dag_id) 32 | assert :ok == Gust.PubSub.broadcast_run_started(dag_id, run_id) 33 | 34 | assert_receive {:dag, :run_started, %{run_id: ^run_id}} 35 | end 36 | 37 | test "does not leak messages to other dag:scheduled topics" do 38 | subscribed_dag = "d-a" 39 | other_dag = "d-b" 40 | 41 | assert :ok == Gust.PubSub.subscribe_runs_for_dag(subscribed_dag) 42 | assert :ok == Gust.PubSub.broadcast_run_started(other_dag, "r-x") 43 | 44 | refute_receive _any, 50 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # Configure your database 4 | # 5 | # The MIX_TEST_PARTITION environment variable can be used 6 | # to provide built-in test partitioning in CI environment. 7 | # Run `mix help test` for more information. 8 | config :gust, Gust.Repo, 9 | hostname: System.get_env("PGHOST"), 10 | username: System.get_env("PGUSER"), 11 | password: System.get_env("PGPASSWORD"), 12 | database: "gust_rc_test#{System.get_env("MIX_TEST_PARTITION")}", 13 | pool: Ecto.Adapters.SQL.Sandbox, 14 | pool_size: System.schedulers_online() * 2 15 | 16 | # We don't run a server during test. If one is required, 17 | # you can enable the server option below. 18 | config :gust_web, GustWeb.Endpoint, 19 | http: [ip: {127, 0, 0, 1}, port: 4002], 20 | secret_key_base: System.get_env("SECRET_KEY_BASE"), 21 | server: false 22 | 23 | # Print only warnings and errors during test 24 | config :logger, level: :warning 25 | 26 | # In test we don't send emails 27 | config :gust, Gust.Mailer, adapter: Swoosh.Adapters.Test 28 | 29 | # Disable swoosh api client as it is only required for production adapters 30 | config :swoosh, :api_client, false 31 | 32 | # Initialize plugs at runtime for faster test compilation 33 | config :phoenix, :plug_init_mode, :runtime 34 | 35 | # Enable helpful, but potentially expensive runtime checks 36 | config :phoenix_live_view, 37 | enable_expensive_runtime_checks: true 38 | 39 | config :gust, dag_runner_supervisor: Gust.DAGRunnerSupervisorMock 40 | config :gust, dag_task_runner_supervisor: Gust.DAGTaskRunnerSupervisorMock 41 | config :gust, file_reload_delay: 0 42 | config :gust, b64_secrets_cloak_key: System.get_env("B64_SECRETS_CLOAK_KEY") 43 | -------------------------------------------------------------------------------- /apps/gust/test/dag/terminator/process_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.Terminator.ProcessTest do 2 | import Gust.FlowsFixtures 3 | use Gust.DataCase 4 | alias Gust.DAG.Terminator.Process, as: Terminator 5 | 6 | test "kill_task/2" do 7 | dag = dag_fixture(%{name: "test_dag"}) 8 | run = run_fixture(%{dag_id: dag.id}) 9 | task = task_fixture(%{run_id: run.id, name: "test_task"}) 10 | task_id = task.id 11 | 12 | {:ok, _} = Registry.register(Gust.Registry, "stage_run_#{task.run_id}", nil) 13 | 14 | parent = self() 15 | 16 | spawn(fn -> 17 | {:ok, _} = Registry.register(Gust.Registry, "task_#{task.id}", nil) 18 | send(parent, :registered) 19 | Process.sleep(3_000) 20 | end) 21 | 22 | receive do 23 | :registered -> :ok 24 | after 25 | 100 -> flunk("Registry did not register in time") 26 | end 27 | 28 | [{task_pid, _val}] = Registry.lookup(Gust.Registry, "task_#{task.id}") 29 | ref = Process.monitor(task_pid) 30 | status = :cancelled 31 | 32 | Terminator.kill_task(task, status) 33 | 34 | assert_receive {:DOWN, ^ref, :process, ^task_pid, :killed}, 200 35 | 36 | assert_receive {:task_result, nil, ^task_id, ^status}, 200 37 | end 38 | 39 | test "cancel_timer/2" do 40 | dag = dag_fixture(%{name: "test_dag"}) 41 | run = run_fixture(%{dag_id: dag.id}) 42 | task = task_fixture(%{run_id: run.id, name: "test_task"}) 43 | task_id = task.id 44 | 45 | {:ok, _} = Registry.register(Gust.Registry, "stage_run_#{task.run_id}", nil) 46 | status = :cancelled 47 | 48 | Terminator.cancel_timer(task, status) 49 | 50 | assert_receive {:cancel_timer, ^task_id, ^status}, 200 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Hex Publish 2 | on: 3 | push: 4 | tags: 5 | - "v*.*.*" 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - uses: erlef/setup-beam@v1 15 | with: 16 | elixir-version: '1.18' 17 | otp-version: '27.0' 18 | 19 | - name: Set up Node.js 20 | uses: actions/setup-node@v3 21 | with: 22 | node-version: "18" 23 | 24 | - name: Install NPM packages 25 | working-directory: apps/gust_web/assets 26 | run: npm install 27 | 28 | - name: Gust-web assets deploy 29 | run: | 30 | mix deps.get 31 | mix assets.deploy 32 | 33 | - name: Archive built assets 34 | run: | 35 | mkdir -p release_assets 36 | cp -r apps/gust_web/priv/static release_assets/ 37 | cd release_assets 38 | tar -czf gust_static_assets.tar.gz static 39 | 40 | - name: Upload Release Asset 41 | uses: softprops/action-gh-release@v1 42 | with: 43 | files: release_assets/gust_static_assets.tar.gz 44 | token: ${{ secrets.PUBLISH_RELEASE_GH }} 45 | 46 | - name: Publish gust 47 | working-directory: apps/gust 48 | env: 49 | HEX_API_KEY: ${{ secrets.HEX }} 50 | PUBLISH_DEP: true 51 | run: | 52 | mix deps.get 53 | mix hex.build 54 | mix hex.publish --yes --replace 55 | 56 | - name: Publish gust-web 57 | working-directory: apps/gust_web 58 | env: 59 | HEX_API_KEY: ${{ secrets.HEX }} 60 | PUBLISH_DEP: true 61 | run: | 62 | mix deps.get 63 | mix assets.deploy 64 | mix hex.build 65 | mix hex.publish --yes --replace 66 | 67 | -------------------------------------------------------------------------------- /apps/gust/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | Application.ensure_all_started(:mox) 2 | Mox.defmock(Gust.DAGLoaderMock, for: Gust.DAG.Loader) 3 | Mox.defmock(Gust.FileMonitorMock, for: Gust.FileMonitor) 4 | Mox.defmock(Gust.DAGParserMock, for: Gust.DAG.Parser) 5 | Mox.defmock(Gust.DAGSchedulerMock, for: Gust.DAG.Scheduler) 6 | Mox.defmock(Gust.DAGRunnerSupervisorMock, for: Gust.DAG.RunnerSupervisor) 7 | Mox.defmock(Gust.DAGTaskRunnerSupervisorMock, for: Gust.DAG.TaskRunnerSupervisor) 8 | Mox.defmock(Gust.DAGStageRunnerSupervisorMock, for: Gust.DAG.StageRunnerSupervisor) 9 | Mox.defmock(Gust.DAGStageCoordinatorMock, for: Gust.DAG.StageCoordinator) 10 | Mox.defmock(Gust.DAGCompilerMock, for: Gust.DAG.Compiler) 11 | Mox.defmock(Gust.DAGLoggerMock, for: Gust.DAG.Logger) 12 | Mox.defmock(Gust.DAGRunRestarterMock, for: Gust.DAG.RunRestarter) 13 | Mox.defmock(Gust.DAGTaskDelayerMock, for: Gust.DAG.TaskDelayer) 14 | 15 | Application.put_env(:gust, :dag_parser, Gust.DAGParserMock) 16 | Application.put_env(:gust, :dag_compiler, Gust.DAGCompilerMock) 17 | Application.put_env(:gust, :dag_loader, Gust.DAGLoaderMock) 18 | Application.put_env(:gust, :dag_scheduler, Gust.DAGSchedulerMock) 19 | Application.put_env(:gust, :file_monitor, Gust.FileMonitorMock) 20 | Application.put_env(:gust, :dag_runner_supervisor, Gust.DAGRunnerSupervisorMock) 21 | Application.put_env(:gust, :dag_stage_runner_supervisor, Gust.DAGStageRunnerSupervisorMock) 22 | Application.put_env(:gust, :dag_task_runner_supervisor, Gust.DAGTaskRunnerSupervisorMock) 23 | Application.put_env(:gust, :dag_stage_coordinator, Gust.DAGStageCoordinatorMock) 24 | Application.put_env(:gust, :dag_logger, Gust.DAGLoggerMock) 25 | Application.put_env(:gust, :dag_run_restarter, Gust.DAGRunRestarterMock) 26 | Application.put_env(:gust, :dag_task_delayer, Gust.DAGTaskDelayerMock) 27 | 28 | ExUnit.start() 29 | Ecto.Adapters.SQL.Sandbox.mode(Gust.Repo, :manual) 30 | -------------------------------------------------------------------------------- /apps/gust/test/support/data_case.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DataCase do 2 | @moduledoc """ 3 | This module defines the setup for tests requiring 4 | access to the application's data layer. 5 | 6 | You may define functions here to be used as helpers in 7 | your tests. 8 | 9 | Finally, if the test case interacts with the database, 10 | we enable the SQL sandbox, so changes done to the database 11 | are reverted at the end of every test. If you are using 12 | PostgreSQL, you can even run database tests asynchronously 13 | by setting `use Gust.DataCase, async: true`, although 14 | this option is not recommended for other databases. 15 | """ 16 | 17 | use ExUnit.CaseTemplate 18 | alias Ecto.Adapters.SQL.Sandbox 19 | 20 | using do 21 | quote do 22 | alias Gust.Repo 23 | 24 | import Ecto 25 | import Ecto.Changeset 26 | import Ecto.Query 27 | import Gust.DataCase 28 | end 29 | end 30 | 31 | setup tags do 32 | Gust.DataCase.setup_sandbox(tags) 33 | :ok 34 | end 35 | 36 | @doc """ 37 | Sets up the sandbox based on the test tags. 38 | """ 39 | def setup_sandbox(tags) do 40 | pid = Sandbox.start_owner!(Gust.Repo, shared: not tags[:async]) 41 | on_exit(fn -> Sandbox.stop_owner(pid) end) 42 | end 43 | 44 | @doc """ 45 | A helper that transforms changeset errors into a map of messages. 46 | 47 | assert {:error, changeset} = Accounts.create_user(%{password: "short"}) 48 | assert "password is too short" in errors_on(changeset).password 49 | assert %{password: ["password is too short"]} = errors_on(changeset) 50 | 51 | """ 52 | def errors_on(changeset) do 53 | Ecto.Changeset.traverse_errors(changeset, fn {message, opts} -> 54 | Regex.replace(~r"%{(\w+)}", message, fn _, key -> 55 | opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() 56 | end) 57 | end) 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Umbrella.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | apps_path: "apps", 7 | version: "0.1.0", 8 | start_permanent: Mix.env() == :prod, 9 | deps: deps(), 10 | aliases: aliases(), 11 | listeners: [Phoenix.CodeReloader], 12 | test_coverage: [tool: ExCoveralls] 13 | ] 14 | end 15 | 16 | def cli do 17 | [ 18 | preferred_envs: [precommit: :test] 19 | ] 20 | end 21 | 22 | # Dependencies can be Hex packages: 23 | # 24 | # {:mydep, "~> 0.3.0"} 25 | # 26 | # Or git/path repositories: 27 | # 28 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1.0"} 29 | # 30 | # Type "mix help deps" for more examples and options. 31 | # 32 | # Dependencies listed here are available only for this project 33 | # and cannot be accessed from applications inside the apps/ folder. 34 | defp deps do 35 | [ 36 | # Required to run "mix format" on ~H/.heex files from the umbrella root 37 | {:phoenix_live_view, ">= 0.0.0"}, 38 | {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, 39 | {:excoveralls, "~> 0.18", only: :test}, 40 | {:mox, "~> 1.0", only: :test} 41 | ] 42 | end 43 | 44 | # Aliases are shortcuts or tasks specific to the current project. 45 | # For example, to install project dependencies and perform other setup tasks, run: 46 | # 47 | # $ mix setup 48 | # 49 | # See the documentation for `Mix` for more info on aliases. 50 | # 51 | # Aliases listed here are available only for this project 52 | # and cannot be accessed from applications inside the apps/ folder. 53 | defp aliases do 54 | [ 55 | # run `mix setup` in all child apps 56 | setup: ["cmd mix setup"], 57 | precommit: ["compile --warning-as-errors", "deps.unlock --unused", "format", "test"], 58 | lint: ["credo --strict"] 59 | ] 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/flows/task.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.Task do 2 | @moduledoc false 3 | 4 | use Ecto.Schema 5 | import Ecto.Changeset 6 | 7 | schema "tasks" do 8 | field :name, :string 9 | 10 | field :status, Ecto.Enum, 11 | values: [ 12 | :created, 13 | :running, 14 | :succeeded, 15 | :failed, 16 | :retrying, 17 | :upstream_failed, 18 | :enqueued 19 | ], 20 | default: :created 21 | 22 | field :result, :map, default: %{} 23 | field :error, :map, default: %{} 24 | field :attempt, :integer, default: 1 25 | belongs_to :run, Gust.Flows.Run 26 | has_many :logs, Gust.Flows.Log 27 | 28 | timestamps(type: :utc_datetime) 29 | end 30 | 31 | @type t :: %__MODULE__{ 32 | id: integer() | nil, 33 | name: String.t() | nil, 34 | status: 35 | :created 36 | | :running 37 | | :succeeded 38 | | :failed 39 | | :retrying 40 | | :upstream_failed 41 | | :enqueued, 42 | result: map(), 43 | error: map(), 44 | attempt: integer(), 45 | run_id: integer() | nil, 46 | run: Gust.Flows.Run.t() | Ecto.Association.NotLoaded.t(), 47 | logs: [Gust.Flows.Log.t()] | Ecto.Association.NotLoaded.t(), 48 | inserted_at: DateTime.t() | nil, 49 | updated_at: DateTime.t() | nil 50 | } 51 | 52 | @doc false 53 | def changeset(task, attrs) do 54 | task 55 | |> cast(attrs, [:name, :status, :run_id, :result, :attempt, :error]) 56 | |> validate_required([:name, :status, :run_id, :result, :error]) 57 | end 58 | 59 | @doc false 60 | def test_changeset(run, attrs) do 61 | run 62 | |> cast(attrs, [:inserted_at, :updated_at, :name, :status, :run_id, :result, :attempt, :error]) 63 | |> validate_required([:name, :status, :run_id, :result, :error]) 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/endpoint.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Endpoint do 2 | use Phoenix.Endpoint, otp_app: :gust_web 3 | 4 | # The session will be stored in the cookie and signed, 5 | # this means its contents can be read but not tampered with. 6 | # Set :encryption_salt if you would also like to encrypt it. 7 | @session_options [ 8 | store: :cookie, 9 | key: "_gust_web_key", 10 | signing_salt: "bfUJ4RCX", 11 | same_site: "Lax" 12 | ] 13 | 14 | socket "/live", Phoenix.LiveView.Socket, 15 | websocket: [connect_info: [session: @session_options]], 16 | longpoll: [connect_info: [session: @session_options]] 17 | 18 | # Serve at "/" the static files from "priv/static" directory. 19 | # 20 | # When code reloading is disabled (e.g., in production), 21 | # the `gzip` option is enabled to serve compressed 22 | # static files generated by running `phx.digest`. 23 | app_name = Application.compile_env(:gust, :app_name, :gust_web) 24 | 25 | plug Plug.Static, 26 | at: "/", 27 | from: app_name, 28 | gzip: not code_reloading?, 29 | only: GustWeb.static_paths() 30 | 31 | # Code reloading can be explicitly enabled under the 32 | # :code_reloader configuration of your endpoint. 33 | if code_reloading? do 34 | socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket 35 | plug Phoenix.LiveReloader 36 | plug Phoenix.CodeReloader 37 | plug Phoenix.Ecto.CheckRepoStatus, otp_app: :gust_web 38 | end 39 | 40 | plug Phoenix.LiveDashboard.RequestLogger, 41 | param_key: "request_logger", 42 | cookie_key: "request_logger" 43 | 44 | plug Plug.RequestId 45 | plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint] 46 | 47 | plug Plug.Parsers, 48 | parsers: [:urlencoded, :multipart, :json], 49 | pass: ["*/*"], 50 | json_decoder: Phoenix.json_library() 51 | 52 | plug Plug.MethodOverride 53 | plug Plug.Head 54 | plug Plug.Session, @session_options 55 | plug GustWeb.Router 56 | end 57 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/runner/task_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Runner.TaskWorker do 2 | @moduledoc false 3 | use GenServer 4 | alias Gust.DAG 5 | 6 | @impl true 7 | def init(init_arg) do 8 | {:ok, init_arg, {:continue, :init_run}} 9 | end 10 | 11 | def start_link(args) do 12 | GenServer.start_link(__MODULE__, args, name: via_tuple("task_#{args[:task].id}")) 13 | end 14 | 15 | defp via_tuple(name) do 16 | {:via, Registry, {Gust.Registry, name}} 17 | end 18 | 19 | def child_spec(args) do 20 | %{ 21 | id: __MODULE__, 22 | start: {__MODULE__, :start_link, [args]}, 23 | restart: :temporary, 24 | type: :worker 25 | } 26 | end 27 | 28 | @impl true 29 | def handle_continue(:init_run, state) do 30 | send(self(), :run) 31 | 32 | {:noreply, state} 33 | end 34 | 35 | @impl true 36 | def handle_info( 37 | :run, 38 | %{task: task, mod: mod, stage_pid: stage_pid, opts: opts} = state 39 | ) do 40 | fun_name = String.to_atom(task.name) 41 | args = [%{run_id: task.run_id}] 42 | 43 | DAG.Logger.set_task(task.id, task.attempt) 44 | 45 | {status, result} = 46 | case try_run(mod, fun_name, args, opts[:store_result]) do 47 | {:ok, result} -> 48 | {:ok, result} 49 | 50 | {:error, error} -> 51 | {:error, error} 52 | end 53 | 54 | DAG.Logger.unset() 55 | 56 | send(stage_pid, {:task_result, result, task.id, status}) 57 | 58 | {:stop, :normal, state} 59 | end 60 | 61 | defp try_run(mod, fun_name, args, store_result) do 62 | apply_and_validate(mod, fun_name, args, store_result) 63 | rescue 64 | e -> {:error, e} 65 | end 66 | 67 | defp apply_and_validate(mod, fun_name, args, store_result) do 68 | result = apply(mod, fun_name, args) 69 | 70 | if store_result && !is_map(result) do 71 | raise "Task returned #{inspect(result)} but store_result requires a map" 72 | else 73 | {:ok, result} 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/router.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Router do 2 | use GustWeb, :router 3 | 4 | pipeline :browser do 5 | plug :accepts, ["html"] 6 | plug :fetch_session 7 | plug :fetch_live_flash 8 | plug :put_root_layout, html: {GustWeb.Layouts, :root} 9 | plug :protect_from_forgery 10 | plug :put_secure_browser_headers 11 | end 12 | 13 | auth_enabled? = Application.compile_env(:gust_web, :basic_auth) 14 | 15 | if auth_enabled? do 16 | defp basic_auth(conn, _opts) do 17 | Plug.BasicAuth.basic_auth(conn, 18 | username: System.get_env("BASIC_AUTH_USER"), 19 | password: System.get_env("BASIC_AUTH_PASS") 20 | ) 21 | end 22 | end 23 | 24 | scope "/", GustWeb do 25 | pipe_through if auth_enabled?, do: [:browser, :basic_auth], else: :browser 26 | 27 | get "/", PageController, :home 28 | live "/dags", DagLive.Index, :index 29 | live "/dags/:name/dashboard", DagLive.Dashboard, :dashboard 30 | live "/dags/:name/runs", RunLive.Index, :index 31 | live "/secrets", SecretLive.Index, :index 32 | live "/secrets/new", SecretLive.Index, :new 33 | live "/secrets/:id/edit", SecretLive.Index, :edit 34 | end 35 | 36 | # Other scopes may use custom stacks. 37 | # scope "/api", GustWeb do 38 | # pipe_through :api 39 | # end 40 | 41 | # Enable LiveDashboard and Swoosh mailbox preview in development 42 | if Application.compile_env(:gust_web, :dev_routes) do 43 | # If you want to use the LiveDashboard in production, you should put 44 | # it behind authentication and allow only admins to access it. 45 | # If your application does not have an admins-only section yet, 46 | # you can use Plug.BasicAuth to set up some basic authentication 47 | # as long as you are also using SSL (which you should anyway). 48 | import Phoenix.LiveDashboard.Router 49 | 50 | scope "/dev" do 51 | pipe_through :browser 52 | 53 | live_dashboard "/dashboard", metrics: GustWeb.Telemetry 54 | forward "/mailbox", Plug.Swoosh.MailboxPreview 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | name: Build and test 15 | runs-on: ubuntu-latest 16 | env: 17 | MIX_ENV: test 18 | PGUSER: postgres 19 | PGPASSWORD: postgres 20 | PGHOST: localhost 21 | PGPORT: 5432 22 | B64_SECRETS_CLOAK_KEY: PF4/V7ZLI8Nyjymua0eUy2K8q/lcSFi3USe3UUDb7sY= 23 | SECRET_KEY_BASE: dWKtgAuY24rgqDQCLkN9S5CaOqSFzdrnqO4iHqVlIp5uLQPB6MjVJqlEN7w1nOvX 24 | 25 | services: 26 | postgres: 27 | image: postgres:14 28 | ports: ['5432:5432'] 29 | env: 30 | POSTGRES_USER: postgres 31 | POSTGRES_PASSWORD: postgres 32 | options: >- 33 | --health-cmd "pg_isready -U postgres" 34 | --health-interval 10s 35 | --health-timeout 5s 36 | --health-retries 5 37 | 38 | steps: 39 | - uses: actions/checkout@v4 40 | 41 | - name: Set up Elixir 42 | uses: erlef/setup-beam@v1 43 | with: 44 | elixir-version: '1.18' 45 | otp-version: '27.0' 46 | 47 | - name: Restore dependencies cache 48 | uses: actions/cache@v3 49 | with: 50 | path: deps 51 | key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} 52 | restore-keys: ${{ runner.os }}-mix- 53 | 54 | - name: Install deps 55 | run: mix deps.get 56 | 57 | - name: Check formatting 58 | run: mix format --check-formatted 59 | 60 | - name: Check warnings 61 | run: mix compile --warnings-as-errors 62 | 63 | - name: Run lint 64 | run: mix lint 65 | 66 | - name: Create and migrate DB 67 | run: | 68 | mix ecto.create 69 | mix ecto.migrate 70 | 71 | - name: Run tests 72 | run: mix coveralls.json --umbrella 73 | 74 | - name: Coveralls 75 | uses: coverallsapp/github-action@v2 76 | with: 77 | file: cover/excoveralls.json 78 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/mermaid_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.MermaidTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias GustWeb.Mermaid 5 | 6 | defp edges_from_chart(chart) do 7 | chart 8 | |> String.split("\n", parts: 2) 9 | |> List.last() 10 | |> String.trim() 11 | |> String.split(~r/\R+/, trim: true) 12 | |> Enum.reject(&(&1 == "")) 13 | |> MapSet.new() 14 | end 15 | 16 | test "chart/1 with a single task and no upstream deps" do 17 | tasks = [ 18 | {"A", %{downstream: MapSet.new(), upstream: MapSet.new()}} 19 | ] 20 | 21 | assert Mermaid.chart(tasks) == "flowchart LR\n \nA" 22 | end 23 | 24 | test "chart/1 with a single upstream dependency" do 25 | tasks = [ 26 | {"A", %{downstream: MapSet.new(), upstream: MapSet.new(["X"])}} 27 | ] 28 | 29 | assert Mermaid.chart(tasks) == "flowchart LR\n \nX --> A" 30 | end 31 | 32 | test "chart/1 with multiple tasks and multiple upstreams (order-agnostic)" do 33 | tasks = [ 34 | {"B", %{downstream: MapSet.new(), upstream: MapSet.new(["A", "C"])}}, 35 | {"D", %{downstream: MapSet.new(), upstream: MapSet.new(["B"])}} 36 | ] 37 | 38 | chart = Mermaid.chart(tasks) 39 | 40 | assert String.starts_with?(chart, "flowchart LR\n") 41 | 42 | expected_edges = 43 | MapSet.new([ 44 | "A --> B", 45 | "C --> B", 46 | "B --> D" 47 | ]) 48 | 49 | assert edges_from_chart(chart) == expected_edges 50 | end 51 | 52 | test "chart/1 accepts a map of tasks as well (order-agnostic)" do 53 | tasks = 54 | %{ 55 | "B" => %{downstream: MapSet.new(), upstream: MapSet.new(["A"])}, 56 | "C" => %{downstream: MapSet.new(), upstream: MapSet.new(["A"])}, 57 | "D" => %{downstream: MapSet.new(), upstream: MapSet.new(["B", "C"])} 58 | } 59 | 60 | chart = Mermaid.chart(tasks) 61 | 62 | expected_edges = 63 | MapSet.new([ 64 | "A --> B", 65 | "A --> C", 66 | "B --> D", 67 | "C --> D" 68 | ]) 69 | 70 | assert edges_from_chart(chart) == expected_edges 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/graph.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Graph do 2 | @moduledoc false 3 | 4 | def build_branch(tasks, direction, name) do 5 | branch = tasks[name][direction] 6 | 7 | if branch == MapSet.new([]) do 8 | [name] 9 | else 10 | siblings = for sibling_name <- branch, do: build_branch(tasks, direction, sibling_name) 11 | [name | siblings] 12 | end 13 | end 14 | 15 | def to_stages(tasks) do 16 | {:ok, sort(tasks)} 17 | rescue 18 | e in Gust.DAG.Graph.CycleDection -> 19 | {:error, e} 20 | end 21 | 22 | def link_tasks(task_list) do 23 | downstreams = build_downstreams(task_list) 24 | upstreams = build_upstreams(downstreams) 25 | merge_streams(downstreams, upstreams) 26 | end 27 | 28 | defp build_downstreams(task_list) do 29 | Map.new(task_list, fn 30 | {name, opts} when is_list(opts) -> 31 | {to_string(name), 32 | opts |> Keyword.get(:downstream, []) |> Enum.map(&to_string/1) |> MapSet.new()} 33 | end) 34 | end 35 | 36 | defp build_upstreams(downstreams) do 37 | Enum.reduce(downstreams, %{}, fn {node, ds}, acc -> 38 | Enum.reduce(ds, acc, fn d, acc2 -> 39 | Map.update(acc2, to_string(d), MapSet.new([node]), &MapSet.put(&1, node)) 40 | end) 41 | end) 42 | end 43 | 44 | defp merge_streams(downstreams, upstreams) do 45 | Map.new(downstreams, fn {node, ds} -> 46 | {node, %{downstream: ds, upstream: Map.get(upstreams, node, MapSet.new())}} 47 | end) 48 | end 49 | 50 | def sort(tasks, sorted \\ []) 51 | def sort(tasks, sorted) when map_size(tasks) == 0, do: sorted 52 | 53 | def sort(tasks, sorted) do 54 | {current_task_layer, next_tasks} = 55 | tasks |> Map.split_with(fn {_k, v} -> MapSet.size(v[:upstream]) == 0 end) 56 | 57 | layer_keys = Map.keys(current_task_layer) 58 | 59 | next_tasks = 60 | next_tasks 61 | |> Map.new(fn {k, v} -> 62 | removed_up = MapSet.difference(v[:upstream], MapSet.new(layer_keys)) 63 | {k, %{v | upstream: removed_up}} 64 | end) 65 | 66 | if map_size(current_task_layer) == 0, do: raise(Gust.DAG.Graph.CycleDection) 67 | 68 | sort(next_tasks, sorted ++ [layer_keys]) 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /apps/gust/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.1.25" 5 | 6 | def project do 7 | [ 8 | app: :gust, 9 | version: @version, 10 | build_path: "../../_build", 11 | config_path: "../../config/config.exs", 12 | deps_path: "../../deps", 13 | lockfile: "../../mix.lock", 14 | elixir: "~> 1.15", 15 | elixirc_paths: elixirc_paths(Mix.env()), 16 | start_permanent: Mix.env() == :prod, 17 | aliases: aliases(), 18 | deps: deps(), 19 | test_coverage: [tool: ExCoveralls], 20 | description: "A DAG-Based Workflow Orchestration Engine for Elixir", 21 | package: [ 22 | licenses: ["Apache-2.0"], 23 | links: %{"GitHub" => "https://github.com/marciok/gust"} 24 | ] 25 | ] 26 | end 27 | 28 | # Configuration for the OTP application. 29 | # 30 | # Type `mix help compile.app` for more information. 31 | def application do 32 | [ 33 | mod: {Gust.Application, []}, 34 | extra_applications: [:logger, :runtime_tools] 35 | ] 36 | end 37 | 38 | # Specifies which paths to compile per environment. 39 | defp elixirc_paths(:test), do: ["lib", "test/support"] 40 | defp elixirc_paths(_), do: ["lib"] 41 | 42 | # Specifies your project dependencies. 43 | # 44 | # Type `mix help deps` for examples and options. 45 | defp deps do 46 | [ 47 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 48 | {:dns_cluster, "~> 0.2.0"}, 49 | {:phoenix_pubsub, "~> 2.1"}, 50 | {:ecto_sql, "~> 3.13"}, 51 | {:postgrex, ">= 0.0.0"}, 52 | {:jason, "~> 1.2"}, 53 | {:swoosh, "~> 1.16"}, 54 | {:req, "~> 0.5"}, 55 | {:quantum, "~> 3.0"}, 56 | {:cloak_ecto, "~> 1.2.0"}, 57 | {:file_system, "~> 1.1", only: [:dev, :test]} 58 | ] 59 | end 60 | 61 | # Aliases are shortcuts or tasks specific to the current project. 62 | # 63 | # See the documentation for `Mix` for more info on aliases. 64 | defp aliases do 65 | [ 66 | setup: ["deps.get", "ecto.setup"], 67 | "ecto.setup": ["ecto.create", "ecto.migrate", "run #{__DIR__}/priv/repo/seeds.exs"], 68 | "ecto.reset": ["ecto.drop", "ecto.setup"], 69 | test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"] 70 | ] 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/pub_sub.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.PubSub do 2 | @moduledoc false 3 | 4 | # Topic bases 5 | @topic_run "dag:run" 6 | @topic_task "dag:task" 7 | @topic_file "dag:file" 8 | 9 | # Event atoms 10 | @event_run_started :run_started 11 | @event_run_status :run_status 12 | @event_file_update :file_updated 13 | 14 | ## Broadcasts 15 | # 16 | def broadcast_log(task_id, log_id) do 17 | Phoenix.PubSub.broadcast( 18 | __MODULE__, 19 | "#{@topic_task}:#{task_id}", 20 | {:task, :log, %{task_id: task_id, log_id: log_id}} 21 | ) 22 | end 23 | 24 | def broadcast_run_status(run_id, status) do 25 | Phoenix.PubSub.broadcast( 26 | __MODULE__, 27 | "#{@topic_run}:#{run_id}", 28 | {:dag, @event_run_status, %{run_id: run_id, status: status}} 29 | ) 30 | end 31 | 32 | def broadcast_run_started(dag_id, run_id) do 33 | Phoenix.PubSub.broadcast( 34 | __MODULE__, 35 | "#{@topic_run}:#{dag_id}", 36 | {:dag, @event_run_started, %{dag_id: dag_id, run_id: run_id}} 37 | ) 38 | end 39 | 40 | def broadcast_file_update(name, parse_result, action) do 41 | payload = 42 | {:dag, @event_file_update, %{dag_name: name, parse_result: parse_result, action: action}} 43 | 44 | # Broadcast to all-file topic 45 | Phoenix.PubSub.broadcast(__MODULE__, "#{@topic_file}:update", payload) 46 | 47 | # Broadcast to specific-file topic 48 | Phoenix.PubSub.broadcast(__MODULE__, "#{@topic_file}:#{name}", payload) 49 | end 50 | 51 | ## Subscriptions 52 | 53 | # Subscribe to updates for *all* files 54 | def subscribe_all_files(action) do 55 | Phoenix.PubSub.subscribe(__MODULE__, "#{@topic_file}:#{action}") 56 | end 57 | 58 | # Subscribe to updates for a *specific* file 59 | def subscribe_file(name) do 60 | Phoenix.PubSub.subscribe(__MODULE__, "#{@topic_file}:#{name}") 61 | end 62 | 63 | # Subscribe to a single run’s status 64 | def subscribe_run(run_id) do 65 | Phoenix.PubSub.subscribe(__MODULE__, "#{@topic_run}:#{run_id}") 66 | end 67 | 68 | def subscribe_task(task_id) do 69 | Phoenix.PubSub.subscribe(__MODULE__, "#{@topic_task}:#{task_id}") 70 | end 71 | 72 | # Subscribe to all runs under a given DAG 73 | def subscribe_runs_for_dag(dag_id) do 74 | Phoenix.PubSub.subscribe(__MODULE__, "#{@topic_run}:#{dag_id}") 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/run_live/index.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.RunLive.Index do 2 | alias Gust.Flows 3 | alias Gust.PubSub 4 | use GustWeb, :live_view 5 | 6 | @impl true 7 | def mount(_params, _session, socket) do 8 | {:ok, assign(socket, :page_title, "Listing Runs")} 9 | end 10 | 11 | @impl true 12 | def handle_params(%{"name" => name, "page_size" => page_size, "page" => page}, _uri, socket) do 13 | page_size = String.to_integer(page_size) 14 | page = String.to_integer(page) 15 | 16 | dag = load_dag(page, page_size, name) 17 | runs_count = Flows.count_runs_on_dag(dag.id) 18 | pages = div(runs_count + page_size - 1, page_size) 19 | 20 | if connected?(socket) do 21 | PubSub.subscribe_runs_for_dag(dag.id) 22 | Enum.each(dag.runs, fn %{id: id} -> PubSub.subscribe_run(id) end) 23 | end 24 | 25 | {:noreply, 26 | socket 27 | |> assign(:dag, dag) 28 | |> assign(:page_size, page_size) 29 | |> assign(:runs_count, runs_count) 30 | |> assign(:page, page) 31 | |> assign(:pages, 1..pages) 32 | |> stream(:runs, dag.runs, reset: true)} 33 | end 34 | 35 | @impl true 36 | def handle_event("select_page", %{"page" => num}, socket) do 37 | dag = socket.assigns.dag 38 | page_size = socket.assigns.page_size 39 | 40 | {:noreply, 41 | socket |> push_patch(to: ~p"/dags/#{dag.name}/runs?page_size=#{page_size}&page=#{num}")} 42 | end 43 | 44 | @impl true 45 | def handle_event("delete", %{"id" => id}, socket) do 46 | run = Flows.get_run!(id) 47 | {:ok, _} = Flows.delete_run(run) 48 | 49 | {:noreply, socket |> stream_delete(:runs, run)} 50 | end 51 | 52 | @impl true 53 | def handle_info( 54 | {:dag, :run_started, %{run_id: run_id}}, 55 | socket 56 | ) do 57 | run = Flows.get_run!(run_id) 58 | PubSub.subscribe_run(run_id) 59 | 60 | {:noreply, stream_insert(socket, :runs, run, at: 0)} 61 | end 62 | 63 | @impl true 64 | def handle_info( 65 | {:dag, :run_status, %{run_id: run_id, status: _status}}, 66 | socket 67 | ) do 68 | run = Flows.get_run!(run_id) 69 | 70 | {:noreply, stream_insert(socket, :runs, run)} 71 | end 72 | 73 | defp load_dag(page, size, name) do 74 | offset = (page - 1) * size 75 | 76 | Flows.get_dag_by_name_with_runs!(name, limit: size, offset: offset) 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /apps/gust/test/dag/graph_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Dag.GraphTest do 2 | alias Gust.DAG.Graph 3 | use Gust.DataCase 4 | 5 | @graph %{ 6 | "bye" => %{ 7 | downstream: MapSet.new([]), 8 | upstream: MapSet.new(["for_me", "wait"]) 9 | }, 10 | "hi" => %{ 11 | downstream: MapSet.new(["for_me", "wait"]), 12 | upstream: MapSet.new([]) 13 | }, 14 | "for_me" => %{ 15 | downstream: MapSet.new(["bye"]), 16 | upstream: MapSet.new(["hi"]) 17 | }, 18 | "wait" => %{ 19 | downstream: MapSet.new(["bye"]), 20 | upstream: MapSet.new(["hi"]) 21 | } 22 | } 23 | 24 | describe "build_branch/3" do 25 | test "when direction is upstream" do 26 | direction = :upstream 27 | 28 | assert [ 29 | "bye", 30 | ["for_me", ["hi"]], 31 | ["wait", ["hi"]] 32 | ] = 33 | Graph.build_branch(@graph, direction, "bye") 34 | end 35 | 36 | test "when direction is downstream" do 37 | direction = :downstream 38 | 39 | assert [ 40 | "hi", 41 | ["for_me", ["bye"]], 42 | ["wait", ["bye"]] 43 | ] = 44 | Graph.build_branch(@graph, direction, "hi") 45 | end 46 | end 47 | 48 | describe "to_stage/1" do 49 | test "no cycle detected" do 50 | stages = [ 51 | ["hi"], 52 | ["for_me", "wait"], 53 | ["bye"] 54 | ] 55 | 56 | assert Graph.to_stages(@graph) == {:ok, stages} 57 | end 58 | 59 | test "cycle is detected" do 60 | graph = 61 | %{ 62 | "bye" => %{ 63 | downstream: MapSet.new(["hi"]), 64 | upstream: MapSet.new(["hi"]) 65 | }, 66 | "hi" => %{ 67 | downstream: MapSet.new(["bye"]), 68 | upstream: MapSet.new(["bye"]) 69 | } 70 | } 71 | 72 | assert {:error, %Graph.CycleDection{message: "Possible cycle detected"}} == 73 | Graph.to_stages(graph) 74 | end 75 | end 76 | 77 | describe "link_tasks/1" do 78 | test "linked tasks map" do 79 | tasks_list = [ 80 | {:bye, []}, 81 | {:wait, [downstream: [:bye]]}, 82 | {:for_me, [downstream: [:bye]]}, 83 | {:hi, [downstream: [:wait, :for_me]]} 84 | ] 85 | 86 | assert @graph == Graph.link_tasks(tasks_list) 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/dag_live/index.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.DagLive.Index do 2 | alias Gust.DAG.{Loader, RunRestarter} 3 | alias Gust.Flows 4 | alias Gust.PubSub 5 | use GustWeb, :live_view 6 | 7 | @impl true 8 | def mount(_params, _session, socket) do 9 | PubSub.subscribe_all_files("update") 10 | 11 | dag_defs = Loader.get_definitions() 12 | 13 | dags = 14 | for {dag_id, {:ok, dag_def}} <- dag_defs do 15 | dag = Flows.get_dag!(dag_id) 16 | %{id: dag.name, dag: dag, dag_def: dag_def} 17 | end 18 | 19 | broken_dags = 20 | for {dag_id, {:error, error}} <- dag_defs do 21 | dag = Flows.get_dag!(dag_id) 22 | %{id: dag.name, dag: dag, error: error} 23 | end 24 | 25 | {:ok, 26 | socket 27 | |> assign(:page_title, "DAGs Listing") 28 | |> stream(:dags, dags) 29 | |> stream(:broken_dags, broken_dags)} 30 | end 31 | 32 | @impl true 33 | def handle_event("trigger_run", %{"id" => id}, socket) do 34 | run = RunRestarter.start_dag(String.to_integer(id)) 35 | run = Flows.get_run_with_tasks!(run.id) 36 | 37 | {:noreply, socket |> put_flash(:info, "Run #{run.id} triggered")} 38 | end 39 | 40 | @impl true 41 | def handle_info( 42 | {:dag, :file_updated, 43 | %{action: "removed", dag_name: name, parse_result: {:error, _error}}}, 44 | socket 45 | ) do 46 | {:noreply, socket |> stream_delete(:dags, %{id: name})} 47 | end 48 | 49 | @impl true 50 | def handle_info( 51 | {:dag, :file_updated, %{action: "reload", dag_name: name, parse_result: {:error, error}}}, 52 | socket 53 | ) do 54 | dag = Flows.get_dag_by_name(name) 55 | socket = stream_insert(socket, :broken_dags, %{id: name, dag: dag, error: error}) 56 | socket = stream_delete(socket, :dags, %{id: name}) 57 | {:noreply, socket} 58 | end 59 | 60 | @impl true 61 | def handle_info( 62 | {:dag, :file_updated, %{action: "reload", parse_result: {:ok, dag_def}}}, 63 | socket 64 | ) do 65 | name = dag_def.name 66 | dag = Flows.get_dag_by_name(name) 67 | socket = insert_dag(socket, dag, dag_def) 68 | socket = stream_delete(socket, :broken_dags, %{id: dag.name}) 69 | {:noreply, socket} 70 | end 71 | 72 | defp insert_dag(socket, dag, dag_def) do 73 | stream_insert(socket, :dags, %{id: dag.name, dag: dag, dag_def: dag_def}) 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/live/breadcrumbs_live_component_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.BreadcrumbsLiveComponentTest do 2 | use GustWeb.ConnCase 3 | import Phoenix.LiveViewTest 4 | import Gust.FlowsFixtures 5 | 6 | require GustWeb.LiveComponentTest 7 | import GustWeb.LiveComponentTest 8 | 9 | setup do 10 | dag = dag_fixture(%{name: "my_dag"}) 11 | Gust.Flows.get_dag!(dag.id) 12 | 13 | dag_def = %Gust.DAG.Definition{ 14 | name: "my_dag", 15 | options: [schedule: "* * * *"] 16 | } 17 | 18 | %{dag: dag, dag_def: dag_def} 19 | end 20 | 21 | test "only dag is provided", %{conn: conn, dag_def: dag_def} do 22 | {:ok, breadcrumbs, _html} = 23 | live_component_isolated(conn, GustWeb.BreadcrumbsComponent, %{ 24 | run: nil, 25 | task: nil, 26 | dag_def: dag_def 27 | }) 28 | 29 | assert breadcrumbs |> element("#dag-runs-link") |> render_click() 30 | assert_redirect breadcrumbs, "/dags/#{dag_def.name}/dashboard" 31 | 32 | {:ok, breadcrumbs, _html} = 33 | live_component_isolated(conn, GustWeb.BreadcrumbsComponent, %{ 34 | run: nil, 35 | task: nil, 36 | dag_def: dag_def 37 | }) 38 | 39 | assert breadcrumbs |> element("#dags-link") |> render_click() 40 | assert_redirect breadcrumbs, "/dags" 41 | end 42 | 43 | test "dag and run is provided", %{conn: conn, dag: dag, dag_def: dag_def} do 44 | run = run_fixture(%{dag_id: dag.id}) 45 | 46 | {:ok, breadcrumbs, _html} = 47 | live_component_isolated(conn, GustWeb.BreadcrumbsComponent, %{ 48 | run: run, 49 | task: nil, 50 | dag_def: dag_def 51 | }) 52 | 53 | assert breadcrumbs |> element("#dag-run-link") |> render_click() 54 | assert_redirect breadcrumbs, "/dags/#{dag_def.name}/dashboard?run_id=#{run.id}" 55 | end 56 | 57 | test "dag, run and task is provided", %{conn: conn, dag: dag, dag_def: dag_def} do 58 | run = run_fixture(%{dag_id: dag.id}) 59 | task = task_fixture(%{run_id: run.id, name: "hello_breadcrumb"}) 60 | 61 | {:ok, breadcrumbs, _html} = 62 | live_component_isolated(conn, GustWeb.BreadcrumbsComponent, %{ 63 | run: run, 64 | task: task, 65 | dag_def: dag_def 66 | }) 67 | 68 | assert breadcrumbs |> element("#dag-run-task-link") |> render_click() 69 | 70 | assert_redirect breadcrumbs, 71 | "/dags/#{dag_def.name}/dashboard?run_id=#{run.id}&task_name=#{task.name}" 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/run_live/index.html.heex: -------------------------------------------------------------------------------- 1 | 2 |
3 | 13 |
14 | 15 |
16 |
17 |
18 |

19 | <.icon name="hero-queue-list" class="h-5 w-5 text-sky-600" /> 20 | Runs for {@dag.name}: 21 | {@runs_count} 22 |

23 | <.form for={%{}} id="page-select" phx-change="select_page" class=""> 24 | <.input 25 | id="pages" 26 | value={@page} 27 | name="page" 28 | type="select" 29 | options={Enum.map(@pages, fn num -> {"Page: #{num}", num} end)} 30 | class="select select-bordered select-sm w-28" 31 | /> 32 | 33 |
34 |
35 | 36 |
37 |
38 | <.table id="runs" rows={@streams.runs}> 39 | <:col :let={{_id, run}} label="Status"> 40 | <.status_badge status={run.status} data-testid="status-badge" /> 41 | 42 | 43 | --%> 44 | <:col :let={{_id, run}} label="Inserted At"> 45 | {to_string(run.inserted_at)} 46 | 47 | 48 | <:col :let={{_id, run}} label="Updated At"> 49 | {to_string(run.updated_at)} 50 | 51 | 52 | <:col :let={{_id, run}} label="ID"> 53 | {run.id} 54 | 55 | 56 | <:action :let={{id, run}}> 57 | <.link 58 | phx-click={JS.push("delete", value: %{id: run.id}) |> hide("##{id}")} 59 | data-confirm="Are you sure? Make sure it is completed, otherwise it may cause unexpected behavior." 60 | class="btn btn-sm btn-soft btn-error" 61 | > 62 | Delete 63 | 64 | 65 | 66 |
67 |
68 |
69 | 70 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/secret_live/index.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.SecretLive.Index do 2 | use GustWeb, :live_view 3 | 4 | alias Gust.Flows 5 | alias Gust.Flows.Secret 6 | 7 | @impl true 8 | def mount(_params, _session, socket) do 9 | {:ok, stream(socket, :secrets, Flows.list_secrets())} 10 | end 11 | 12 | @impl true 13 | def handle_params(params, _url, socket) do 14 | {:noreply, apply_action(socket, socket.assigns.live_action, params)} 15 | end 16 | 17 | defp apply_action(socket, :index, _params) do 18 | socket 19 | |> assign(:page_title, "Listing Secrets") 20 | |> assign(:secret, nil) 21 | end 22 | 23 | defp apply_action(socket, :edit, %{"id" => id}) do 24 | secret = Flows.get_secret!(id) 25 | secret = secret |> Map.put(:value, "") 26 | 27 | socket 28 | |> assign(:page_title, "Edit Secret") 29 | |> assign(:secret, secret) 30 | |> assign(:form, to_form(Flows.change_secret(secret))) 31 | end 32 | 33 | defp apply_action(socket, :new, _params) do 34 | secret = %Secret{} 35 | 36 | socket 37 | |> assign(:page_title, "New Secret") 38 | |> assign(:secret, %Secret{}) 39 | |> assign(:form, to_form(Flows.change_secret(secret))) 40 | end 41 | 42 | @impl true 43 | def handle_event("validate", %{"secret" => secret_params}, socket) do 44 | changeset = Flows.change_secret(socket.assigns.secret, secret_params) 45 | {:noreply, assign(socket, form: to_form(changeset, action: :validate))} 46 | end 47 | 48 | def handle_event("save", %{"secret" => secret_params}, socket) do 49 | action = socket.assigns.live_action 50 | save_secret(socket, action, secret_params) 51 | end 52 | 53 | @impl true 54 | def handle_event("delete", %{"id" => id}, socket) do 55 | secret = Flows.get_secret!(id) 56 | {:ok, _} = Flows.delete_secret(secret) 57 | 58 | {:noreply, socket |> stream_delete(:secrets, secret)} 59 | end 60 | 61 | defp save_secret(socket, :new, secret_params) do 62 | case Flows.create_secret(secret_params) do 63 | {:ok, secret} -> 64 | socket |> apply_secret(secret, "created") 65 | end 66 | end 67 | 68 | defp save_secret(socket, :edit, secret_params) do 69 | case Flows.update_secret(socket.assigns.secret, secret_params) do 70 | {:ok, secret} -> 71 | socket |> apply_secret(secret, "updated") 72 | end 73 | end 74 | 75 | defp apply_secret(socket, secret, action_verb) do 76 | {:noreply, 77 | socket 78 | |> stream_insert(:secrets, secret) 79 | |> put_flash(:info, "Secret #{action_verb} successfully") 80 | |> push_patch(to: ~p"/secrets")} 81 | end 82 | end 83 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your umbrella 2 | # and **all applications** and their dependencies with the 3 | # help of the Config module. 4 | # 5 | # Note that all applications in your umbrella share the 6 | # same configuration and dependencies, which is why they 7 | # all use the same configuration file. If you want different 8 | # configurations or dependencies per app, it is best to 9 | # move said applications out of the umbrella. 10 | import Config 11 | 12 | # Configure Mix tasks and generators 13 | config :gust, 14 | ecto_repos: [Gust.Repo] 15 | 16 | # Configures the mailer 17 | # 18 | # By default it uses the "Local" adapter which stores the emails 19 | # locally. You can see the emails in your browser, at "/dev/mailbox". 20 | # 21 | # For production it's recommended to configure a different adapter 22 | # at the `config/runtime.exs`. 23 | config :gust, Gust.Mailer, adapter: Swoosh.Adapters.Local 24 | 25 | config :gust_web, 26 | ecto_repos: [Gust.Repo], 27 | generators: [context_app: :gust] 28 | 29 | # Configures the endpoint 30 | config :gust_web, GustWeb.Endpoint, 31 | url: [host: "localhost"], 32 | adapter: Bandit.PhoenixAdapter, 33 | render_errors: [ 34 | formats: [html: GustWeb.ErrorHTML, json: GustWeb.ErrorJSON], 35 | layout: false 36 | ], 37 | pubsub_server: Gust.PubSub, 38 | live_view: [signing_salt: "SCG6tRFf"] 39 | 40 | # Configure esbuild (the version is required) 41 | config :esbuild, 42 | version: "0.25.4", 43 | gust_web: [ 44 | args: 45 | ~w(js/app.js --bundle --target=es2022 --outdir=../priv/static/assets/js --external:/fonts/* --external:/images/* --alias:@=.), 46 | cd: Path.expand("../apps/gust_web/assets", __DIR__), 47 | env: %{"NODE_PATH" => [Path.expand("../deps", __DIR__), Mix.Project.build_path()]} 48 | ] 49 | 50 | # Configure tailwind (the version is required) 51 | config :tailwind, 52 | version: "4.1.7", 53 | gust_web: [ 54 | args: ~w( 55 | --input=assets/css/app.css 56 | --output=priv/static/assets/css/app.css 57 | ), 58 | cd: Path.expand("../apps/gust_web", __DIR__) 59 | ] 60 | 61 | config :gust, dag_logger: Gust.DAG.Logger.Database 62 | # Configures Elixir's Logger 63 | config :logger, :default_formatter, 64 | format: "$time $metadata[$level] $message\n", 65 | metadata: [:request_id, :task_id, :attempt] 66 | 67 | config :logger, backends: [:console, Gust.DAG.Logger.Database] 68 | 69 | # Use Jason for JSON parsing in Phoenix 70 | config :phoenix, :json_library, Jason 71 | 72 | # Import environment specific config. This must remain at the bottom 73 | # of this file so it overrides the configuration defined above. 74 | import_config "#{config_env()}.exs" 75 | -------------------------------------------------------------------------------- /apps/gust/test/dag/scheduler/worker_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.Scheduler.WorkerTest do 2 | import Crontab.CronExpression 3 | use Gust.DataCase 4 | import Gust.FlowsFixtures 5 | import Gust.FSHelpers 6 | alias Gust.DAG.Cron 7 | alias Gust.DAG.Scheduler.Worker 8 | 9 | import Mox 10 | 11 | setup :verify_on_exit! 12 | setup :set_mox_from_context 13 | 14 | describe "handle_call/2 when {:load_dags, dags} is passed" do 15 | setup do 16 | start_link_supervised!(Cron) 17 | name = "blink_182" 18 | dag = dag_fixture(%{name: name}) 19 | dag_folder = make_rand_dir!("dags") 20 | 21 | %{dag: dag, folder: dag_folder} 22 | end 23 | 24 | test "do not add task for dag with errors", %{dag: dag} do 25 | name = dag.name 26 | 27 | dag_def = %Gust.DAG.Definition{ 28 | mod: MockDagMod, 29 | name: name, 30 | error: %CompileError{description: "opsss"}, 31 | options: [schedule: "* * * * *"] 32 | } 33 | 34 | start_link_supervised!({Gust.DAG.Scheduler.Worker, {}}) 35 | 36 | Worker.schedule(%{dag.id => dag_def}) 37 | 38 | assert Cron.find_job(String.to_atom(name)) == nil 39 | end 40 | 41 | test "do not add errored dag", %{dag: dag} do 42 | name = dag.name 43 | 44 | start_link_supervised!({Gust.DAG.Scheduler.Worker, {}}) 45 | 46 | Worker.schedule(%{dag.id => {:error, {}}}) 47 | 48 | assert Cron.find_job(String.to_atom(name)) == nil 49 | end 50 | 51 | test "do not add task for dag without scheduler", %{dag: dag} do 52 | name = dag.name 53 | 54 | dag_def = %Gust.DAG.Definition{ 55 | mod: MockDagMod, 56 | name: name, 57 | options: [] 58 | } 59 | 60 | start_link_supervised!({Gust.DAG.Scheduler.Worker, {}}) 61 | 62 | Worker.schedule(%{dag.id => {:ok, dag_def}}) 63 | 64 | assert Cron.find_job(String.to_atom(name)) == nil 65 | end 66 | 67 | test "add task for dag with scheduler", %{dag: dag} do 68 | name = dag.name 69 | 70 | dag_def = %Gust.DAG.Definition{ 71 | mod: MockDagMod, 72 | name: name, 73 | error: %{}, 74 | options: [schedule: "* * * * *"] 75 | } 76 | 77 | start_link_supervised!({Gust.DAG.Scheduler.Worker, {}}) 78 | atom_name = String.to_atom(name) 79 | schedule = ~e[* * * * *] 80 | 81 | start_dag_task = { 82 | Gust.DAG.RunRestarter, 83 | :start_dag, 84 | [dag.id] 85 | } 86 | 87 | [:ok] = Worker.schedule(%{dag.id => {:ok, dag_def}}) 88 | 89 | assert %Quantum.Job{name: ^atom_name, schedule: ^schedule, task: ^start_dag_task} = 90 | Cron.find_job(String.to_atom(name)) 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /apps/gust/test/dsl_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DSLTest do 2 | use Gust.DataCase 3 | 4 | test "use macro with schedule option" do 5 | dag_code = """ 6 | defmodule MyValidDagEmpty do 7 | use Gust.DSL, schedule: "0 17 * * *" 8 | 9 | task :hi do 10 | # saying hi 11 | 1 + 1 12 | end 13 | 14 | end 15 | """ 16 | 17 | [{mod, _bin}] = Code.compile_string(dag_code) 18 | 19 | assert mod.__dag_options__() == [schedule: "0 17 * * *"] 20 | 21 | :code.purge(mod) 22 | :code.delete(mod) 23 | end 24 | 25 | test "task macro without opts" do 26 | dag_code = """ 27 | defmodule MyValidDagEmpty do 28 | use Gust.DSL 29 | 30 | task :hi do 31 | # saying hi 32 | 1 + 1 33 | end 34 | 35 | end 36 | """ 37 | 38 | [{mod, _bin}] = Code.compile_string(dag_code) 39 | 40 | assert mod.__dag_tasks__() == [{:hi, []}] 41 | 42 | :code.purge(mod) 43 | :code.delete(mod) 44 | end 45 | 46 | test "task macro with context option" do 47 | run_id = 1234 48 | ctx = %{run_id: 1234} 49 | 50 | dag_code = """ 51 | defmodule MyValidDagEmpty do 52 | use Gust.DSL 53 | 54 | task :hi, ctx: %{run_id: run_id} do 55 | run_id 56 | end 57 | 58 | end 59 | """ 60 | 61 | [{mod, _bin}] = Code.compile_string(dag_code) 62 | assert mod.__dag_tasks__() == [{:hi, []}] 63 | 64 | # credo:disable-for-next-line Credo.Check.Refactor.Apply 65 | assert apply(mod, :hi, [ctx]) == run_id 66 | 67 | :code.purge(mod) 68 | :code.delete(mod) 69 | end 70 | 71 | test "task macro with store_result option" do 72 | dag_code = """ 73 | defmodule MyValidDagEmpty do 74 | use Gust.DSL 75 | 76 | task :hi, store_result: true do 77 | # saying hi 78 | 1 + 1 79 | end 80 | 81 | end 82 | """ 83 | 84 | [{mod, _bin}] = Code.compile_string(dag_code) 85 | 86 | assert mod.__dag_tasks__() == [{:hi, [store_result: true]}] 87 | 88 | :code.purge(mod) 89 | :code.delete(mod) 90 | end 91 | 92 | test "task macro without downstream opts" do 93 | dag_code = """ 94 | defmodule MyValidDagEmpty do 95 | use Gust.DSL 96 | 97 | task :bye do 98 | # saying bye 99 | 2 + 2 100 | end 101 | 102 | task :hi, downstreams: [:bye] do 103 | # saying hi 104 | 1 + 1 105 | end 106 | 107 | end 108 | """ 109 | 110 | [{mod, _bin}] = Code.compile_string(dag_code) 111 | 112 | assert mod.__dag_tasks__() == [{:hi, [downstreams: [:bye]]}, {:bye, []}] 113 | 114 | :code.purge(mod) 115 | :code.delete(mod) 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/secret_live/index.html.heex: -------------------------------------------------------------------------------- 1 | 2 |
3 | <.header> 4 | Listing Secrets 5 | <:actions> 6 | <.link patch={~p"/secrets/new"}> 7 | <.button>New Secret 8 | 9 | 10 | 11 |
12 | 13 |
14 |
15 | <.table id="secrets" rows={@streams.secrets}> 16 | <:col :let={{_id, secret}} label="Name"> 17 | {secret.name} 18 | 19 | 20 | <:col :let={{_id, secret}} label="Type"> 21 | {secret.value_type} 22 | 23 | 24 | <:action :let={{_id, secret}}> 25 | <.link patch={~p"/secrets/#{secret}/edit"} class="btn btn-sm btn-soft btn-primary"> 26 | Edit 27 | 28 | 29 | 30 | <:action :let={{id, secret}}> 31 | <.link 32 | phx-click={JS.push("delete", value: %{id: secret.id}) |> hide("##{id}")} 33 | data-confirm="Are you sure?" 34 | class="btn btn-sm btn-soft btn-error" 35 | > 36 | Delete 37 | 38 | 39 | 40 |
41 |
42 | 43 | <%= if @live_action in [:new, :edit] do %> 44 | <.form :let={f} for={@form} id="secret-form" phx-change="validate" phx-submit="save"> 45 |
46 |
47 |
48 | 51 | <.input field={f[:name]} type="text" class="input input-bordered w-full" /> 52 |
53 | 54 |
55 | 58 | <.input 59 | field={f[:value_type]} 60 | type="select" 61 | options={[:string, :json]} 62 | class="select select-bordered w-full" 63 | /> 64 |
65 | 66 |
67 | 70 | <.input 71 | field={f[:value]} 72 | type="textarea" 73 | class="textarea textarea-bordered w-full h-24" 74 | /> 75 |
76 | 77 |
78 | <.button class="btn btn-primary" phx-disable-with="Saving...">Save Secret 79 |
80 |
81 |
82 | 83 | <% end %> 84 | 85 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/components/dag_run_components.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.DagRunComponents do 2 | @moduledoc false 3 | 4 | use Phoenix.Component 5 | use Gettext, backend: GustWeb.Gettext 6 | 7 | attr :status, :atom, required: true 8 | attr :rest, :global, doc: "data-testid, etc." 9 | 10 | def status_badge(assigns) do 11 | ~H""" 12 |
"badge-success" 19 | :failed -> "badge-error" 20 | :upstream_failed -> "badge-warning" 21 | _ -> "badge-info" 22 | end 23 | ]} 24 | > 25 | {@status} 26 |
27 | """ 28 | end 29 | 30 | attr :level, :string, required: true 31 | 32 | def log_badge(assigns) do 33 | ~H""" 34 |
"badge-info" 39 | "info" -> "badge-info" 40 | "warn" -> "badge-warning" 41 | "error" -> "badge-error" 42 | end 43 | ]}> 44 | {@level} 45 |
46 | """ 47 | end 48 | 49 | attr :id, :string, required: true 50 | attr :selected, :string 51 | attr :status, :string 52 | 53 | def task_cell(assigns) do 54 | assigns = 55 | assign_new(assigns, :classes, fn -> 56 | base_classes = 57 | if assigns[:status], do: ["status-#{assigns[:status]}", "active"], else: ["status-none"] 58 | 59 | classes = base_classes ++ if assigns[:selected], do: ["selected"], else: [] 60 | 61 | Enum.join(classes, " ") 62 | end) 63 | 64 | ~H""" 65 |
69 |
70 | """ 71 | end 72 | 73 | attr :run_id, :string, required: true 74 | attr :ran_tasks, :list, required: true 75 | attr :name, :string, required: true 76 | attr :selected_task, :map, required: true 77 | attr :navigate, :string, required: true 78 | attr :rest, :global 79 | 80 | def interactive_task_cell(assigns) do 81 | assigns = 82 | assign_new(assigns, :current_task_ran, fn -> 83 | assigns[:ran_tasks] |> Enum.find(&(&1.name == assigns[:name])) 84 | end) 85 | 86 | if assigns[:current_task_ran] do 87 | ~H""" 88 | <.link navigate={@navigate} {@rest}> 89 | <.task_cell 90 | selected={if @selected_task, do: @selected_task.id == @current_task_ran.id, else: false} 91 | status={@current_task_ran.status} 92 | id={"#{@name}-at-run-#{@run_id}"} 93 | /> 94 | 95 | """ 96 | else 97 | ~H""" 98 | <.task_cell id={"#{@name}-at-run-#{@run_id}"} /> 99 | """ 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /apps/gust/test/flows/secret_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Gust.Flows.SecretTest do 2 | use ExUnit.Case, async: true 3 | alias Gust.Flows.Secret 4 | 5 | describe "name format validation" do 6 | test "accepts valid names" do 7 | valid_names = ["MY_SECRET", "HELLO_WORLD", "SECRET_123"] 8 | 9 | for name <- valid_names do 10 | changeset = Secret.changeset(%Secret{}, %{name: name, value: "abc", value_type: :string}) 11 | 12 | assert changeset.valid?, """ 13 | Expected #{name} to be valid, but got errors: #{inspect(changeset.errors)} 14 | """ 15 | end 16 | end 17 | 18 | test "rejects invalid names with correct error message" do 19 | invalid_names = [ 20 | {"my_secret", "must be uppercase with underscores"}, 21 | {"MySecret", "must be uppercase with underscores"}, 22 | {"MY-SECRET", "must be uppercase with underscores"}, 23 | {"MY SECRET", "must be uppercase with underscores"}, 24 | {"", "can't be blank"} 25 | ] 26 | 27 | for {name, expected_msg} <- invalid_names do 28 | changeset = Secret.changeset(%Secret{}, %{name: name, value: "abc", value_type: :string}) 29 | refute changeset.valid?, "Expected #{inspect(name)} to be invalid" 30 | 31 | {field, {msg, opts}} = hd(changeset.errors) 32 | 33 | assert field == :name 34 | assert msg == expected_msg 35 | 36 | if expected_msg == "must be uppercase with underscores" do 37 | assert opts[:validation] == :format 38 | end 39 | end 40 | end 41 | end 42 | 43 | describe "required fields" do 44 | test "rejects missing required fields" do 45 | changeset = Secret.changeset(%Secret{}, %{}) 46 | refute changeset.valid? 47 | assert Keyword.keys(changeset.errors) == [:name, :value, :value_type] 48 | end 49 | end 50 | 51 | describe "JSON value validation" do 52 | test "accepts valid JSON when value_type is :json" do 53 | json_value = ~s({"foo": "bar"}) 54 | 55 | changeset = 56 | Secret.changeset(%Secret{}, %{name: "VALID_JSON", value: json_value, value_type: :json}) 57 | 58 | assert changeset.valid?, "Expected valid JSON to be accepted" 59 | end 60 | 61 | test "rejects invalid JSON when value_type is :json" do 62 | invalid_json = ~s({foo: bar}) 63 | 64 | changeset = 65 | Secret.changeset(%Secret{}, %{ 66 | name: "INVALID_JSON", 67 | value: invalid_json, 68 | value_type: :json 69 | }) 70 | 71 | refute changeset.valid? 72 | assert {:value, {"must be valid JSON", []}} = hd(changeset.errors) 73 | end 74 | 75 | test "rejects nil JSON value" do 76 | changeset = 77 | Secret.changeset(%Secret{}, %{name: "EMPTY_JSON", value: nil, value_type: :json}) 78 | 79 | refute changeset.valid? 80 | assert {:value, {"it cannot be empty", []}} = hd(changeset.errors) 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/live/run_live_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.RunLiveTest do 2 | alias Gust.Flows 3 | use GustWeb.ConnCase 4 | 5 | import Phoenix.LiveViewTest 6 | import Gust.FlowsFixtures 7 | import Mox 8 | 9 | setup :verify_on_exit! 10 | 11 | describe "Index" do 12 | setup %{conn: conn} do 13 | dag = dag_fixture(%{name: "dag_with_runs"}) 14 | run = run_fixture(%{dag_id: dag.id}) 15 | 16 | %{conn: conn, run: run, dag: dag} 17 | end 18 | 19 | test "list runs", %{conn: conn, dag: dag, run: run} do 20 | {:ok, _index_live, html} = 21 | live(conn, ~p"/dags/#{dag.name}/runs?page_size=30&page=1") 22 | 23 | assert html =~ "Listing Runs" 24 | assert html =~ dag.name 25 | assert html =~ to_string(run.status) 26 | assert html =~ to_string(run.inserted_at) 27 | assert html =~ to_string(run.updated_at) 28 | assert html =~ to_string(run.id) 29 | end 30 | 31 | test "list runs paged", %{conn: conn, dag: dag, run: _first_run} do 32 | page_size = 3 33 | 34 | run_fixture(%{dag_id: dag.id}) 35 | prev_page_run = run_fixture(%{dag_id: dag.id}) 36 | 37 | current_page_run = run_fixture(%{dag_id: dag.id}) 38 | 39 | {:ok, index_live, _html} = 40 | live(conn, ~p"/dags/#{dag.name}/runs?page_size=#{page_size}&page=2") 41 | 42 | assert index_live |> has_element?("#runs-#{current_page_run.id}") 43 | refute index_live |> has_element?("#runs-#{prev_page_run.id}") 44 | 45 | assert index_live |> has_element?("#pages option[value='2']:checked") 46 | refute index_live |> has_element?("#pages option[value='3']") 47 | 48 | index_live 49 | |> element("#page-select") 50 | |> render_change(%{"_target" => "page", "page" => "1"}) 51 | 52 | assert_patch index_live, ~p"/dags/#{dag.name}/runs?page_size=3&page=1" 53 | end 54 | 55 | test "deletes run in listing", %{conn: conn, dag: dag, run: run} do 56 | {:ok, index_live, _html} = live(conn, ~p"/dags/#{dag.name}/runs?page_size=30&page=1") 57 | 58 | assert index_live |> element("#runs-#{run.id} a", "Delete") |> render_click() 59 | refute has_element?(index_live, "#runs-#{run.id}") 60 | end 61 | 62 | test "new dag run was created", %{conn: conn, dag: dag} do 63 | {:ok, index_live, _html} = 64 | live(conn, ~p"/dags/#{dag.name}/runs?page_size=30&page=1") 65 | 66 | new_run = run_fixture(%{dag_id: dag.id}) 67 | 68 | Gust.PubSub.broadcast_run_started(dag.id, new_run.id) 69 | 70 | assert index_live |> has_element?("#runs-#{new_run.id}") 71 | end 72 | 73 | test "run is updated", %{conn: conn, dag: dag, run: run} do 74 | {:ok, index_live, _html} = 75 | live(conn, ~p"/dags/#{dag.name}/runs?page_size=30&page=1") 76 | 77 | Flows.update_run_status(run, :succeeded) 78 | 79 | Gust.PubSub.broadcast_run_status(run.id, :succeeded) 80 | 81 | badge_html = 82 | index_live |> element("#runs-#{run.id} [data-testid='status-badge']") |> render() 83 | 84 | assert badge_html =~ "succeeded" 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/live/dag_summary_live_component_test.exs: -------------------------------------------------------------------------------- 1 | # TODO: test schedule change 2 | defmodule GustWeb.DagSummaryLiveComponentTest do 3 | alias Gust.Flows 4 | use GustWeb.ConnCase 5 | import Phoenix.LiveViewTest 6 | import Gust.FlowsFixtures 7 | 8 | require GustWeb.LiveComponentTest 9 | import GustWeb.LiveComponentTest 10 | import Mox 11 | 12 | setup :verify_on_exit! 13 | 14 | setup do 15 | dag = dag_fixture(%{name: "my_dag"}) 16 | Gust.Flows.get_dag!(dag.id) 17 | 18 | dag_def = %Gust.DAG.Definition{ 19 | name: "my_dag", 20 | options: [schedule: "* * * *"] 21 | } 22 | 23 | %{dag: dag, dag_def: dag_def} 24 | end 25 | 26 | describe "when dag_define has error" do 27 | test "cannot be triggered", %{conn: conn, dag: dag, dag_def: dag_def} do 28 | error_description = "Ops, something went kaboom" 29 | message = "check your code!" 30 | 31 | dag_def = %{ 32 | dag_def 33 | | messages: [%{message: message}], 34 | error: %CompileError{description: error_description} 35 | } 36 | 37 | {:ok, dag_summary, html} = 38 | live_component_isolated(conn, GustWeb.DagSummaryComponent, %{ 39 | id: dag.id, 40 | dag: dag, 41 | dag_def: dag_def 42 | }) 43 | 44 | assert dag_summary |> has_element?("#trigger-dag-run-#{dag.id}:disabled") 45 | 46 | assert html =~ error_description 47 | assert html =~ message 48 | end 49 | end 50 | 51 | test "re-enabled dag", %{conn: conn, dag: dag, dag_def: dag_def} do 52 | dag_id = dag.id 53 | {:ok, dag} = Flows.toggle_enabled(dag) 54 | 55 | GustWeb.DAGRunRestarterMock |> expect(:restart_enqueued, fn ^dag_id -> nil end) 56 | 57 | {:ok, dag_summary, _html} = 58 | live_component_isolated(conn, GustWeb.DagSummaryComponent, %{ 59 | id: dag_id, 60 | dag: dag, 61 | dag_def: dag_def 62 | }) 63 | 64 | dag_summary |> element("[name='dag-enabling-toggle-#{dag.id}']") |> render_click() 65 | 66 | assert Gust.Flows.get_dag!(dag.id).enabled == true 67 | end 68 | 69 | test "toggle dag enabled", %{conn: conn, dag: dag, dag_def: dag_def} do 70 | dag_id = dag.id 71 | 72 | {:ok, dag_summary, _html} = 73 | live_component_isolated(conn, GustWeb.DagSummaryComponent, %{ 74 | id: dag_id, 75 | dag: dag, 76 | dag_def: dag_def 77 | }) 78 | 79 | dag_summary |> element("[name='dag-enabling-toggle-#{dag.id}']:checked") |> render_click() 80 | 81 | assert Gust.Flows.get_dag!(dag.id).enabled == false 82 | end 83 | 84 | test "link to dag runs", %{conn: conn, dag: dag, dag_def: dag_def} do 85 | {:ok, dag_summary, _html} = 86 | live_component_isolated(conn, GustWeb.DagSummaryComponent, %{ 87 | dag: dag, 88 | dag_def: dag_def 89 | }) 90 | 91 | assert dag_summary |> has_element?(~s{[href="/dags/#{dag.name}/dashboard"]}) 92 | dag_summary |> element(~s{[href="/dags/#{dag.name}/dashboard"]}) |> render_click() 93 | 94 | assert_redirect dag_summary, "/dags/#{dag.name}/dashboard" 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb do 2 | @moduledoc """ 3 | The entrypoint for defining your web interface, such 4 | as controllers, components, channels, and so on. 5 | 6 | This can be used in your application as: 7 | 8 | use GustWeb, :controller 9 | use GustWeb, :html 10 | 11 | The definitions below will be executed for every controller, 12 | component, etc, so keep them short and clean, focused 13 | on imports, uses and aliases. 14 | 15 | Do NOT define functions inside the quoted expressions 16 | below. Instead, define additional modules and import 17 | those modules here. 18 | """ 19 | 20 | def static_paths, do: ~w(assets fonts images favicon.ico robots.txt) 21 | 22 | def router do 23 | quote do 24 | use Phoenix.Router, helpers: false 25 | 26 | # Import common connection and controller functions to use in pipelines 27 | import Plug.Conn 28 | import Phoenix.Controller 29 | import Phoenix.LiveView.Router 30 | end 31 | end 32 | 33 | def channel do 34 | quote do 35 | use Phoenix.Channel 36 | end 37 | end 38 | 39 | def controller do 40 | quote do 41 | use Phoenix.Controller, formats: [:html, :json] 42 | 43 | use Gettext, backend: GustWeb.Gettext 44 | 45 | import Plug.Conn 46 | 47 | unquote(verified_routes()) 48 | end 49 | end 50 | 51 | def live_view do 52 | quote do 53 | use Phoenix.LiveView 54 | 55 | unquote(html_helpers()) 56 | end 57 | end 58 | 59 | def live_component do 60 | quote do 61 | use Phoenix.LiveComponent 62 | 63 | unquote(html_helpers()) 64 | end 65 | end 66 | 67 | def html do 68 | quote do 69 | use Phoenix.Component 70 | 71 | # Import convenience functions from controllers 72 | import Phoenix.Controller, 73 | only: [get_csrf_token: 0, view_module: 1, view_template: 1] 74 | 75 | # Include general helpers for rendering HTML 76 | unquote(html_helpers()) 77 | end 78 | end 79 | 80 | defp html_helpers do 81 | quote do 82 | # Translation 83 | use Gettext, backend: GustWeb.Gettext 84 | 85 | # HTML escaping functionality 86 | import Phoenix.HTML 87 | # Core UI components 88 | import GustWeb.CoreComponents 89 | 90 | # DAG run components 91 | import GustWeb.DagRunComponents 92 | 93 | # Common modules used in templates 94 | alias GustWeb.Layouts 95 | alias Phoenix.LiveView.JS 96 | 97 | # Routes generation with the ~p sigil 98 | unquote(verified_routes()) 99 | end 100 | end 101 | 102 | def verified_routes do 103 | quote do 104 | use Phoenix.VerifiedRoutes, 105 | endpoint: GustWeb.Endpoint, 106 | router: GustWeb.Router, 107 | statics: GustWeb.static_paths() 108 | end 109 | end 110 | 111 | @doc """ 112 | When used, dispatch to the appropriate controller/view/etc. 113 | """ 114 | defmacro __using__(which) when is_atom(which) do 115 | apply(__MODULE__, which, []) 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /apps/gust_web/priv/gettext/en/LC_MESSAGES/errors.po: -------------------------------------------------------------------------------- 1 | ## `msgid`s in this file come from POT (.pot) files. 2 | ## 3 | ## Do not add, change, or remove `msgid`s manually here as 4 | ## they're tied to the ones in the corresponding POT file 5 | ## (with the same domain). 6 | ## 7 | ## Use `mix gettext.extract --merge` or `mix gettext.merge` 8 | ## to merge POT files into PO files. 9 | msgid "" 10 | msgstr "" 11 | "Language: en\n" 12 | 13 | ## From Ecto.Changeset.cast/4 14 | msgid "can't be blank" 15 | msgstr "" 16 | 17 | ## From Ecto.Changeset.unique_constraint/3 18 | msgid "has already been taken" 19 | msgstr "" 20 | 21 | ## From Ecto.Changeset.put_change/3 22 | msgid "is invalid" 23 | msgstr "" 24 | 25 | ## From Ecto.Changeset.validate_acceptance/3 26 | msgid "must be accepted" 27 | msgstr "" 28 | 29 | ## From Ecto.Changeset.validate_format/3 30 | msgid "has invalid format" 31 | msgstr "" 32 | 33 | ## From Ecto.Changeset.validate_subset/3 34 | msgid "has an invalid entry" 35 | msgstr "" 36 | 37 | ## From Ecto.Changeset.validate_exclusion/3 38 | msgid "is reserved" 39 | msgstr "" 40 | 41 | ## From Ecto.Changeset.validate_confirmation/3 42 | msgid "does not match confirmation" 43 | msgstr "" 44 | 45 | ## From Ecto.Changeset.no_assoc_constraint/3 46 | msgid "is still associated with this entry" 47 | msgstr "" 48 | 49 | msgid "are still associated with this entry" 50 | msgstr "" 51 | 52 | ## From Ecto.Changeset.validate_length/3 53 | msgid "should have %{count} item(s)" 54 | msgid_plural "should have %{count} item(s)" 55 | msgstr[0] "" 56 | msgstr[1] "" 57 | 58 | msgid "should be %{count} character(s)" 59 | msgid_plural "should be %{count} character(s)" 60 | msgstr[0] "" 61 | msgstr[1] "" 62 | 63 | msgid "should be %{count} byte(s)" 64 | msgid_plural "should be %{count} byte(s)" 65 | msgstr[0] "" 66 | msgstr[1] "" 67 | 68 | msgid "should have at least %{count} item(s)" 69 | msgid_plural "should have at least %{count} item(s)" 70 | msgstr[0] "" 71 | msgstr[1] "" 72 | 73 | msgid "should be at least %{count} character(s)" 74 | msgid_plural "should be at least %{count} character(s)" 75 | msgstr[0] "" 76 | msgstr[1] "" 77 | 78 | msgid "should be at least %{count} byte(s)" 79 | msgid_plural "should be at least %{count} byte(s)" 80 | msgstr[0] "" 81 | msgstr[1] "" 82 | 83 | msgid "should have at most %{count} item(s)" 84 | msgid_plural "should have at most %{count} item(s)" 85 | msgstr[0] "" 86 | msgstr[1] "" 87 | 88 | msgid "should be at most %{count} character(s)" 89 | msgid_plural "should be at most %{count} character(s)" 90 | msgstr[0] "" 91 | msgstr[1] "" 92 | 93 | msgid "should be at most %{count} byte(s)" 94 | msgid_plural "should be at most %{count} byte(s)" 95 | msgstr[0] "" 96 | msgstr[1] "" 97 | 98 | ## From Ecto.Changeset.validate_number/3 99 | msgid "must be less than %{number}" 100 | msgstr "" 101 | 102 | msgid "must be greater than %{number}" 103 | msgstr "" 104 | 105 | msgid "must be less than or equal to %{number}" 106 | msgstr "" 107 | 108 | msgid "must be greater than or equal to %{number}" 109 | msgstr "" 110 | 111 | msgid "must be equal to %{number}" 112 | msgstr "" 113 | -------------------------------------------------------------------------------- /apps/gust_web/priv/gettext/errors.pot: -------------------------------------------------------------------------------- 1 | ## This is a PO Template file. 2 | ## 3 | ## `msgid`s here are often extracted from source code. 4 | ## Add new translations manually only if they're dynamic 5 | ## translations that can't be statically extracted. 6 | ## 7 | ## Run `mix gettext.extract` to bring this file up to 8 | ## date. Leave `msgstr`s empty as changing them here has no 9 | ## effect: edit them in PO (`.po`) files instead. 10 | ## From Ecto.Changeset.cast/4 11 | msgid "can't be blank" 12 | msgstr "" 13 | 14 | ## From Ecto.Changeset.unique_constraint/3 15 | msgid "has already been taken" 16 | msgstr "" 17 | 18 | ## From Ecto.Changeset.put_change/3 19 | msgid "is invalid" 20 | msgstr "" 21 | 22 | ## From Ecto.Changeset.validate_acceptance/3 23 | msgid "must be accepted" 24 | msgstr "" 25 | 26 | ## From Ecto.Changeset.validate_format/3 27 | msgid "has invalid format" 28 | msgstr "" 29 | 30 | ## From Ecto.Changeset.validate_subset/3 31 | msgid "has an invalid entry" 32 | msgstr "" 33 | 34 | ## From Ecto.Changeset.validate_exclusion/3 35 | msgid "is reserved" 36 | msgstr "" 37 | 38 | ## From Ecto.Changeset.validate_confirmation/3 39 | msgid "does not match confirmation" 40 | msgstr "" 41 | 42 | ## From Ecto.Changeset.no_assoc_constraint/3 43 | msgid "is still associated with this entry" 44 | msgstr "" 45 | 46 | msgid "are still associated with this entry" 47 | msgstr "" 48 | 49 | ## From Ecto.Changeset.validate_length/3 50 | msgid "should have %{count} item(s)" 51 | msgid_plural "should have %{count} item(s)" 52 | msgstr[0] "" 53 | msgstr[1] "" 54 | 55 | msgid "should be %{count} character(s)" 56 | msgid_plural "should be %{count} character(s)" 57 | msgstr[0] "" 58 | msgstr[1] "" 59 | 60 | msgid "should be %{count} byte(s)" 61 | msgid_plural "should be %{count} byte(s)" 62 | msgstr[0] "" 63 | msgstr[1] "" 64 | 65 | msgid "should have at least %{count} item(s)" 66 | msgid_plural "should have at least %{count} item(s)" 67 | msgstr[0] "" 68 | msgstr[1] "" 69 | 70 | msgid "should be at least %{count} character(s)" 71 | msgid_plural "should be at least %{count} character(s)" 72 | msgstr[0] "" 73 | msgstr[1] "" 74 | 75 | msgid "should be at least %{count} byte(s)" 76 | msgid_plural "should be at least %{count} byte(s)" 77 | msgstr[0] "" 78 | msgstr[1] "" 79 | 80 | msgid "should have at most %{count} item(s)" 81 | msgid_plural "should have at most %{count} item(s)" 82 | msgstr[0] "" 83 | msgstr[1] "" 84 | 85 | msgid "should be at most %{count} character(s)" 86 | msgid_plural "should be at most %{count} character(s)" 87 | msgstr[0] "" 88 | msgstr[1] "" 89 | 90 | msgid "should be at most %{count} byte(s)" 91 | msgid_plural "should be at most %{count} byte(s)" 92 | msgstr[0] "" 93 | msgstr[1] "" 94 | 95 | ## From Ecto.Changeset.validate_number/3 96 | msgid "must be less than %{number}" 97 | msgstr "" 98 | 99 | msgid "must be greater than %{number}" 100 | msgstr "" 101 | 102 | msgid "must be less than or equal to %{number}" 103 | msgstr "" 104 | 105 | msgid "must be greater than or equal to %{number}" 106 | msgstr "" 107 | 108 | msgid "must be equal to %{number}" 109 | msgstr "" 110 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/live/secret_live_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.SecretLiveTest do 2 | use GustWeb.ConnCase 3 | 4 | import Phoenix.LiveViewTest 5 | import Gust.FlowsFixtures 6 | 7 | @create_attrs %{name: "SOME_SECRET", value: "some value", value_type: :string} 8 | @update_attrs %{name: "SOME_SECRET", value: "some updated value", value_type: :string} 9 | @invalid_attrs %{name: nil, value: nil} 10 | @invalid_json %{name: "SOME_SECRET", value: "invalid json value", value_type: :json} 11 | 12 | defp create_secret(_) do 13 | secret = secret_fixture() 14 | %{secret: secret} 15 | end 16 | 17 | describe "Index" do 18 | setup [:create_secret] 19 | 20 | test "lists all secrets", %{conn: conn, secret: secret} do 21 | {:ok, _index_live, html} = live(conn, ~p"/secrets") 22 | 23 | assert html =~ "Listing Secrets" 24 | assert html =~ secret.name 25 | end 26 | 27 | test "saves new secret", %{conn: conn} do 28 | {:ok, index_live, _html} = live(conn, ~p"/secrets") 29 | 30 | assert index_live |> element("a", "New Secret") |> render_click() =~ 31 | "New Secret" 32 | 33 | assert_patch(index_live, ~p"/secrets/new") 34 | 35 | assert index_live 36 | |> form("#secret-form", secret: @invalid_attrs) 37 | |> render_change() =~ "can't be blank" 38 | 39 | assert index_live 40 | |> form("#secret-form", secret: @invalid_json) 41 | |> render_change() =~ "must be valid JSON" 42 | 43 | assert index_live 44 | |> form("#secret-form", secret: @create_attrs) 45 | |> render_submit() 46 | 47 | assert_patch(index_live, ~p"/secrets") 48 | 49 | html = render(index_live) 50 | assert html =~ "Secret created successfully" 51 | assert html =~ "SOME_NAME" 52 | end 53 | 54 | test "updates secret in listing", %{conn: conn, secret: secret} do 55 | {:ok, index_live, _html} = live(conn, ~p"/secrets") 56 | 57 | assert index_live |> element("#secrets-#{secret.id} a", "Edit") |> render_click() =~ 58 | "Edit" 59 | 60 | assert_patch(index_live, ~p"/secrets/#{secret}/edit") 61 | 62 | secret_value_html = index_live |> element("#secret-form_value") |> render() 63 | 64 | case Regex.run(~r/]*>(.*?)<\/textarea>/, secret_value_html) do 65 | [_, value] -> assert value == "" 66 | end 67 | 68 | assert index_live 69 | |> form("#secret-form", secret: @invalid_attrs) 70 | |> render_change() =~ "can't be blank" 71 | 72 | assert index_live 73 | |> form("#secret-form", secret: @update_attrs) 74 | |> render_submit() 75 | 76 | assert_patch(index_live, ~p"/secrets") 77 | 78 | html = render(index_live) 79 | assert html =~ "Secret updated successfully" 80 | end 81 | 82 | test "deletes secret in listing", %{conn: conn, secret: secret} do 83 | {:ok, index_live, _html} = live(conn, ~p"/secrets") 84 | 85 | assert index_live |> element("#secrets-#{secret.id} a", "Delete") |> render_click() 86 | refute has_element?(index_live, "#secrets-#{secret.id}") 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/telemetry.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Telemetry do 2 | use Supervisor 3 | import Telemetry.Metrics 4 | 5 | def start_link(arg) do 6 | Supervisor.start_link(__MODULE__, arg, name: __MODULE__) 7 | end 8 | 9 | @impl true 10 | def init(_arg) do 11 | children = [ 12 | # Telemetry poller will execute the given period measurements 13 | # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics 14 | {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} 15 | # Add reporters as children of your supervision tree. 16 | # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} 17 | ] 18 | 19 | Supervisor.init(children, strategy: :one_for_one) 20 | end 21 | 22 | def metrics do 23 | [ 24 | # Phoenix Metrics 25 | summary("phoenix.endpoint.start.system_time", 26 | unit: {:native, :millisecond} 27 | ), 28 | summary("phoenix.endpoint.stop.duration", 29 | unit: {:native, :millisecond} 30 | ), 31 | summary("phoenix.router_dispatch.start.system_time", 32 | tags: [:route], 33 | unit: {:native, :millisecond} 34 | ), 35 | summary("phoenix.router_dispatch.exception.duration", 36 | tags: [:route], 37 | unit: {:native, :millisecond} 38 | ), 39 | summary("phoenix.router_dispatch.stop.duration", 40 | tags: [:route], 41 | unit: {:native, :millisecond} 42 | ), 43 | summary("phoenix.socket_connected.duration", 44 | unit: {:native, :millisecond} 45 | ), 46 | sum("phoenix.socket_drain.count"), 47 | summary("phoenix.channel_joined.duration", 48 | unit: {:native, :millisecond} 49 | ), 50 | summary("phoenix.channel_handled_in.duration", 51 | tags: [:event], 52 | unit: {:native, :millisecond} 53 | ), 54 | 55 | # Database Metrics 56 | summary("gust.repo.query.total_time", 57 | unit: {:native, :millisecond}, 58 | description: "The sum of the other measurements" 59 | ), 60 | summary("gust.repo.query.decode_time", 61 | unit: {:native, :millisecond}, 62 | description: "The time spent decoding the data received from the database" 63 | ), 64 | summary("gust.repo.query.query_time", 65 | unit: {:native, :millisecond}, 66 | description: "The time spent executing the query" 67 | ), 68 | summary("gust.repo.query.queue_time", 69 | unit: {:native, :millisecond}, 70 | description: "The time spent waiting for a database connection" 71 | ), 72 | summary("gust.repo.query.idle_time", 73 | unit: {:native, :millisecond}, 74 | description: 75 | "The time the connection spent waiting before being checked out for the query" 76 | ), 77 | 78 | # VM Metrics 79 | summary("vm.memory.total", unit: {:byte, :kilobyte}), 80 | summary("vm.total_run_queue_lengths.total"), 81 | summary("vm.total_run_queue_lengths.cpu"), 82 | summary("vm.total_run_queue_lengths.io") 83 | ] 84 | end 85 | 86 | defp periodic_measurements do 87 | [ 88 | # A module, function and arguments to be invoked periodically. 89 | # This function must call :telemetry.execute/3 and a metric must be added above. 90 | # {GustWeb, :count_users, []} 91 | ] 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/live/dag_summary_component.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.DagSummaryComponent do 2 | @moduledoc false 3 | use GustWeb, :live_component 4 | alias Gust.DAG.RunRestarter 5 | alias Gust.Flows 6 | 7 | @impl true 8 | def handle_event("toggle_enabled", %{"id" => dag_id}, socket) do 9 | {:ok, dag} = Flows.get_dag!(dag_id) |> Flows.toggle_enabled() 10 | 11 | if dag.enabled do 12 | RunRestarter.restart_enqueued(dag.id) 13 | end 14 | 15 | {:noreply, socket |> assign(:dag, dag)} 16 | end 17 | 18 | defp format_error(%CompileError{file: _file, description: description, line: _line}), 19 | do: description 20 | 21 | @impl true 22 | def render(assigns) do 23 | ~H""" 24 |
25 |
0} class="dag-card__notice"> 26 | 31 |
32 | 33 |
0} class="dag-card__notice"> 34 |
35 |

Warnings

36 |
    37 | <%= for message <- @dag_def.messages do %> 38 |
  • {message.message}
    • 39 | <% end %> 40 |
41 |
42 |
43 | 44 |
45 |
46 | 51 | {if(@dag.enabled, do: "Enabled", else: "Paused")} 52 | 53 | 54 | 65 |
66 |
67 |

68 | <.link navigate={~p"/dags/#{@dag.name}/dashboard"}>{@dag.name} 69 |

70 | 71 |
72 | 81 |
82 |
83 |
84 | 85 |
86 |
87 |
Schedule
88 |
{@dag_def.options[:schedule]}
89 |
90 |
91 |
92 | """ 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/runner/dag_worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Runner.DAGWorker do 2 | @moduledoc false 3 | use GenServer 4 | 5 | alias Gust.DAG.{Compiler, Definition, StageRunnerSupervisor} 6 | alias Gust.Flows 7 | alias Gust.PubSub 8 | 9 | alias __MODULE__, as: State 10 | 11 | defstruct run: nil, dag_def: %Definition{}, stages: [] 12 | 13 | @status_map %{ 14 | ok: :succeeded, 15 | upstream_failed: :failed, 16 | error: :failed, 17 | cancelled: :failed 18 | } 19 | 20 | @impl true 21 | def init(%State{dag_def: dag_def} = state) do 22 | runtime_mod = Compiler.compile(dag_def) 23 | dag_def = %{dag_def | mod: runtime_mod} 24 | state = %{state | dag_def: dag_def} 25 | 26 | {:ok, state, {:continue, :init_stage}} 27 | end 28 | 29 | def child_spec(args) do 30 | %{ 31 | id: __MODULE__, 32 | start: {__MODULE__, :start_link, [args]}, 33 | restart: :temporary, 34 | type: :worker 35 | } 36 | end 37 | 38 | def start_link(args) do 39 | GenServer.start_link(__MODULE__, struct!(State, args), 40 | name: via_tuple("dag_run_#{args[:run].id}") 41 | ) 42 | end 43 | 44 | defp via_tuple(name) do 45 | {:via, Registry, {Gust.Registry, name}} 46 | end 47 | 48 | @impl true 49 | def handle_continue( 50 | :init_stage, 51 | %State{run: run, dag_def: %Definition{stages: [stage | next_stages]} = dag_def} = state 52 | ) do 53 | dag_id = run.dag_id 54 | id = run.id 55 | PubSub.broadcast_run_started(dag_id, id) 56 | start_stage(stage, run.id, dag_def) 57 | update_status(run, :running) 58 | state = Map.put(state, :stages, next_stages) 59 | 60 | {:noreply, state} 61 | end 62 | 63 | defp start_stage(stage, run_id, dag_def) do 64 | task_ids = 65 | for name <- stage do 66 | {:ok, task} = ensure_task(name, run_id) 67 | task.id 68 | end 69 | 70 | {:ok, _stage_pid} = 71 | StageRunnerSupervisor.start_child(dag_def, task_ids, run_id) 72 | end 73 | 74 | defp ensure_task(name, run_id) do 75 | case Flows.get_task_by_name_run(name, run_id) do 76 | nil -> 77 | Flows.create_task(%{run_id: run_id, name: name}) 78 | 79 | %Flows.Task{status: :running} = task -> 80 | Flows.update_task_status(task, :created) 81 | 82 | %Flows.Task{} = task -> 83 | {:ok, task} 84 | end 85 | end 86 | 87 | @impl true 88 | def handle_info( 89 | {:stage_completed, status}, 90 | %State{stages: [], dag_def: dag_def, run: run} = state 91 | ) do 92 | update_status(run, @status_map[status]) 93 | options = dag_def.options 94 | 95 | {callback, _options} = Keyword.pop(options, :on_finished_callback) 96 | if callback, do: apply(dag_def.mod, callback, [status, run]) 97 | 98 | Compiler.purge(dag_def.mod) 99 | {:stop, :normal, state} 100 | end 101 | 102 | def handle_info( 103 | {:stage_completed, _status}, 104 | %State{stages: [stage | next_stages], dag_def: dag_def, run: run} = state 105 | ) do 106 | start_stage(stage, run.id, dag_def) 107 | 108 | {:noreply, %{state | stages: next_stages}} 109 | end 110 | 111 | defp update_status(run, status) do 112 | Flows.update_run_status(run, status) |> broadcast() 113 | end 114 | 115 | defp broadcast({:ok, %Flows.Run{id: id, status: status}}) do 116 | Gust.PubSub.broadcast_run_status(id, status) 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /apps/gust_web/test/gust_web/live/dag_live_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.DagLiveTest do 2 | alias Gust.Flows 3 | use GustWeb.ConnCase 4 | 5 | import Phoenix.LiveViewTest 6 | import Gust.FlowsFixtures 7 | import Mox 8 | 9 | setup :verify_on_exit! 10 | 11 | describe "Index" do 12 | setup %{conn: conn} do 13 | dag = dag_fixture() 14 | broken_dag = dag_fixture(%{name: "broken_dag"}) 15 | 16 | dag_def = %Gust.DAG.Definition{name: dag.name} 17 | 18 | GustWeb.DAGLoaderMock 19 | |> expect(:get_definitions, 2, fn -> 20 | %{dag.id => {:ok, dag_def}, broken_dag.id => {:error, {}}} 21 | end) 22 | 23 | %{conn: conn, dag: dag, dag_def: dag_def, broken_dag: broken_dag} 24 | end 25 | 26 | test "lists all valid dags on dag folder", %{conn: conn, dag: dag, broken_dag: broken_dag} do 27 | {:ok, _index_live, html} = live(conn, ~p"/dags") 28 | 29 | assert html =~ "DAGs Listing" 30 | assert html =~ dag.name 31 | assert html =~ broken_dag.name 32 | end 33 | 34 | test "dag file was reloaded", %{conn: conn, dag: dag, broken_dag: broken_dag} do 35 | {:ok, index_live, _html} = live(conn, ~p"/dags") 36 | dag_name = dag.name 37 | 38 | dag_def = %Gust.DAG.Definition{name: broken_dag.name} 39 | 40 | send( 41 | index_live.pid, 42 | {:dag, :file_updated, %{action: "reload", parse_result: {:ok, dag_def}}} 43 | ) 44 | 45 | broken_dag_html = index_live |> element("#broken-dags") |> render() 46 | assert render(index_live) =~ dag_name 47 | refute broken_dag_html =~ broken_dag.name 48 | end 49 | 50 | test "file reload event when parse errored", %{conn: conn, dag: dag} do 51 | {:ok, index_live, _html} = live(conn, ~p"/dags") 52 | 53 | dag_name = dag.name 54 | 55 | send( 56 | index_live.pid, 57 | {:dag, :file_updated, %{action: "reload", dag_name: dag_name, parse_result: {:error, {}}}} 58 | ) 59 | 60 | Process.sleep(200) 61 | broken_dag_html = index_live |> element("#broken-dags") |> render() 62 | dag_html = index_live |> element("#dags") |> render() 63 | 64 | assert broken_dag_html =~ dag.name 65 | refute dag_html =~ dag.name 66 | end 67 | 68 | test "file deletion event when a dag exists", %{conn: conn, dag: dag} do 69 | {:ok, index_live, _html} = live(conn, ~p"/dags") 70 | 71 | dag_name = dag.name 72 | Flows.delete_dag!(dag) 73 | 74 | send( 75 | index_live.pid, 76 | {:dag, :file_updated, 77 | %{action: "removed", dag_name: dag_name, parse_result: {:error, nil}}} 78 | ) 79 | 80 | refute render(index_live) =~ dag.name 81 | end 82 | 83 | test "navigate to runs afger click into a dag", %{conn: conn, dag: dag} do 84 | {:ok, index_live, _html} = live(conn, ~p"/dags") 85 | 86 | assert has_element?(index_live, ~s{[href="/dags/#{dag.name}/dashboard"]}) 87 | end 88 | 89 | test "click on dag run trigger", %{conn: conn, dag: dag} do 90 | dag_id = dag.id 91 | {:ok, index_live, _html} = live(conn, ~p"/dags") 92 | 93 | new_run = run_fixture(%{dag_id: dag_id}) 94 | 95 | GustWeb.DAGRunRestarterMock |> expect(:start_dag, fn ^dag_id -> new_run end) 96 | 97 | assert index_live |> element("#trigger-dag-run-#{dag.id}") |> render_click() =~ 98 | "Run #{new_run.id} triggered" 99 | end 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/stage_coordinator/retrying_runner.ex: -------------------------------------------------------------------------------- 1 | # TODO: Rename to RetryingTask 2 | defmodule Gust.DAG.StageCoordinator.RetryingRunner do 3 | @moduledoc false 4 | alias Gust.DAG.TaskDelayer 5 | alias Gust.Flows 6 | @behaviour Gust.DAG.StageCoordinator 7 | 8 | alias __MODULE__, as: Coord 9 | 10 | defstruct running: MapSet.new(), retrying: %{} 11 | 12 | def new(pending_task_ids), do: %Coord{running: MapSet.new(pending_task_ids)} 13 | 14 | def update_restart_timer(%{retrying: retrying} = coord, %{id: task_id}, ref) do 15 | updated_retrying = 16 | Map.update!(retrying, task_id, &Map.put(&1, :restart_timer, ref)) 17 | 18 | %{coord | retrying: updated_retrying} 19 | end 20 | 21 | def process_task(%{status: :created, name: name, run_id: run_id}, tasks) do 22 | upstream_statuses = 23 | tasks[name][:upstream] 24 | |> Enum.map(&Flows.get_task_by_name_run(&1, run_id)) 25 | |> Enum.map(& &1.status) 26 | 27 | if any_upstream_failed?(upstream_statuses), do: :upstream_failed, else: :ok 28 | end 29 | 30 | def process_task(%{status: status}, _tasks) when status in [:retrying], do: :ok 31 | 32 | def process_task(%{status: status}, _tasks) 33 | when status in [:succeeded, :failed, :upstream_failed], 34 | do: :already_processed 35 | 36 | def put_running(%{running: running} = coord, task_id) do 37 | %{coord | running: MapSet.put(running, task_id)} 38 | end 39 | 40 | def apply_task_result(coord, task, :upstream_failed), do: coord |> remove_pending_task(task) 41 | def apply_task_result(coord, task, :ok), do: coord |> remove_pending_task(task) 42 | def apply_task_result(coord, task, :already_processed), do: coord |> remove_pending_task(task) 43 | def apply_task_result(coord, task, :cancelled), do: coord |> remove_pending_task(task) 44 | 45 | def apply_task_result(%Coord{running: _running, retrying: retrying} = coord, task, :error) do 46 | task_id = task.id 47 | 48 | if retrying[task_id][:attempt] == 3 do 49 | fail_task(coord, task, task_id) 50 | else 51 | retry_task(coord, task, task_id) 52 | end 53 | end 54 | 55 | defp remove_pending_task(%Coord{running: running, retrying: retrying} = coord, task) do 56 | task_id = task.id 57 | 58 | coord_status(%{ 59 | coord 60 | | running: MapSet.delete(running, task_id), 61 | retrying: Map.delete(retrying, task_id) 62 | }) 63 | end 64 | 65 | defp any_upstream_failed?(upstream_tasks) do 66 | Enum.any?(upstream_tasks, fn status -> status in [:failed, :upstream_failed] end) 67 | end 68 | 69 | defp coord_status(coord), do: {if(any_running?(coord), do: :continue, else: :finished), coord} 70 | 71 | defp fail_task(%{running: running, retrying: retrying} = coord, task, task_id) do 72 | retrying = Map.delete(retrying, task.id) 73 | 74 | coord_status(%{ 75 | coord 76 | | running: MapSet.delete(running, task_id), 77 | retrying: retrying 78 | }) 79 | end 80 | 81 | defp retry_task(%Coord{running: running, retrying: retrying} = coord, task, task_id) do 82 | retrying = 83 | Map.update(retrying, task.id, %{attempt: 1}, fn %{attempt: attempt_num} -> 84 | %{attempt: attempt_num + 1} 85 | end) 86 | 87 | delay = TaskDelayer.calc_delay(task.attempt) 88 | 89 | {:reschedule, %{coord | running: MapSet.delete(running, task_id), retrying: retrying}, task, 90 | delay} 91 | end 92 | 93 | defp any_running?(%Coord{running: running, retrying: retrying}) do 94 | not Enum.empty?(running) or map_size(retrying) > 0 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /apps/gust_web/test/support/live_component_tests.ex: -------------------------------------------------------------------------------- 1 | # Copied from: https://gist.github.com/mcrumm/8e6b0a98196dd74a841d850c70805f50 2 | defmodule GustWeb.LiveComponentTest do 3 | @moduledoc """ 4 | Conveniences for testing a LiveComponent in isolation. 5 | """ 6 | 7 | defmodule Driver do 8 | @moduledoc """ 9 | A LiveView for driving a LiveComponent under test. 10 | """ 11 | use Phoenix.LiveView 12 | 13 | def render(assigns) do 14 | ~H""" 15 | <.live_component :if={@lc_module} module={@lc_module} {@lc_attrs} /> 16 | """ 17 | end 18 | 19 | def handle_call({:run, func}, _, socket) when is_function(func, 1) do 20 | func.(socket) 21 | end 22 | 23 | def mount(_, _, socket) do 24 | {:ok, assign(socket, lc_module: nil, lc_attrs: %{})} 25 | end 26 | 27 | ## Test Helpers 28 | 29 | def run(lv, func) do 30 | GenServer.call(lv.pid, {:run, func}) 31 | end 32 | end 33 | 34 | ## Test helpers 35 | require Phoenix.LiveViewTest 36 | 37 | @doc """ 38 | Spawns a Driver process to mount a LiveComponent in isolation as the sole rendered element. 39 | ## Examples 40 | Starting a LiveComponent under test: 41 | {:ok, lcd, html} = LiveComponentTest.live_component_isolated(conn, MyComponent) 42 | Starting a LiveComponent under test with attributes: 43 | {:ok, lcd, html} = LiveComponentTest.live_component_isolated(conn, MyComponent, foo: :bar) 44 | """ 45 | defmacro live_component_isolated(conn, module, attrs \\ []) do 46 | quote bind_quoted: binding() do 47 | # Starts the Driver LiveView. It will render empty until we give it a `@module`. 48 | {:ok, lcd, _html} = Phoenix.LiveViewTest.live_isolated(conn, Driver) 49 | 50 | # <.live_component> requires an :id, so we set one if it's not already included. 51 | attrs = attrs |> Map.new() |> Map.put_new(:id, module) 52 | 53 | # Runs the given function _in the LiveView process_. 54 | Driver.run(lcd, fn socket -> 55 | {:reply, :ok, Phoenix.Component.assign(socket, lc_module: module, lc_attrs: attrs)} 56 | end) 57 | 58 | {:ok, lcd, Phoenix.LiveViewTest.render(lcd)} 59 | end 60 | end 61 | 62 | @doc """ 63 | Intercepts messages on the LiveComponentTest LiveView. 64 | Use this function to intercept messages sent by the LiveComponent to the LiveView. 65 | ## Examples 66 | {:ok, lcd, _html} = LiveComponentTest.live_component_isolated(conn, MyLiveComponent) 67 | test_pid = self() 68 | live_component_test_intercept(lv, fn 69 | :message_to_intercept, socket -> 70 | send(test_pid, :intercepted) 71 | {:halt, socket} 72 | _other, socket -> 73 | {:cont, socket} 74 | end) 75 | assert_received :intercepted 76 | """ 77 | def live_component_intercept(lv, func) when is_function(func) do 78 | Driver.run(lv, fn socket -> 79 | name = :"lcd_intercept_#{System.unique_integer([:positive, :monotonic])}" 80 | ref = {:intercept, lv, name, :handle_info} 81 | {:reply, ref, Phoenix.LiveView.attach_hook(socket, name, :handle_info, func)} 82 | end) 83 | end 84 | 85 | @doc """ 86 | Removes an intercept from the LiveComponentTest LiveView. 87 | ## Examples 88 | ref = LiveComponentTest.intercept(lv, fn msg, socket -> {:halt, socket} end) 89 | :ok = LiveComponentTest.remove_intercept(ref) 90 | """ 91 | def live_component_remove_intercept({:intercept, lv, name, stage}) do 92 | Driver.run(lv, fn socket -> 93 | {:reply, :ok, Phoenix.LiveView.detach_hook(socket, name, stage)} 94 | end) 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /apps/gust_web/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.1.25" 5 | 6 | def project do 7 | [ 8 | app: :gust_web, 9 | version: @version, 10 | build_path: "../../_build", 11 | config_path: "../../config/config.exs", 12 | deps_path: "../../deps", 13 | lockfile: "../../mix.lock", 14 | elixir: "~> 1.15", 15 | elixirc_paths: elixirc_paths(Mix.env()), 16 | start_permanent: Mix.env() == :prod, 17 | aliases: aliases(), 18 | deps: deps(), 19 | compilers: [:phoenix_live_view] ++ Mix.compilers(), 20 | listeners: [Phoenix.CodeReloader], 21 | test_coverage: [tool: ExCoveralls], 22 | description: "The web interface for Gust", 23 | package: [ 24 | licenses: ["Apache-2.0"], 25 | links: %{"GitHub" => "https://github.com/marciok/gust"}, 26 | files: [ 27 | "lib", 28 | "mix.exs", 29 | "README.md" 30 | ] 31 | ] 32 | ] 33 | end 34 | 35 | # Configuration for the OTP application. 36 | # 37 | # Type `mix help compile.app` for more information. 38 | def application do 39 | [ 40 | mod: {GustWeb.Application, []}, 41 | extra_applications: [:logger, :runtime_tools] 42 | ] 43 | end 44 | 45 | # Specifies which paths to compile per environment. 46 | defp elixirc_paths(:test), do: ["lib", "test/support"] 47 | defp elixirc_paths(_), do: ["lib"] 48 | 49 | # Specifies your project dependencies. 50 | # 51 | # Type `mix help deps` for examples and options. 52 | defp deps do 53 | [ 54 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 55 | {:phoenix, "~> 1.8.0"}, 56 | {:phoenix_ecto, "~> 4.5"}, 57 | {:phoenix_html, "~> 4.1"}, 58 | {:phoenix_live_reload, "~> 1.2", only: :dev}, 59 | {:phoenix_live_view, "~> 1.1.0"}, 60 | {:lazy_html, ">= 0.1.0", only: :test}, 61 | {:phoenix_live_dashboard, "~> 0.8.3"}, 62 | {:esbuild, "~> 0.10", runtime: Mix.env() == :dev}, 63 | {:tailwind, "~> 0.3", runtime: Mix.env() == :dev}, 64 | {:telemetry_metrics, "~> 1.0"}, 65 | {:telemetry_poller, "~> 1.0"}, 66 | {:gettext, "~> 0.26"}, 67 | gust_dep(), 68 | {:jason, "~> 1.2"}, 69 | {:bandit, "~> 1.5"} 70 | ] 71 | |> maybe_add_heroicons() 72 | end 73 | 74 | defp gust_dep() do 75 | if publish_dep?() do 76 | {:gust, "#{@version}"} 77 | else 78 | {:gust, in_umbrella: true} 79 | end 80 | end 81 | 82 | defp maybe_add_heroicons(deps) do 83 | if publish_dep?() do 84 | deps 85 | else 86 | deps ++ 87 | [ 88 | {:heroicons, 89 | github: "tailwindlabs/heroicons", 90 | tag: "v2.2.0", 91 | sparse: "optimized", 92 | app: false, 93 | compile: false, 94 | depth: 1} 95 | ] 96 | end 97 | end 98 | 99 | defp publish_dep?(), do: System.get_env("PUBLISH_DEP") == "true" 100 | 101 | # Aliases are shortcuts or tasks specific to the current project. 102 | # 103 | # See the documentation for `Mix` for more info on aliases. 104 | defp aliases do 105 | [ 106 | setup: ["deps.get", "assets.setup", "assets.build"], 107 | test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"], 108 | "assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"], 109 | "assets.build": ["tailwind gust_web", "esbuild gust_web"], 110 | "assets.deploy": [ 111 | "tailwind gust_web --minify", 112 | "esbuild gust_web --minify", 113 | "phx.digest" 114 | ] 115 | ] 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /apps/gust/test/dag/logger/database_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.Logger.DatabaseTest do 2 | require Logger 3 | use Gust.DataCase 4 | import Gust.FlowsFixtures 5 | alias Gust.DAG.Logger.Database 6 | alias Gust.Flows 7 | alias Gust.PubSub 8 | import ExUnit.CaptureLog 9 | 10 | describe "set_task/1" do 11 | setup do 12 | attempt = 1..10 |> Enum.random() 13 | task_name = "hi" 14 | dag = dag_fixture() 15 | run = run_fixture(%{dag_id: dag.id}) 16 | task = task_fixture(%{run_id: run.id, name: task_name, attempt: attempt}) 17 | 18 | log_content = "boogie" 19 | %{task: task, log_content: log_content} 20 | end 21 | 22 | test "do not for non task log", %{task: task, log_content: log_content} do 23 | task_id = task.id 24 | 25 | {:ok, log} = 26 | with_log(fn -> 27 | Logger.warning(log_content) 28 | Process.sleep(20) 29 | end) 30 | 31 | refute log =~ ":gen_event handler Gust.DAG.Logger.Database installed in Logger terminating" 32 | 33 | assert Flows.get_task_with_logs!(task_id).logs == [] 34 | end 35 | 36 | test "log for nil messages", %{task: task} do 37 | task_id = task.id 38 | task_attempt = task.attempt 39 | Database.set_task(task.id, task_attempt) 40 | PubSub.subscribe_task(task_id) 41 | 42 | {:ok, _log} = 43 | with_log(fn -> 44 | Logger.warning("") 45 | Process.sleep(20) 46 | end) 47 | 48 | [task_log] = Flows.get_task_with_logs!(task_id).logs 49 | assert task_log.content == "nil or empty was logged!" 50 | assert task_log.level == "error" 51 | end 52 | 53 | test "log for list messages", %{task: task} do 54 | task_id = task.id 55 | task_attempt = task.attempt 56 | Database.set_task(task.id, task_attempt) 57 | PubSub.subscribe_task(task_id) 58 | 59 | {:ok, _log} = 60 | with_log(fn -> 61 | Logger.warning(["hello", "world"]) 62 | Process.sleep(20) 63 | end) 64 | 65 | [task_log] = Flows.get_task_with_logs!(task_id).logs 66 | assert task_log.content == "hello; world" 67 | end 68 | 69 | test "log for maps messages", %{task: task} do 70 | task_id = task.id 71 | task_attempt = task.attempt 72 | Database.set_task(task.id, task_attempt) 73 | PubSub.subscribe_task(task_id) 74 | msg = %{"hello" => "world"} 75 | 76 | {:ok, _log} = 77 | with_log(fn -> 78 | Logger.warning(msg) 79 | Process.sleep(20) 80 | end) 81 | 82 | [task_log] = Flows.get_task_with_logs!(task_id).logs 83 | assert task_log.content == "[{\"hello\", \"world\"}]" 84 | end 85 | 86 | test "creates a warning log for task", %{task: task, log_content: log_content} do 87 | task_id = task.id 88 | task_attempt = task.attempt 89 | Database.set_task(task.id, task_attempt) 90 | PubSub.subscribe_task(task_id) 91 | 92 | {:ok, _log} = 93 | with_log(fn -> 94 | Logger.warning(log_content) 95 | Process.sleep(20) 96 | end) 97 | 98 | Database.unset() 99 | 100 | assert [%Flows.Log{level: "warn", content: ^log_content, attempt: ^task_attempt}] = 101 | logs = 102 | Flows.get_task_with_logs!(task_id).logs 103 | 104 | log_id = List.first(logs).id 105 | assert_receive {:task, :log, %{task_id: ^task_id, log_id: ^log_id}} 106 | 107 | assert :ok = Logger.configure_backend(Database, foo: :bar) 108 | end 109 | end 110 | 111 | describe "unset" do 112 | test "remove logger metadata" do 113 | Logger.metadata(task_id: 123) 114 | Database.unset() 115 | 116 | assert Logger.metadata() == [] 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/loader/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Loader.Worker do 2 | @behaviour Gust.DAG.Loader 3 | @moduledoc false 4 | alias Gust.DAG.{Parser, RunRestarter, Scheduler} 5 | alias Gust.Flows 6 | alias Gust.PubSub 7 | use GenServer 8 | require Logger 9 | 10 | @impl true 11 | def init(args) do 12 | {:ok, args, {:continue, :bootstrap}} 13 | end 14 | 15 | def child_spec(arg) do 16 | %{ 17 | id: __MODULE__, 18 | start: {__MODULE__, :start_link, [arg]}, 19 | restart: :transient, 20 | type: :worker 21 | } 22 | end 23 | 24 | def start_link(args) do 25 | GenServer.start_link(__MODULE__, args, name: __MODULE__) 26 | end 27 | 28 | @impl true 29 | def get_definitions do 30 | GenServer.call(__MODULE__, :get_definitions) 31 | end 32 | 33 | @impl true 34 | def get_definition(dag_id) do 35 | GenServer.call(__MODULE__, {:get_definition, dag_id}) 36 | end 37 | 38 | @impl true 39 | def handle_info( 40 | {dag_name, {:error, _error} = parse_result, "removed"}, 41 | %{dag_defs: dag_defs} = state 42 | ) do 43 | dag = Flows.get_dag_by_name(dag_name) 44 | removed_dag = Flows.delete_dag!(dag) 45 | dag_defs = Map.delete(dag_defs, removed_dag.id) 46 | 47 | state |> apply_dag_def_update(dag.name, parse_result, dag_defs, "removed") 48 | end 49 | 50 | @impl true 51 | def handle_info( 52 | {dag_name, {:error, _error} = parse_result, "reload"}, 53 | %{dag_defs: dag_defs} = state 54 | ) do 55 | case Flows.get_dag_by_name(dag_name) do 56 | %Flows.Dag{id: id, name: name} -> 57 | dag_defs = Map.put(dag_defs, id, parse_result) 58 | state |> apply_dag_def_update(name, parse_result, dag_defs, "reload") 59 | 60 | nil -> 61 | {:noreply, state} 62 | end 63 | end 64 | 65 | @impl true 66 | def handle_info( 67 | {dag_name, {:ok, _dag_def} = parse_result, "reload"}, 68 | %{dag_defs: dag_defs} = state 69 | ) do 70 | dag = get_or_create_dag(dag_name) 71 | dag_defs = Map.put(dag_defs, dag.id, parse_result) 72 | 73 | state |> apply_dag_def_update(dag_name, parse_result, dag_defs, "reload") 74 | end 75 | 76 | @impl true 77 | def handle_call(:get_definitions, _from, state) do 78 | {:reply, state[:dag_defs], state} 79 | end 80 | 81 | @impl true 82 | def handle_call({:get_definition, dag_id}, _from, state) do 83 | {:reply, state[:dag_defs][dag_id], state} 84 | end 85 | 86 | @impl true 87 | def handle_continue(:bootstrap, %{dags_folder: folder} = state) do 88 | dag_defs = load_folder(folder) 89 | Flows.delete_not_found_ids(Map.keys(dag_defs)) 90 | 91 | Scheduler.schedule(dag_defs) 92 | RunRestarter.restart_dags(dag_defs) 93 | 94 | {:noreply, state |> put_dag_defs(dag_defs)} 95 | end 96 | 97 | defp apply_dag_def_update(state, name, parse_result, dag_defs, action) do 98 | state = state |> put_dag_defs(dag_defs) 99 | PubSub.broadcast_file_update(name, parse_result, action) 100 | {:noreply, state} 101 | end 102 | 103 | defp put_dag_defs(state, dag_defs) do 104 | Map.put(state, :dag_defs, dag_defs) 105 | end 106 | 107 | defp load_folder(folder) do 108 | Parser.parse_folder(folder) 109 | |> Enum.map(fn {name, parser_result} -> 110 | dag = get_or_create_dag(name) 111 | {dag.id, parser_result} 112 | end) 113 | |> Map.new() 114 | end 115 | 116 | def get_or_create_dag(name) do 117 | case Flows.get_dag_by_name(name) do 118 | %Flows.Dag{} = dag -> 119 | Logger.warning("FOUND DAG: #{name}") 120 | dag 121 | 122 | nil -> 123 | {:ok, dag} = Flows.create_dag(%{name: name}) 124 | Logger.warning("CREATED DAG: #{name}") 125 | dag 126 | end 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # Configure your database 4 | config :gust, Gust.Repo, 5 | hostname: System.get_env("PGHOST"), 6 | username: System.get_env("PGUSER"), 7 | password: System.get_env("PGPASSWORD"), 8 | database: "gust_rc_dev", 9 | stacktrace: true, 10 | show_sensitive_data_on_connection_error: true, 11 | pool_size: 10 12 | 13 | # For development, we disable any cache and enable 14 | # debugging and code reloading. 15 | # 16 | # The watchers configuration can be used to run external 17 | # watchers to your application. For example, we can use it 18 | # to bundle .js and .css sources. 19 | config :gust_web, GustWeb.Endpoint, 20 | # Binding to loopback ipv4 address prevents access from other machines. 21 | # Change to `ip: {0, 0, 0, 0}` to allow access from other machines. 22 | http: [ip: {127, 0, 0, 1}, port: String.to_integer(System.get_env("PORT") || "4000")], 23 | check_origin: false, 24 | code_reloader: true, 25 | debug_errors: true, 26 | secret_key_base: System.get_env("SECRET_KEY_BASE"), 27 | watchers: [ 28 | esbuild: {Esbuild, :install_and_run, [:gust_web, ~w(--sourcemap=inline --watch)]}, 29 | tailwind: {Tailwind, :install_and_run, [:gust_web, ~w(--watch)]} 30 | ] 31 | 32 | # ## SSL Support 33 | # 34 | # In order to use HTTPS in development, a self-signed 35 | # certificate can be generated by running the following 36 | # Mix task: 37 | # 38 | # mix phx.gen.cert 39 | # 40 | # Run `mix help phx.gen.cert` for more information. 41 | # 42 | # The `http:` config above can be replaced with: 43 | # 44 | # https: [ 45 | # port: 4001, 46 | # cipher_suite: :strong, 47 | # keyfile: "priv/cert/selfsigned_key.pem", 48 | # certfile: "priv/cert/selfsigned.pem" 49 | # ], 50 | # 51 | # If desired, both `http:` and `https:` keys can be 52 | # configured to run both http and https servers on 53 | # different ports. 54 | 55 | # Watch static and templates for browser reloading. 56 | config :gust_web, GustWeb.Endpoint, 57 | live_reload: [ 58 | web_console_logger: true, 59 | patterns: [ 60 | ~r"priv/static/(?!uploads/).*(js|css|png|jpeg|jpg|gif|svg)$", 61 | ~r"priv/gettext/.*(po)$", 62 | ~r"lib/gust_web/(?:controllers|live|components|router)/?.*\.(ex|heex)$" 63 | ] 64 | ] 65 | 66 | # Enable dev routes for dashboard and mailbox 67 | config :gust_web, dev_routes: true 68 | 69 | # Do not include metadata nor timestamps in development logs 70 | config :logger, :default_formatter, format: "[$level] $message\n" 71 | 72 | # Initialize plugs at runtime for faster development compilation 73 | config :phoenix, :plug_init_mode, :runtime 74 | 75 | config :phoenix_live_view, 76 | # Include debug annotations and locations in rendered markup. 77 | # Changing this configuration will require mix clean and a full recompile. 78 | debug_heex_annotations: true, 79 | debug_attributes: true, 80 | # Enable helpful, but potentially expensive runtime checks 81 | enable_expensive_runtime_checks: true 82 | 83 | # Disable swoosh api client as it is only required for production adapters. 84 | config :swoosh, :api_client, false 85 | 86 | # Set a higher stacktrace during development. Avoid configuring such 87 | # in production as building large stacktraces may be expensive. 88 | config :phoenix, :stacktrace_depth, 20 89 | 90 | config :gust, file_reload_delay: 1_000 91 | config :gust, b64_secrets_cloak_key: System.get_env("B64_SECRETS_CLOAK_KEY") 92 | config :gust, dags_folder: Path.join(File.cwd!(), "dags") 93 | config :gust, dag_runner_supervisor: Gust.DAG.RunnerSupervisor.DynamicSupervisor 94 | config :gust, dag_task_runner_supervisor: Gust.DAG.TaskRunnerSupervisor.DynamicSupervisor 95 | config :gust, dag_stage_runner_supervisor: Gust.DAG.StageRunnerSupervisor.DynamicSupervisor 96 | config :gust, dag_scheduler: Gust.DAG.Scheduler.Worker 97 | config :gust, dag_loader: Gust.DAG.Loader.Worker 98 | config :gust, dag_stage_runner: Gust.DAG.Runner.StageWorker 99 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gust 2 | 3 | ![Gust Logo](https://gust-github.s3.us-east-1.amazonaws.com/gust-symbol-logo.png) 4 | 5 | 6 | 7 | A task orchestration system designed to be efficient, fast and developer-friendly. 8 | 9 | 10 | [![Test](https://github.com/marciok/gust/actions/workflows/test.yml/badge.svg)](https://github.com/marciok/gust/actions/workflows/test.yml)[![Coverage Status](https://coveralls.io/repos/github/marciok/gust/badge.svg?branch=main)](https://coveralls.io/github/marciok/gust?branch=main) 11 | 12 | [![Gust Web](https://img.shields.io/hexpm/v/gust_web?color=0084d1&label=Gust+Web)](https://hexdocs.pm/gust_web) 13 | 14 | [![Gust](https://img.shields.io/hexpm/v/gust_web?color=0084d1&label=Gust)](https://hexdocs.pm/gust) 15 | 16 | --- 17 | ## Motivation 18 | As a CTO and founder, I was tired of spending buckets of money to set up and manage [Airflow](https://airflow.apache.org/), dealing with multiple databases, countless processes, Docker complexity, and of course its outdated and buggy UI. So we decided to build something that kept what we liked about Airflow and ditched what we didn’t. The result is Gust: a platform that’s 10× more efficient, faster, and far easier to set up. 19 | 20 | Gust is the perfect fit for our needs, and I encourage you to try it and push it even further. There’s still plenty of room for improvements and new features. If you spot something or want to contribute an idea, don’t be shy! Drop an Issue or submit a PR. 21 | 22 | --- 23 | ## Overview 24 | 25 | ### DAG Code Example 26 | ```elixir 27 | defmodule HelloWorld do 28 | alias Gust.Flows 29 | use Gust.DSL 30 | require Logger 31 | 32 | task :first_task, downstream: [:second_task], store_result: true do 33 | greetings = "Hi from first_task" 34 | Logger.info(greetings) 35 | %{result: greetings} 36 | end 37 | 38 | task :second_task, ctx: %{run_id: run_id} do 39 | task = Flows.get_task_by_name_run("first_task", run_id) 40 | Logger.info(task.result) 41 | end 42 | end 43 | 44 | ``` 45 | 46 | ### Web Interface 47 | 48 | ![ss-1](https://gust-github.s3.us-east-1.amazonaws.com/gust-ss-1.png) 49 | 50 | ![ss2](https://gust-github.s3.us-east-1.amazonaws.com/gust-ss-2.png) 51 | --- 52 | 53 | 54 | 55 | 56 | 57 | ## Getting started 58 | 59 | ### Prerequisites 60 | 61 | - [x] macOS/Ubuntu 62 | - [x] Elixir must be at least [this version](https://github.com/marciok/gust/blob/main/.tool-versions) 63 | - [x] Postgres 64 | 65 | 66 | ## Creating a new Gust app 67 | 68 | 1. Replace `my_app` for your app name and run: 69 | 70 | ``` 71 | GUST_APP=my_app bash -c "$(curl -fsSL https://raw.githubusercontent.com/marciok/gust/main/setup_gust_app.sh)" 72 | 73 | ``` 74 | 75 | 2. Configure Postgres credentials on `my_app/config/dev.exs` 76 | 77 | 3. Run database setup: 78 | `mix ecto.create --repo Gust.Repo && mix ecto.migrate --repo Gust.Repo` 79 | 80 | 4. Run Gust start: 81 | `mix gust.start` 82 | 83 | 5. Check [the docs](https://hexdocs.pm/gust/Gust.DSL.html) on how to customize your DAG 🎉 84 | 85 | 86 | --- 87 | 88 | ### Core Features 89 | 90 | - Task orchestration with Cron-style scheduling and dependency-aware DAGs via the Gust DSL. 91 | - Manual task controls: stop running tasks, cancel retries, and restart tasks on demand. 92 | - Run-time tracking, corrupted-state recovery, and graceful handling of syntax errors during development. 93 | - Retry logic with backoff, plus state clearing for clean restarts. 94 | - Hook for finished dag run. 95 | - Web UI for live monitoring and secrets editing. 96 | 97 | 98 | 99 | --- 100 | ### Sponsors 101 | 102 | 103 | ![Comparacar](https://gust-github.s3.us-east-1.amazonaws.com/comparacar-sponsor-v2.jpg) 104 | 105 | 106 | [Find the best offers and save money on car subscription service.](https://comparacar.com.br) 107 | 108 | 109 | ## License 110 | 111 | Gust is released under the MIT License. 112 | 113 | 114 | --- 115 | 116 | ![No more Astronomer hefty bills](https://gust-github.s3.us-east-1.amazonaws.com/gust-airflow.png) 117 | -------------------------------------------------------------------------------- /apps/gust_web/assets/js/app.js: -------------------------------------------------------------------------------- 1 | // If you want to use Phoenix channels, run `mix help phx.gen.channel` 2 | // to get started and then uncomment the line below. 3 | // import "./user_socket.js" 4 | 5 | // You can include dependencies in two ways. 6 | // 7 | // The simplest option is to put them in assets/vendor and 8 | // import them using relative paths: 9 | // 10 | // import "../vendor/some-package.js" 11 | // 12 | // Alternatively, you can `npm install some-package --prefix assets` and import 13 | // them using a path starting with the package name: 14 | // 15 | // import "some-package" 16 | // 17 | // If you have dependencies that try to import CSS, esbuild will generate a separate `app.css` file. 18 | // To load it, simply add a second `` to your `root.html.heex` file. 19 | 20 | // Include phoenix_html to handle method=PUT/DELETE in forms and buttons. 21 | import "phoenix_html" 22 | // Establish Phoenix Socket and LiveView configuration. 23 | import { Socket } from "phoenix" 24 | import { LiveSocket } from "phoenix_live_view" 25 | import { hooks as colocatedHooks } from "phoenix-colocated/gust_web" 26 | import topbar from "../vendor/topbar" 27 | import mermaid from "mermaid" 28 | 29 | import Prism from "prismjs" 30 | import "prismjs/components/prism-elixir.js" 31 | import "prismjs/plugins/line-numbers/prism-line-numbers.js" 32 | import "prismjs/plugins/line-highlight/prism-line-highlight.js" 33 | 34 | let Hooks = {} 35 | 36 | Hooks.Mermaid = { 37 | 38 | mounted() { 39 | this.initializeMermaid() 40 | }, 41 | updated() { 42 | this.initializeMermaid() 43 | }, 44 | 45 | initializeMermaid() { 46 | mermaid.contentLoaded() 47 | }, 48 | } 49 | 50 | Hooks.CodeHighlight = { 51 | mounted() { 52 | Prism.highlightElement(this.el) 53 | }, 54 | updated() { 55 | Prism.highlightElement(this.el) 56 | } 57 | } 58 | 59 | const csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("content") 60 | const liveSocket = new LiveSocket("/live", Socket, { 61 | longPollFallbackMs: 2500, 62 | params: { _csrf_token: csrfToken }, 63 | hooks: { ...colocatedHooks, ...Hooks }, 64 | }) 65 | 66 | // Show progress bar on live navigation and form submits 67 | topbar.config({ barColors: { 0: "#29d" }, shadowColor: "rgba(0, 0, 0, .3)" }) 68 | window.addEventListener("phx:page-loading-start", _info => topbar.show(300)) 69 | window.addEventListener("phx:page-loading-stop", _info => topbar.hide()) 70 | 71 | // connect if there are any LiveViews on the page 72 | liveSocket.connect() 73 | 74 | // expose liveSocket on window for web console debug logs and latency simulation: 75 | // >> liveSocket.enableDebug() 76 | // >> liveSocket.enableLatencySim(1000) // enabled for duration of browser session 77 | // >> liveSocket.disableLatencySim() 78 | window.liveSocket = liveSocket 79 | 80 | // The lines below enable quality of life phoenix_live_reload 81 | // development features: 82 | // 83 | // 1. stream server logs to the browser console 84 | // 2. click on elements to jump to their definitions in your code editor 85 | // 86 | if (process.env.NODE_ENV === "development") { 87 | window.addEventListener("phx:live_reload:attached", ({ detail: reloader }) => { 88 | // Enable server log streaming to client. 89 | // Disable with reloader.disableServerLogs() 90 | reloader.enableServerLogs() 91 | 92 | // Open configured PLUG_EDITOR at file:line of the clicked element's HEEx component 93 | // 94 | // * click with "c" key pressed to open at caller location 95 | // * click with "d" key pressed to open at function component definition location 96 | let keyDown 97 | window.addEventListener("keydown", e => keyDown = e.key) 98 | window.addEventListener("keyup", e => keyDown = null) 99 | window.addEventListener("click", e => { 100 | if (keyDown === "c") { 101 | e.preventDefault() 102 | e.stopImmediatePropagation() 103 | reloader.openEditorAtCaller(e.target) 104 | } else if (keyDown === "d") { 105 | e.preventDefault() 106 | e.stopImmediatePropagation() 107 | reloader.openEditorAtDef(e.target) 108 | } 109 | }, true) 110 | 111 | window.liveReloader = reloader 112 | }) 113 | } 114 | 115 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/parser/file.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.Parser.File do 2 | @moduledoc false 3 | 4 | @behaviour Gust.DAG.Parser 5 | 6 | alias Gust.DAG.Definition 7 | alias Gust.DAG.Graph 8 | 9 | @impl true 10 | def parse_folder(folder) do 11 | ex_files(folder) 12 | |> Enum.map(&"#{Path.absname(folder)}/#{&1}") 13 | |> Enum.map(fn path -> 14 | name = Path.basename(path, ".ex") 15 | {name, parse(path)} 16 | end) 17 | end 18 | 19 | @impl true 20 | def parse(file_path) do 21 | if File.exists?(file_path) do 22 | parse_file(file_path) 23 | else 24 | {:error, :enoent} 25 | end 26 | end 27 | 28 | defp parse_file(file_path) do 29 | with {:ok, ast} <- quote_content(file_path), true <- use_dsl?(ast) do 30 | define_dag(file_path) 31 | else 32 | false -> 33 | error = {[], "use Gust.DSL not found", ""} 34 | {:error, error} 35 | 36 | {:error, erros} -> 37 | {:error, erros} 38 | end 39 | end 40 | 41 | defp quote_content(path) do 42 | content = File.read!(path) 43 | Code.string_to_quoted(content) 44 | end 45 | 46 | defp define_dag(file_path) do 47 | name = Path.basename(file_path, ".ex") 48 | dag_def = default_dag_def(name, file_path) 49 | 50 | dag_def = 51 | case compile(file_path) do 52 | {:error, error, messages} -> 53 | %{dag_def | error: error, messages: messages} 54 | 55 | {:ok, {mod, opts, all_tasks}, warnings} -> 56 | task_list = build_task_list(mod) 57 | 58 | tasks = Graph.link_tasks(all_tasks) |> put_store_result(all_tasks) 59 | 60 | stages = build_stages(mod) 61 | 62 | :code.purge(mod) 63 | :code.delete(mod) 64 | 65 | %{ 66 | dag_def 67 | | mod: mod, 68 | messages: warnings, 69 | tasks: tasks, 70 | task_list: task_list, 71 | options: opts, 72 | stages: stages 73 | } 74 | end 75 | 76 | {:ok, dag_def} 77 | end 78 | 79 | defp default_dag_def(name, file_path) do 80 | %Definition{name: name, file_path: file_path} 81 | end 82 | 83 | defp put_store_result(tasks, all_tasks) do 84 | for {t_name, opts} <- tasks, into: %{} do 85 | {t_name, Map.put(opts, :store_result, all_tasks[String.to_atom(t_name)][:store_result])} 86 | end 87 | end 88 | 89 | defp build_stages(mod) do 90 | list_tasks!(mod) 91 | |> Graph.link_tasks() 92 | |> Graph.to_stages() 93 | |> then(fn {:ok, stages} -> stages end) 94 | end 95 | 96 | defp build_task_list(mod) do 97 | build_stages(mod) 98 | |> List.flatten() 99 | end 100 | 101 | defp options!(mod) do 102 | opts = mod.__dag_options__() 103 | Keyword.validate!(opts, [:schedule, :on_finished_callback]) 104 | end 105 | 106 | defp list_tasks!(mod) do 107 | tasks = mod.__dag_tasks__() 108 | 109 | tasks 110 | |> Enum.each(fn {_task_name, opts} -> 111 | Keyword.validate!(opts, [:downstream, :store_result, :ctx]) 112 | end) 113 | 114 | tasks 115 | end 116 | 117 | @impl true 118 | def maybe_ex_file(path) do 119 | if Path.extname(path) == ".ex", do: path, else: nil 120 | end 121 | 122 | defp use_dsl?(ast) do 123 | Macro.prewalker(ast) 124 | |> Enum.filter(fn 125 | {:use, _meta, [{:__aliases__, _, [:Gust, :DSL]} | _config]} -> 126 | true 127 | 128 | _node -> 129 | false 130 | end) 131 | |> length() > 0 132 | end 133 | 134 | defp compile(file) do 135 | code_result = 136 | Code.with_diagnostics(fn -> 137 | try do 138 | [{mod, _bin}] = Code.compile_file(file) 139 | opts = options!(mod) 140 | tasks = list_tasks!(mod) 141 | 142 | {:ok, mod, opts, tasks} 143 | rescue 144 | err -> {:error, err} 145 | end 146 | end) 147 | 148 | case code_result do 149 | {{:ok, dag_module, opts, tasks}, warnings} -> 150 | {:ok, {dag_module, opts, tasks}, warnings} 151 | 152 | {{:error, error_type}, errors} -> 153 | {:error, error_type, errors} 154 | end 155 | end 156 | 157 | defp ex_files(folder) do 158 | folder 159 | |> File.ls!() 160 | |> Enum.filter(&maybe_ex_file(&1)) 161 | end 162 | end 163 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dag/run_restarter/worker.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DAG.RunRestarter.Worker do 2 | @moduledoc false 3 | alias Gust.DAG.{Definition, Graph, Loader, RunnerSupervisor} 4 | alias Gust.Flows 5 | @behaviour Gust.DAG.RunRestarter 6 | use GenServer 7 | 8 | @impl true 9 | def init(init_arg) do 10 | {:ok, init_arg} 11 | end 12 | 13 | def child_spec(arg) do 14 | %{ 15 | id: __MODULE__, 16 | start: {__MODULE__, :start_link, [arg]}, 17 | type: :worker 18 | } 19 | end 20 | 21 | def start_link(args) do 22 | GenServer.start_link(__MODULE__, args, name: __MODULE__) 23 | end 24 | 25 | @impl true 26 | def start_dag(dag_id) do 27 | GenServer.call(__MODULE__, {:start_dag, dag_id}) 28 | end 29 | 30 | @impl true 31 | def restart_run(run) do 32 | GenServer.call(__MODULE__, {:restart_run, run}) 33 | end 34 | 35 | @impl true 36 | def restart_task(graph, task) do 37 | GenServer.call(__MODULE__, {:restart_task, graph, task}) 38 | end 39 | 40 | @impl true 41 | def restart_dags(dags) do 42 | GenServer.call(__MODULE__, {:restart_dags, dags}) 43 | end 44 | 45 | @impl true 46 | def restart_enqueued(dag_id) do 47 | GenServer.cast(__MODULE__, {:restart_enqueued, dag_id}) 48 | end 49 | 50 | @impl true 51 | def handle_cast({:restart_enqueued, dag_id}, state) do 52 | with {:ok, dag_def} <- Loader.get_definition(dag_id), 53 | true <- Definition.empty_errors?(dag_def) do 54 | get_runs([dag_id], [:enqueued]) |> Enum.each(&start_run(&1, {:ok, dag_def})) 55 | end 56 | 57 | {:noreply, state} 58 | end 59 | 60 | @impl true 61 | def handle_call({:start_dag, dag_id}, _from, state) do 62 | {:ok, run} = Flows.create_run(%{dag_id: dag_id}) 63 | 64 | run = 65 | if Flows.get_dag!(dag_id).enabled do 66 | with {:ok, dag_def} <- Loader.get_definition(run.dag_id), 67 | true <- Definition.empty_errors?(dag_def) do 68 | {:ok, _pid} = RunnerSupervisor.start_child(run, dag_def) 69 | run 70 | else 71 | false -> nil 72 | end 73 | else 74 | {:ok, run} = Flows.update_run_status(run, :enqueued) 75 | run 76 | end 77 | 78 | {:reply, run, state} 79 | end 80 | 81 | @impl true 82 | def handle_call({:restart_run, run}, _from, state) do 83 | Flows.get_run_with_tasks!(run.id) 84 | |> then(fn run -> run.tasks end) 85 | |> Enum.each(fn task -> 86 | {:ok, _task} = Flows.update_task_status(task, :created) 87 | end) 88 | 89 | {:ok, run} = Flows.update_run_status(run, :running) 90 | 91 | {:ok, dag_def} = Loader.get_definition(run.dag_id) 92 | {:ok, _pid} = RunnerSupervisor.start_child(run, dag_def) 93 | 94 | {:reply, run, state} 95 | end 96 | 97 | @impl true 98 | def handle_call({:restart_task, graph, task}, _from, state) do 99 | tasks_to_clear = 100 | graph 101 | |> Graph.build_branch(:downstream, task.name) 102 | |> List.flatten() 103 | |> MapSet.new() 104 | |> Enum.map(fn t_name -> 105 | t = Flows.get_task_by_name_run(t_name, task.run_id) 106 | 107 | {:ok, t} = Flows.update_task_status(t, :created) 108 | t 109 | end) 110 | 111 | run = Flows.get_run!(task.run_id) 112 | {:ok, run} = Flows.update_run_status(run, :running) 113 | 114 | {:ok, dag_def} = Loader.get_definition(run.dag_id) 115 | {:ok, _pid} = RunnerSupervisor.start_child(run, dag_def) 116 | 117 | {:reply, tasks_to_clear, state} 118 | end 119 | 120 | @impl true 121 | def handle_call({:restart_dags, dags}, _from, state) do 122 | dag_ids = Map.keys(dags) 123 | 124 | runs = 125 | get_runs(dag_ids, [:running, :retrying]) 126 | |> Stream.filter(fn run -> 127 | case dags[run.dag_id] do 128 | {:ok, dag_def} -> 129 | Definition.empty_errors?(dag_def) 130 | 131 | {:error, _err} -> 132 | false 133 | end 134 | end) 135 | |> Stream.map(fn run -> start_run(run, dags[run.dag_id]) end) 136 | |> Enum.to_list() 137 | 138 | {:reply, runs, state} 139 | end 140 | 141 | defp get_runs(dag_ids, statuses) do 142 | Flows.get_running_runs_by_dag(dag_ids, statuses) 143 | end 144 | 145 | defp start_run(run, {:ok, dag_def}) do 146 | {:ok, _pid} = RunnerSupervisor.start_child(run, dag_def) 147 | run 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /apps/gust_web/lib/gust_web/components/layouts.ex: -------------------------------------------------------------------------------- 1 | defmodule GustWeb.Layouts do 2 | @moduledoc """ 3 | This module holds layouts and related functionality 4 | used by your application. 5 | """ 6 | use GustWeb, :html 7 | 8 | # Embed all files in layouts/* within this module. 9 | # The default root.html.heex file contains the HTML 10 | # skeleton of your application, namely HTML headers 11 | # and other static content. 12 | embed_templates "layouts/*" 13 | 14 | @doc """ 15 | Renders your app layout. 16 | 17 | This function is typically invoked from every template, 18 | and it often contains your application menu, sidebar, 19 | or similar. 20 | 21 | ## Examples 22 | 23 | 24 |

Content

25 | 26 | 27 | """ 28 | attr :flash, :map, required: true, doc: "the map of flash messages" 29 | 30 | attr :current_scope, :map, 31 | default: nil, 32 | doc: "the current [scope](https://hexdocs.pm/phoenix/scopes.html)" 33 | 34 | slot :inner_block, required: true 35 | 36 | def app(assigns) do 37 | ~H""" 38 |
39 |
40 | 57 | 58 |
59 |
60 |
61 | {render_slot(@inner_block)} 62 |
63 |
64 |
65 |
66 | 67 |
68 |
69 |
70 | {System.get_env("MIX_ENV")} 71 |
72 |
73 |
74 | 75 | <.flash_group flash={@flash} /> 76 |
77 | """ 78 | end 79 | 80 | @doc """ 81 | Shows the flash group with standard titles and content. 82 | 83 | ## Examples 84 | 85 | <.flash_group flash={@flash} /> 86 | """ 87 | attr :flash, :map, required: true, doc: "the map of flash messages" 88 | attr :id, :string, default: "flash-group", doc: "the optional id of flash container" 89 | 90 | def flash_group(assigns) do 91 | ~H""" 92 |
97 | <.flash kind={:info} flash={@flash} /> 98 | <.flash kind={:error} flash={@flash} /> 99 | 100 | <.flash 101 | id="client-error" 102 | kind={:error} 103 | title={gettext("We can't find the internet")} 104 | phx-disconnected={show(".phx-client-error #client-error") |> JS.remove_attribute("hidden")} 105 | phx-connected={hide("#client-error") |> JS.set_attribute({"hidden", ""})} 106 | hidden 107 | > 108 | {gettext("Attempting to reconnect")} 109 | <.icon name="hero-arrow-path" class="ml-1 size-3 motion-safe:animate-spin" /> 110 | 111 | 112 | <.flash 113 | id="server-error" 114 | kind={:error} 115 | title={gettext("Something went wrong!")} 116 | phx-disconnected={show(".phx-server-error #server-error") |> JS.remove_attribute("hidden")} 117 | phx-connected={hide("#server-error") |> JS.set_attribute({"hidden", ""})} 118 | hidden 119 | > 120 | {gettext("Attempting to reconnect")} 121 | <.icon name="hero-arrow-path" class="ml-1 size-3 motion-safe:animate-spin" /> 122 | 123 |
124 | """ 125 | end 126 | 127 | @doc """ 128 | Provides dark vs light theme toggle based on themes defined in app.css. 129 | 130 | See in root.html.heex which applies the theme before page load. 131 | """ 132 | end 133 | -------------------------------------------------------------------------------- /config/runtime.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # config/runtime.exs is executed for all environments, including 4 | # during releases. It is executed after compilation and before the 5 | # system starts, so it is typically used to load production configuration 6 | # and secrets from environment variables or elsewhere. Do not define 7 | # any compile-time configuration in here, as it won't be applied. 8 | # The block below contains prod specific runtime configuration. 9 | if config_env() == :prod do 10 | database_url = 11 | System.get_env("DATABASE_URL") || 12 | raise """ 13 | environment variable DATABASE_URL is missing. 14 | For example: ecto://USER:PASS@HOST/DATABASE 15 | """ 16 | 17 | maybe_ipv6 = if System.get_env("ECTO_IPV6") in ~w(true 1), do: [:inet6], else: [] 18 | 19 | config :gust, Gust.Repo, 20 | # ssl: true, 21 | url: database_url, 22 | pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"), 23 | # For machines with several cores, consider starting multiple pools of `pool_size` 24 | # pool_count: 4, 25 | socket_options: maybe_ipv6 26 | 27 | # The secret key base is used to sign/encrypt cookies and other secrets. 28 | # A default value is used in config/dev.exs and config/test.exs but you 29 | # want to use a different value for prod and you most likely don't want 30 | # to check this value into version control, so we use an environment 31 | # variable instead. 32 | secret_key_base = 33 | System.get_env("SECRET_KEY_BASE") || 34 | raise """ 35 | environment variable SECRET_KEY_BASE is missing. 36 | You can generate one by calling: mix phx.gen.secret 37 | """ 38 | 39 | config :gust_web, GustWeb.Endpoint, 40 | http: [ 41 | # Enable IPv6 and bind on all interfaces. 42 | # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. 43 | ip: {0, 0, 0, 0, 0, 0, 0, 0}, 44 | port: String.to_integer(System.get_env("PORT") || "4000") 45 | ], 46 | secret_key_base: secret_key_base 47 | 48 | # ## Using releases 49 | # 50 | # If you are doing OTP releases, you need to instruct Phoenix 51 | # to start each relevant endpoint: 52 | # 53 | # config :gust_web, GustWeb.Endpoint, server: true 54 | # 55 | # Then you can assemble a release by calling `mix release`. 56 | # See `mix help release` for more information. 57 | 58 | # ## SSL Support 59 | # 60 | # To get SSL working, you will need to add the `https` key 61 | # to your endpoint configuration: 62 | # 63 | # config :gust_web, GustWeb.Endpoint, 64 | # https: [ 65 | # ..., 66 | # port: 443, 67 | # cipher_suite: :strong, 68 | # keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"), 69 | # certfile: System.get_env("SOME_APP_SSL_CERT_PATH") 70 | # ] 71 | # 72 | # The `cipher_suite` is set to `:strong` to support only the 73 | # latest and more secure SSL ciphers. This means old browsers 74 | # and clients may not be supported. You can set it to 75 | # `:compatible` for wider support. 76 | # 77 | # `:keyfile` and `:certfile` expect an absolute path to the key 78 | # and cert in disk or a relative path inside priv, for example 79 | # "priv/ssl/server.key". For all supported SSL configuration 80 | # options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1 81 | # 82 | # We also recommend setting `force_ssl` in your config/prod.exs, 83 | # ensuring no data is ever sent via http, always redirecting to https: 84 | # 85 | # config :gust_web, GustWeb.Endpoint, 86 | # force_ssl: [hsts: true] 87 | # 88 | # Check `Plug.SSL` for all available options in `force_ssl`. 89 | 90 | # ## Configuring the mailer 91 | # 92 | # In production you need to configure the mailer to use a different adapter. 93 | # Here is an example configuration for Mailgun: 94 | # 95 | # config :gust, Gust.Mailer, 96 | # adapter: Swoosh.Adapters.Mailgun, 97 | # api_key: System.get_env("MAILGUN_API_KEY"), 98 | # domain: System.get_env("MAILGUN_DOMAIN") 99 | # 100 | # Most non-SMTP adapters require an API client. Swoosh supports Req, Hackney, 101 | # and Finch out-of-the-box. This configuration is typically done at 102 | # compile-time in your config/prod.exs: 103 | # 104 | # config :swoosh, :api_client, Swoosh.ApiClient.Req 105 | # 106 | # See https://hexdocs.pm/swoosh/Swoosh.html#module-installation for details. 107 | 108 | config :gust, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY") 109 | end 110 | -------------------------------------------------------------------------------- /apps/gust/test/dag/runner/task_worker_test.exs: -------------------------------------------------------------------------------- 1 | defmodule DAG.Runner.TaskWorkerTest do 2 | require Logger 3 | use Gust.DataCase, async: false 4 | import Gust.FlowsFixtures 5 | 6 | import Mox 7 | 8 | setup :verify_on_exit! 9 | setup :set_mox_from_context 10 | 11 | setup do 12 | task_name = "hi" 13 | dag = dag_fixture() 14 | run = run_fixture(%{dag_id: dag.id}) 15 | task = task_fixture(%{run_id: run.id, name: task_name}) 16 | task_id = task.id 17 | task_attempt = task.attempt 18 | 19 | Gust.DAGLoggerMock 20 | |> expect(:set_task, fn ^task_id, ^task_attempt -> nil end) 21 | |> expect(:unset, fn -> nil end) 22 | 23 | %{task: task} 24 | end 25 | 26 | describe "handle_continue/2 when :init_run is given" do 27 | test "run task with context", %{task: task} do 28 | dag_content = """ 29 | defmodule DagToBeRun do 30 | def hi(args) do 31 | args 32 | end 33 | end 34 | """ 35 | 36 | [{mod, _bin}] = Code.compile_string(dag_content) 37 | 38 | run_id = task.run_id 39 | task_id = task.id 40 | 41 | worker_pid = 42 | start_link_supervised!( 43 | {Gust.DAG.Runner.TaskWorker, %{task: task, mod: mod, stage_pid: self(), opts: %{}}} 44 | ) 45 | 46 | ref = Process.monitor(worker_pid) 47 | assert_receive {:task_result, %{run_id: ^run_id}, ^task_id, :ok}, 200 48 | assert_receive {:DOWN, ^ref, :process, _pid, :normal}, 200 49 | 50 | on_exit(fn -> 51 | :code.purge(mod) 52 | :code.delete(mod) 53 | end) 54 | end 55 | 56 | test "run succeed", %{task: task} do 57 | task_id = task.id 58 | result = "i_am_done" 59 | 60 | dag_content = """ 61 | defmodule MySuccessfulDagOne do 62 | use Gust.DSL 63 | require Logger 64 | 65 | task :#{task.name} do 66 | Process.sleep(100) 67 | "#{result}" 68 | end 69 | end 70 | """ 71 | 72 | [{mod, _bin}] = Code.compile_string(dag_content) 73 | 74 | worker_pid = 75 | start_link_supervised!( 76 | {Gust.DAG.Runner.TaskWorker, %{task: task, mod: mod, stage_pid: self(), opts: %{}}} 77 | ) 78 | 79 | ref = Process.monitor(worker_pid) 80 | assert_receive {:task_result, ^result, ^task_id, :ok}, 200 81 | assert_receive {:DOWN, ^ref, :process, _pid, :normal}, 200 82 | 83 | on_exit(fn -> 84 | :code.purge(mod) 85 | :code.delete(mod) 86 | end) 87 | end 88 | 89 | test "run fails", %{task: task} do 90 | task_id = task.id 91 | error_message = "Ops.." 92 | 93 | dag_content = """ 94 | defmodule MySuccessfulDag do 95 | use Gust.DSL 96 | 97 | task :#{task.name} do 98 | Process.sleep(100) 99 | raise "#{error_message}" 100 | end 101 | end 102 | """ 103 | 104 | [{mod, _bin}] = Code.compile_string(dag_content) 105 | 106 | worker_pid = 107 | start_link_supervised!( 108 | {Gust.DAG.Runner.TaskWorker, %{task: task, mod: mod, stage_pid: self(), opts: %{}}} 109 | ) 110 | 111 | ref = Process.monitor(worker_pid) 112 | 113 | result = %RuntimeError{message: error_message, __exception__: true} 114 | 115 | assert_receive {:task_result, ^result, ^task_id, :error}, 200 116 | assert_receive {:DOWN, ^ref, :process, _pid, :normal}, 200 117 | 118 | on_exit(fn -> 119 | :code.purge(mod) 120 | :code.delete(mod) 121 | end) 122 | end 123 | 124 | test "store result is set but type is not map", %{task: task} do 125 | task_id = task.id 126 | error_message = "Task returned :i_am_no_map but store_result requires a map" 127 | 128 | dag_content = """ 129 | defmodule MySuccessfulDag do 130 | use Gust.DSL 131 | 132 | task :#{task.name}, store_result: true do 133 | Process.sleep(100) 134 | :i_am_no_map 135 | end 136 | end 137 | """ 138 | 139 | [{mod, _bin}] = Code.compile_string(dag_content) 140 | 141 | worker_pid = 142 | start_link_supervised!( 143 | {Gust.DAG.Runner.TaskWorker, 144 | %{task: task, mod: mod, stage_pid: self(), opts: %{store_result: true}}} 145 | ) 146 | 147 | ref = Process.monitor(worker_pid) 148 | 149 | result = %RuntimeError{message: error_message, __exception__: true} 150 | 151 | assert_receive {:task_result, ^result, ^task_id, :error}, 200 152 | assert_receive {:DOWN, ^ref, :process, _pid, :normal}, 200 153 | 154 | on_exit(fn -> 155 | :code.purge(mod) 156 | :code.delete(mod) 157 | end) 158 | end 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /apps/gust/lib/gust/dsl.ex: -------------------------------------------------------------------------------- 1 | defmodule Gust.DSL do 2 | @moduledoc """ 3 | The Gust DSL is how you turn a module into a DAG. 4 | When you add `use Gust.DSL` to a module in the `dags/` folder, Gust automatically detects it and creates a DAG based on the file name. 5 | 6 | You can configure a schedule, define callbacks, and in the `dev` environment the code is automatically reloaded when files change. 7 | 8 | After enabling the DSL, use `task` definitions to declare the steps that should be executed. 9 | 10 | ## Example 11 | 12 | defmodule HelloWorld do 13 | # `schedule` and `on_finished_callback` are optional. 14 | # Note: if you change `schedule`, restart the server to update the cron job. 15 | use Gust.DSL, schedule: "* * * * *", on_finished_callback: :notify_something 16 | 17 | # Gust logs are stored and displayed through GustWeb via Logger. 18 | require Logger 19 | 20 | # Gust.Flows is used to query Dag, Run, and Task. 21 | alias Gust.Flows 22 | 23 | def notify_something(status, run) do 24 | dag = Flows.get_dag!(run.dag_id) 25 | message = "DAG: \#{dag.name}; completed with status: \#{status}" 26 | Logger.info(message) 27 | end 28 | 29 | task :first_task, downstream: [:second_task], store_result: true do 30 | greetings = "Hi from first_task" 31 | Logger.info(greetings) 32 | 33 | # You can get secrets created on the Web UI 34 | secret = Flows.get_secret_by_name("SUPER_SECRET") 35 | Logger.warning("I know your secret: \#{secret.value}") 36 | 37 | # The return value must be a map when `store_result` is true. 38 | %{result: greetings} 39 | end 40 | 41 | task :second_task, ctx: %{run_id: run_id} do 42 | 43 | # Getting "first_task"'s result 44 | task = Flows.get_task_by_name_run("first_task", run_id) 45 | 46 | Logger.info(task.result) 47 | end 48 | end 49 | 50 | ## Parameters 51 | 52 | * `schedule` - A valid cron expression string. 53 | * `on_finished_callback` - The name of the function to be called. 54 | """ 55 | 56 | defmacro __using__(dag_options) do 57 | quote do 58 | import unquote(__MODULE__), only: [task: 2, task: 3] 59 | 60 | Module.register_attribute(__MODULE__, :dag_tasks, accumulate: true) 61 | 62 | def __dag_options__, do: unquote(dag_options) 63 | 64 | @before_compile unquote(__MODULE__) 65 | end 66 | end 67 | 68 | defmacro __before_compile__(_env) do 69 | quote do 70 | def __dag_tasks__, do: @dag_tasks 71 | end 72 | end 73 | 74 | @doc """ 75 | Defines a task in the DAG. 76 | 77 | ## Parameters 78 | 79 | * `name` — The name of the task (atom). 80 | * `opts_and_ctx` — A keyword list of options and an optional context pattern. 81 | * `block` — The code block executed when the task runs. 82 | 83 | ## Task Options 84 | 85 | * `:downstream` — A list of task names (atoms) to run after this task completes. 86 | * `:store_result` — When true, the task's return value will be persisted. 87 | * Note: If enabled, the return value **must be a map**. 88 | * `:ctx` — A pattern that will be matched against the context passed to the task. 89 | * Defaults to: `%{run_id: run_id}`. 90 | 91 | ## Example 92 | 93 | task :my_task, ctx: %{run_id: run_id} do 94 | IO.inspect(run_id) 95 | end 96 | 97 | task :first, downstream: [:second] do 98 | :ok 99 | end 100 | 101 | task :persist_result, store_result: true do 102 | %{result: :ok} 103 | end 104 | 105 | When using `store_result: true`, the return value **must** be a map so it can be merged into the overall DAG results. 106 | """ 107 | defmacro task(name, opts_and_ctx, do: block) do 108 | {ctx_pattern, opts} = Keyword.pop(opts_and_ctx, :ctx) 109 | ctx_pattern = ctx_pattern || quote do: %{run_id: run_id} 110 | 111 | quote do 112 | @dag_tasks {unquote(name), unquote(opts)} 113 | 114 | def unquote(name)(ctx) do 115 | unquote(ctx_pattern) = ctx 116 | unquote(block) 117 | end 118 | end 119 | end 120 | 121 | @doc """ 122 | Defines a task in the DAG without options or explicit context matching. 123 | 124 | ## Parameters 125 | 126 | * `name` - The name of the task (atom). 127 | * `block` - The code block to execute for the task. 128 | 129 | ## Example 130 | 131 | task :simple_task do 132 | IO.puts "Hello" 133 | end 134 | """ 135 | defmacro task(name, do: block) do 136 | quote do 137 | @dag_tasks {unquote(name), []} 138 | 139 | def unquote(name)(ctx) do 140 | unquote(block) 141 | end 142 | end 143 | end 144 | end 145 | --------------------------------------------------------------------------------