├── test ├── test_helper.exs ├── cinema_test.exs └── cinema │ ├── projection_test.exs │ └── projection │ └── rank_test.exs ├── .formatter.exs ├── .envrc ├── docker-compose.yml ├── .gitignore ├── lib ├── cinema │ ├── utils.ex │ ├── utils │ │ └── task.ex │ ├── engine │ │ ├── task.ex │ │ └── oban_pro_workflow.ex │ ├── projection │ │ ├── lens.ex │ │ └── rank.ex │ ├── engine.ex │ └── projection.ex └── cinema.ex ├── LICENSE ├── flake.nix ├── flake.lock ├── mix.exs ├── README.md ├── .credo.exs └── mix.lock /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /test/cinema_test.exs: -------------------------------------------------------------------------------- 1 | defmodule CinemaTest do 2 | use ExUnit.Case 3 | end 4 | -------------------------------------------------------------------------------- /test/cinema/projection_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Cinema.ProjectionTest do 2 | use ExUnit.Case 3 | 4 | doctest Cinema.Projection 5 | end 6 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"], 4 | plugins: [Styler] 5 | ] 6 | -------------------------------------------------------------------------------- /test/cinema/projection/rank_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Projection.RankTest do 2 | use ExUnit.Case 3 | 4 | doctest Cinema.Projection.Rank 5 | end 6 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | export ERL_AFLAGS="-kernel shell_history enabled"; 2 | 3 | if command -v nix &> /dev/null; then 4 | if ! has nix_direnv_version || ! nix_direnv_version 2.4.0; then 5 | source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.4.0/direnvrc" "sha256-XQzUAvL6pysIJnRJyR7uVpmUSZfc7LSgWQwq/4mBr1U=" 6 | fi 7 | 8 | use flake 9 | fi 10 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | 3 | services: 4 | db: 5 | image: postgres 6 | environment: 7 | - POSTGRES_USER 8 | - POSTGRES_DB 9 | - POSTGRES_PASSWORD 10 | ports: 11 | - ${POSTGRES_PORT}:5432 12 | jaeger: 13 | image: jaegertracing/opentelemetry-all-in-one:latest 14 | ports: 15 | - 16686:16686 16 | - 55680:55680 17 | - 55681:55681 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | projectionist-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | 28 | .direnv/ 29 | 30 | elixir-ls/ 31 | elixir_ls/ 32 | 33 | priv/plts 34 | -------------------------------------------------------------------------------- /lib/cinema/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Utils do 2 | @moduledoc false 3 | 4 | @spec sanitize_timestamps(map :: map()) :: map() 5 | def sanitize_timestamps(map) when is_map_key(map, :inserted_at) or is_map_key(map, :updated_at) do 6 | map 7 | |> Map.update(:inserted_at, nil, &NaiveDateTime.truncate(&1, :second)) 8 | |> Map.update(:updated_at, nil, &NaiveDateTime.truncate(&1, :second)) 9 | end 10 | 11 | def sanitize_timestamps(map) do 12 | map 13 | end 14 | 15 | @spec implemented?(module(), behaviour :: module()) :: boolean() 16 | def implemented?(module, behaviour) do 17 | behaviours = 18 | :attributes 19 | |> module.module_info() 20 | |> Enum.filter(&match?({:behaviour, _behaviours}, &1)) 21 | |> Enum.map(&elem(&1, 1)) 22 | |> List.flatten() 23 | 24 | behaviour in behaviours 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Vetspire 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | inputs = { 3 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; 4 | flake-utils.url = "github:numtide/flake-utils"; 5 | }; 6 | 7 | outputs = { self, nixpkgs, flake-utils }: 8 | flake-utils.lib.eachDefaultSystem ( 9 | system: 10 | let 11 | pkgs = import nixpkgs { inherit system; }; 12 | in 13 | with pkgs; { 14 | devShells.default = mkShell { 15 | buildInputs = [ 16 | elixir_1_17 17 | docker-compose 18 | ] 19 | ++ lib.optionals stdenv.isLinux ([ libnotify inotify-tools ]) 20 | ++ lib.optionals stdenv.isDarwin ([ terminal-notifier 21 | darwin.apple_sdk.frameworks.CoreFoundation 22 | darwin.apple_sdk.frameworks.CoreServices 23 | ]); 24 | 25 | env = { 26 | POSTGRES_PORT="5432"; 27 | POSTGRES_USER = "postgres"; 28 | POSTGRES_PASSWORD = "postgres"; 29 | POSTGRES_DB = "cinema_repo"; 30 | }; 31 | }; 32 | } 33 | ); 34 | } 35 | -------------------------------------------------------------------------------- /lib/cinema/utils/task.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Utils.Task do 2 | @moduledoc false 3 | 4 | @spec async((-> term())) :: term() 5 | def async(func) when is_function(func, 0) do 6 | if test_mode?() do 7 | func.() 8 | else 9 | Task.async(func) 10 | end 11 | end 12 | 13 | @spec await(term(), timeout :: non_neg_integer()) :: term() 14 | def await(task, timeout) do 15 | if test_mode?() do 16 | task 17 | else 18 | Task.await(task, timeout) 19 | end 20 | end 21 | 22 | @spec await_many([term()], timeout :: non_neg_integer()) :: [term()] 23 | def await_many(tasks, timeout) do 24 | if test_mode?() do 25 | tasks 26 | else 27 | Task.await_many(tasks, timeout) 28 | end 29 | end 30 | 31 | @doc "Returns `true` if the current process is running in test mode, otherwise returns `false`." 32 | @spec test_mode?() :: boolean() 33 | def test_mode? do 34 | Process.get({__MODULE__, :test_mode}, false) 35 | end 36 | 37 | @doc "Sets the test mode to the given boolean value. When in test mode, all Task functions will be executed synchronously." 38 | @spec test_mode(boolean()) :: :ok 39 | def test_mode(bool) when is_boolean(bool) do 40 | Process.put({__MODULE__, :test_mode}, bool) 41 | :ok 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "inputs": { 5 | "systems": "systems" 6 | }, 7 | "locked": { 8 | "lastModified": 1710146030, 9 | "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", 10 | "owner": "numtide", 11 | "repo": "flake-utils", 12 | "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", 13 | "type": "github" 14 | }, 15 | "original": { 16 | "owner": "numtide", 17 | "repo": "flake-utils", 18 | "type": "github" 19 | } 20 | }, 21 | "nixpkgs": { 22 | "locked": { 23 | "lastModified": 1722813957, 24 | "narHash": "sha256-IAoYyYnED7P8zrBFMnmp7ydaJfwTnwcnqxUElC1I26Y=", 25 | "owner": "NixOS", 26 | "repo": "nixpkgs", 27 | "rev": "cb9a96f23c491c081b38eab96d22fa958043c9fa", 28 | "type": "github" 29 | }, 30 | "original": { 31 | "owner": "NixOS", 32 | "ref": "nixos-unstable", 33 | "repo": "nixpkgs", 34 | "type": "github" 35 | } 36 | }, 37 | "root": { 38 | "inputs": { 39 | "flake-utils": "flake-utils", 40 | "nixpkgs": "nixpkgs" 41 | } 42 | }, 43 | "systems": { 44 | "locked": { 45 | "lastModified": 1681028828, 46 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 47 | "owner": "nix-systems", 48 | "repo": "default", 49 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 50 | "type": "github" 51 | }, 52 | "original": { 53 | "owner": "nix-systems", 54 | "repo": "default", 55 | "type": "github" 56 | } 57 | } 58 | }, 59 | "root": "root", 60 | "version": 7 61 | } 62 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Cinema.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :cinema, 7 | version: "0.1.1", 8 | elixir: "~> 1.17", 9 | start_permanent: Mix.env() == :prod, 10 | docs: [main: "Cinema"], 11 | description: "A simple Elixir framework utilizing Ecto and DAGs to incrementally materialize views!", 12 | package: package(), 13 | dialyzer: [ 14 | plt_add_apps: [:iex, :mix, :ex_unit], 15 | plt_file: {:no_warn, "priv/plts/dialyzer.plt"}, 16 | flags: [:error_handling] 17 | ], 18 | test_coverage: [tool: ExCoveralls], 19 | preferred_cli_env: [ 20 | lint: :test, 21 | dialyzer: :test, 22 | coveralls: :test, 23 | "coveralls.detail": :test, 24 | "coveralls.post": :test, 25 | "coveralls.html": :test, 26 | "test.watch": :test 27 | ], 28 | aliases: aliases(), 29 | deps: deps(), 30 | source_url: "https://github.com/vereis/cinema", 31 | homepage_url: "https://github.com/vereis/cinema" 32 | ] 33 | end 34 | 35 | defp package do 36 | [ 37 | licenses: ["MIT"], 38 | links: %{"GitHub" => "https://github.com/vereis/cinema"} 39 | ] 40 | end 41 | 42 | # Run "mix help compile.app" to learn about applications. 43 | def application do 44 | [ 45 | extra_applications: [:logger] 46 | ] 47 | end 48 | 49 | defp has_oban_pro? do 50 | Hex.start() 51 | match?(%{url: "https://getoban.pro/repo"}, Hex.State.fetch!(:repos)["oban"]) 52 | end 53 | 54 | # Run "mix help deps" to learn about dependencies. 55 | defp deps do 56 | [ 57 | {:ecto, "~> 3.6"}, 58 | {:ecto_sql, "~> 3.11"}, 59 | {:jason, "~> 1.2"}, 60 | {:postgrex, "~> 0.15"}, 61 | {:sibyl, "~> 0.1.9"}, 62 | 63 | # Runtime dependencies for tests / linting 64 | {:credo, "~> 1.6", only: [:dev, :test], runtime: false}, 65 | {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, 66 | {:ex_doc, "~> 0.28", only: :dev}, 67 | {:ex_machina, "~> 2.7", only: :test}, 68 | {:excoveralls, "~> 0.10", only: :test}, 69 | {:mix_test_watch, "~> 1.0", only: [:test], runtime: false}, 70 | {:styler, "~> 0.11", only: [:dev, :test], runtime: false} 71 | | (has_oban_pro?() && [{:oban, "~> 2.18"}, {:oban_pro, "~> 1.5.0-rc.1", repo: "oban"}]) || 72 | [] 73 | ] 74 | end 75 | 76 | defp aliases do 77 | [ 78 | test: ["coveralls.html --trace --slowest 10"], 79 | lint: [ 80 | "format --check-formatted --dry-run", 81 | "credo --strict", 82 | "compile --warnings-as-errors", 83 | "dialyzer" 84 | ] 85 | ] 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/cinema/engine/task.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Engine.Task do 2 | @moduledoc """ 3 | The `Cinema.Engine.Task` module provides a simple implementation of the `Cinema.Engine` behaviour 4 | which uses the `Task` module to execute projections. 5 | 6 | When in test mode, all Task functions will be executed synchronously. You can enable test mode by calling 7 | `Cinema.Utils.Task.test_mode(true)` somewhere before executing projections. 8 | 9 | See the `Cinema.Engine` module for more details re: defining engines. 10 | """ 11 | 12 | @behaviour Cinema.Engine 13 | 14 | alias Cinema.Engine 15 | alias Cinema.Projection 16 | alias Cinema.Projection.Lens 17 | alias Cinema.Utils.Task 18 | 19 | @timeout :timer.minutes(1) 20 | 21 | @doc """ 22 | Executes the given `Projection.t()` using the Task engine. Returns `{:ok, projection}` if the projection 23 | was successfully executed, otherwise returns `{:error, projection}`. 24 | 25 | You can fetch the output of the `Projection.t()` (assuming it was successfully executed) by passing the 26 | returned `Projection.t()` to `Cinema.Projection.fetch/1` function. 27 | 28 | ## Options 29 | 30 | * `:timeout` - The timeout to use when executing the projection. Defaults to `1 minute`. 31 | * `:skip_dependencies` - Defaults to `false`. If `true`, skips executing the dependencies of the given projection. 32 | this is very useful for testing purposes where you use `ExMachina` to "mock" the results of all input projections. 33 | 34 | """ 35 | @impl Cinema.Engine 36 | def exec(%Projection{} = projection, opts \\ []) do 37 | timeout = opts[:timeout] || @timeout 38 | 39 | task = 40 | Task.async(fn -> 41 | if opts[:skip_dependencies] do 42 | Engine.do_exec(projection.terminal_projection, projection.lens) 43 | else 44 | projection.exec_graph 45 | |> Enum.map(&List.wrap/1) 46 | |> Enum.each(fn projections -> 47 | projections 48 | |> Enum.map(&Task.async(fn -> Engine.do_exec(&1, projection.lens) end)) 49 | |> Task.await_many(timeout) 50 | end) 51 | end 52 | 53 | Lens.apply(projection.lens, projection.terminal_projection, as_stream: true) 54 | end) 55 | 56 | {:ok, %Projection{projection | meta: %{task: task}}} 57 | end 58 | 59 | @doc """ 60 | Fetches the output of a `Projection.t()` which has been executed using `Cinema.Engine.exec/2`. 61 | """ 62 | @impl Cinema.Engine 63 | def fetch(%Projection{} = projection) do 64 | case projection.meta[:task] do 65 | nil -> 66 | {:error, "The given projection was not executed using the Task engine."} 67 | 68 | task -> 69 | {:ok, Task.await(task, @timeout)} 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/cinema/projection/lens.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Projection.Lens do 2 | @moduledoc """ 3 | The `#{__MODULE__}` module provides a struct for defining lenses which can be used to manipulate 4 | the inputs to a given projection. 5 | 6 | Lenses can be used to filter, scope, or otherwise manipulate the inputs to a given projection. 7 | 8 | See the `#{__MODULE__}.apply/3` and `#{__MODULE__}.build!/2` functions for more information. 9 | """ 10 | 11 | import Ecto.Query 12 | 13 | alias __MODULE__ 14 | alias Cinema.Projection 15 | 16 | @type t :: %__MODULE__{} 17 | defstruct [:reducer, filters: []] 18 | 19 | @doc """ 20 | Builds a new `#{__MODULE__}.t()` struct with the given `filters` and optional `reducer`. 21 | See `#{__MODULE__}.apply/3` for more information. 22 | """ 23 | @spec build!( 24 | filters :: list(), 25 | ({filter :: atom(), value :: term()}, acc :: Ecto.Query.t() | module() -> Ecto.Query.t() | module()) | nil 26 | ) :: t() 27 | def build!(filters, reducer \\ nil) when is_list(filters) do 28 | %__MODULE__{filters: filters, reducer: reducer} 29 | end 30 | 31 | @doc """ 32 | Uses the given `#{__MODULE__}.t()` to manipulate inputs to the given projection. 33 | 34 | If the given `#{__MODULE__}.t()` has a `reducer` function, a given input projection's output 35 | will be reduced over with the given `filters`, allowing you to customize the behaviour of how 36 | a projection's inputs are scoped or modified. 37 | 38 | If the given `#{__MODULE__}.t()` does not have a `reducer` function, the given `filters` will 39 | be used to filter the given input projection's output the same way `Ecto.Query.where/3` does 40 | when given a static `Keyword.t()` as a final argument. 41 | 42 | Returns either the manipulated queryable, or a stream capturing said queryable depending on if 43 | `as_stream: true` is passed in the `opts` argument. 44 | """ 45 | @spec apply(t(), module(), Keyword.t()) :: Enumerable.t() | Ecto.Query.t() 46 | def apply(%Lens{} = lens, projection, opts \\ []) when is_atom(projection) do 47 | unless Projection.implemented?(projection) do 48 | raise ArgumentError, 49 | message: "The given projection does not implement the `Cinema.Projection` behaviour." 50 | end 51 | 52 | output = projection.output() 53 | 54 | if is_struct(output, Ecto.Query) or is_atom(output) do 55 | query = 56 | if is_nil(lens.reducer) do 57 | fields = projection.fields() 58 | filters = Keyword.take(lens.filters, fields) 59 | from(x in output, where: ^filters) 60 | else 61 | Enum.reduce(lens.filters, output, lens.reducer) 62 | end 63 | 64 | if opts[:as_stream] do 65 | (projection.opts[:read_repo] || projection.opts[:write_repo] || projection.opts[:repo]).stream(query) 66 | else 67 | query 68 | end 69 | else 70 | Stream.filter(output, &Enum.all?(lens.filters, fn {k, v} -> &1[k] == v end)) 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/cinema/engine.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Engine do 2 | @moduledoc """ 3 | The `Cinema.Engine` module provides a behaviour for defining engines which can execute projections 4 | defined using the `Cinema.Projection` module. 5 | 6 | See the `Cinema.Engine.Task` module for an example implementation of this behaviour. 7 | Also see the `Cinema.Projection` module for more details re: defining projections. 8 | """ 9 | 10 | alias Cinema.Engine.Task 11 | alias Cinema.Projection 12 | alias Cinema.Projection.Lens 13 | alias Cinema.Utils 14 | 15 | @callback exec(Projection.t()) :: {:ok, Projection.t()} | {:error, Projection.t()} 16 | @callback fetch(Projection.t()) :: term() 17 | 18 | defdelegate test_mode?(), to: Utils.Task 19 | defdelegate test_mode(bool), to: Utils.Task 20 | 21 | @doc """ 22 | Returns `true` if the given module implements the `#{__MODULE__}` behaviour, otherwise returns `false`. 23 | 24 | See examples: 25 | 26 | ```elixir 27 | iex> Cinema.Engine.implemented?(Enum) 28 | false 29 | iex> Cinema.Projection.implemented?(Cinema.Engine.Task) 30 | true 31 | ``` 32 | """ 33 | @spec implemented?(module()) :: boolean() 34 | def implemented?(module) do 35 | Utils.implemented?(module, __MODULE__) 36 | end 37 | 38 | @doc """ 39 | Executes the given `Projection.t()` using the given engine. Returns `{:ok, projection}` if the projection 40 | was successfully executed, otherwise returns `{:error, projection}`. 41 | 42 | You can fetch the output of the `Projection.t()` (assuming it was successfully executed) by passing the 43 | returned `Projection.t()` to `Cinema.Projection.fetch/1` function. 44 | """ 45 | @spec exec(Projection.t(), Keyword.t()) :: 46 | {:ok, Projection.t()} | {:error, Projection.t()} 47 | def exec(%Projection{} = projection, opts \\ [engine: Task]) do 48 | engine = opts[:engine] || Task 49 | 50 | unless implemented?(engine) do 51 | raise ArgumentError, 52 | message: "Engine `#{inspect(engine)}` does not implement the `#{__MODULE__}` behaviour." 53 | end 54 | 55 | with {status, %Projection{} = projection} <- engine.exec(projection, opts) do 56 | {status, %Projection{projection | engine: engine, valid?: status == :ok}} 57 | end 58 | end 59 | 60 | @doc """ 61 | Fetches the output of a `Projection.t()` which has been executed using `Cinema.Engine.exec/2`. 62 | """ 63 | @spec fetch(Projection.t()) :: {:ok, term()} | {:error, String.t()} 64 | def fetch(%Projection{} = projection) when projection.valid? and is_atom(projection.engine) do 65 | projection.engine.fetch(projection) 66 | end 67 | 68 | def fetch(%Projection{} = _projection) do 69 | {:error, "Projection is not valid."} 70 | end 71 | 72 | @doc false 73 | @spec do_exec(module(), Lens.t()) :: :ok 74 | def do_exec(projection, %Lens{} = lens) do 75 | opts = projection.opts() 76 | timeout = opts[:timeout] || :timer.minutes(1) 77 | write_repo = opts[:write_repo] || opts[:repo] 78 | read_repo = opts[:read_repo] || write_repo 79 | 80 | closure = fn -> 81 | Enum.each( 82 | projection.inputs(), 83 | &projection.derivation({&1, Lens.apply(lens, &1, as_stream: true)}, lens) 84 | ) 85 | end 86 | 87 | {:ok, _resp} = 88 | if write_repo != read_repo do 89 | write_repo.transaction( 90 | fn -> 91 | read_repo.transaction(closure, timeout: timeout) 92 | end, 93 | timeout: timeout 94 | ) 95 | else 96 | write_repo.transaction(closure, timeout: timeout) 97 | end 98 | 99 | :ok 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /lib/cinema.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema do 2 | @moduledoc File.read!("README.md") 3 | 4 | alias Cinema.Engine 5 | alias Cinema.Projection 6 | alias Cinema.Projection.Lens 7 | 8 | require Logger 9 | 10 | @doc """ 11 | Projects a module that implements the `Cinema.Projection` behaviour. 12 | 13 | Projection is done by reflecting on the given projection and building a dependency graph of all of it's inputs, 14 | which is then executed in the correct order. 15 | 16 | The output of each dependency projection is streamed into the given projection's `c:Cinema.Projection.derivation/2` 17 | callback. This callback is responsible for materializing database table rows which can then be returned as the final 18 | output of the projection. 19 | 20 | Optionally takes a `Lens.t()` or a `Keyword.t()` filter list to apply to the projection. 21 | 22 | Additionally, you can pass in a `Keyword.t()` list of options to control the behavior of how the projection is 23 | executed. 24 | 25 | Note that by default, the given projection's final `c:Cinema.Projection.output/0` will be awaited and 26 | evaluated by the projection's configured `read_repo`. If you want to control the execute the projection, pass 27 | `async: true` as an option. 28 | 29 | ## Options 30 | 31 | * `:async` - Defaults to `false`. If `true`, the projection will be executed immediately but the given projection's 32 | final output will not be awaited and returned. Instead, the projection itself will be returned. 33 | 34 | You can then await and fetch the final output via `Cinema.fetch/1`. 35 | 36 | * `:engine` - The engine to use to execute the projection. Defaults to `Cinema.Engine.Task`. See 37 | `Cinema.Engine` for more information. 38 | 39 | * `:timeout` - The timeout to use when executing the projection. Defaults to `1 minute`. 40 | 41 | * `:allow_empty_filters` - Defaults to `false`. If `true`, skips the warning message that gets logged when an empty 42 | filter list is provided. 43 | 44 | Options are additionally also passed to the engine that is used to execute the projection, as well as used when 45 | building the projection itself. 46 | """ 47 | @spec project(Projection.t()) :: {:ok, [term()]} | {:error, term()} 48 | @spec project(Projection.t(), Lens.t()) :: {:ok, [term()]} | {:error, term()} 49 | @spec project(Projection.t(), Lens.t(), Keyword.t()) :: 50 | {:ok, Projection.t() | [term()]} | {:error, term()} 51 | def project(projection, lens \\ %Lens{}, opts \\ []) 52 | 53 | def project(projection, filters, opts) when is_list(filters) do 54 | filters 55 | |> Lens.build!() 56 | |> then(&project(projection, &1, opts)) 57 | end 58 | 59 | def project(projection, %Lens{} = lens, opts) do 60 | unless Projection.implemented?(projection) do 61 | raise ArgumentError, 62 | message: """ 63 | `Cinema.project/2` expects a `Cinema.Projection` as its first argument, got `#{inspect(projection)}`. 64 | """ 65 | end 66 | 67 | if lens.filters == [] and !opts[:allow_empty_filters] do 68 | Logger.warning(""" 69 | `Cinema.project/2` was called with an empty filter list. When this happens, all 70 | neccessary projections will be projected and run without any filters and scoping applied. 71 | 72 | If you want to allow this behavior, pass `allow_empty_filters: true` as an option to `Cinema.project/3`. 73 | """) 74 | end 75 | 76 | {:ok, %Projection{} = projection} = 77 | projection 78 | |> Projection.build!(lens, opts) 79 | |> Engine.exec(opts) 80 | 81 | (opts[:async] && {:ok, projection}) || 82 | projection.read_repo.transaction( 83 | fn -> 84 | {:ok, value} = Engine.fetch(projection) 85 | Enum.to_list(value) 86 | end, 87 | timeout: opts[:timeout] || :timer.minutes(1) 88 | ) 89 | end 90 | 91 | defdelegate fetch(projection), to: Engine 92 | end 93 | -------------------------------------------------------------------------------- /lib/cinema/projection/rank.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Projection.Rank do 2 | @moduledoc """ 3 | This module is responsible for ranking a list of projections in a depth-first manner, which can 4 | be used to determine the order in which to execute a list of projections, potentially asynchronously. 5 | 6 | The `derive/2` function is the main entry point, which takes a list of projections and a single 7 | target projection you'd like to run. The function will then proceed to rank all dependencts of the 8 | target projection recursively. 9 | 10 | Note that a single dependency can appear multiple times due to being dependents of different subgraphs 11 | of the target projection. In this case, the dependency will be ranked multiple times, but the final 12 | ranking will be the highest rank of all duplicate dependencies. 13 | 14 | Ranked projections can then be grouped by rank, and the groups can be executed in parallel, as they 15 | are guaranteed to be independent of each other. 16 | 17 | See the `depth_first_rank/2` function for more details on how the ranking is actually done. 18 | """ 19 | 20 | @doc """ 21 | Derives the dependency order of a target projection in a list of projections by ranking them in a 22 | depth-first manner. 23 | 24 | Projections are expected to be given in the form: `[{Module.t(), [Module.t()]}]` 25 | 26 | Note that the order of the returned list is derived simply from the declared hierarchy of the projections, 27 | and does not take into account any additional processing that may be required to actually execute 28 | the projections. This means that the order of the returned list may not be the optimal order for 29 | execution, but it will guarantee that all dependencies are executed before the target projection. 30 | 31 | Additionally, the algorithm used is simple in that it "lazily" ranks dependencies after the whole graph 32 | has been traversed, and does not attempt to optimize the ranking in any way. A graph could theoretically 33 | take into consideration "re-ranking" of dependencies in an attempt to optimize ordering/grouping, but 34 | this is not currently implemented. 35 | 36 | Also do note that currently this function does not handle cycles in the graph, and will raise an error 37 | if a cycle is detected. 38 | 39 | See examples: 40 | 41 | ```elixir 42 | iex> alias #{__MODULE__} 43 | iex> graph = [a: [], b: [], c: [:a, :b], d: [:c], e: [:b], f: [:c, :e]] 44 | iex> Rank.derive!(graph, :d) 45 | [[:a, :b], :c, :d] 46 | iex> Rank.derive!(graph, :e) 47 | [:b, :e] 48 | iex> Rank.derive!(graph, :a) 49 | [:a] 50 | iex> # This isn't neccessarily the optimal order, but it is _correct_. 51 | iex> Rank.derive!(graph, :f) 52 | [[:a, :b], [:c, :e], :f] 53 | ``` 54 | """ 55 | @spec derive!([{module() | [module()]}], module()) :: [module() | [module()]] 56 | def derive!(projections, projection) do 57 | # TODO: pretty sure we can raise from `depth_first_rank/2` if a cycle is detected 58 | # but I'm not bothered to figure this out right now! 59 | ranked_map = 60 | projections 61 | |> Enum.sort() 62 | |> depth_first_rank(projection) 63 | |> Enum.reduce(%{}, fn {idx, node}, acc -> 64 | Map.update(acc, node, idx, &((&1 < idx && idx) || &1)) 65 | end) 66 | 67 | max_rank = 68 | ranked_map 69 | |> Enum.max_by(fn {_, idx} -> idx end) 70 | |> elem(1) 71 | 72 | async_groups = 73 | for rank <- 0..max_rank do 74 | ranked_map 75 | |> Enum.filter(fn {_, idx} -> idx == rank end) 76 | |> Enum.map(fn {node, _} -> node end) 77 | end 78 | 79 | async_groups |> Enum.map(&((match?([_], &1) && hd(&1)) || &1)) |> Enum.reverse() 80 | end 81 | 82 | @doc """ 83 | Recursively traverses a list of KV tuples representing a graph of projections and their dependencies, 84 | and ranks them in a depth-first manner. 85 | 86 | Does not do any additional processing on the ranked projections, see `derive/2` for that. 87 | """ 88 | @spec depth_first_rank([{module() | [module()]}], module()) :: [{integer(), module()}] 89 | def depth_first_rank(projections, projection) do 90 | projections 91 | |> Enum.reject(fn x -> match?({_, []}, x) end) 92 | |> Map.new() 93 | |> depth_first_rank(projection, 0) 94 | end 95 | 96 | defp depth_first_rank(projections, projection, idx) when not is_map_key(projections, projection) do 97 | [{idx, projection}] 98 | end 99 | 100 | defp depth_first_rank(projections, projection, idx) do 101 | [ 102 | {idx, projection} 103 | | projections[projection] 104 | |> Enum.map(fn projection -> depth_first_rank(projections, projection, idx + 1) end) 105 | |> List.flatten() 106 | ] 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cinema 2 | 3 | Cinema is a simple Elixir framework for managing incremental materialized views entirely in Elixir/Ecto. 4 | 5 | ## Installation 6 | 7 | Cinema can be installed by adding `cinema` to your list of dependencies in `mix.exs`: 8 | 9 | ```elixir 10 | def deps do 11 | [ 12 | {:cinema, "~> 0.1.0"} 13 | ] 14 | end 15 | ``` 16 | 17 | Cinema has an optional dependency on [Oban Pro](https://getoban.pro) as an alternate runtime for materializing projection graphs. Oban Pro support is automatically enabled if `Hex` detects the `oban` repo in your global setup. 18 | 19 | Please see the [Oban Pro](https://getoban.pro) documentation for more information on how to install and configure Oban Pro. 20 | 21 | ## Usage 22 | 23 | Cinema introduces two basic concepts: 24 | 25 | - **Projections**: A projection is a behaviour that allows you to declaratively define instructions in Elixir for how to derive rows to write into your materialized views. They statically define all `c:inputs/0` (other projections, if needed), `c:output/1` (usually an `Ecto.Query` or stream passed into subsequent projections), and a `c:derivation/2` callback which is run in tandem with all inputs to produce output. 26 | - **Lenses**: A lens is a struct which contains filters and other options which can be used to modify the "scope" of what a projection is required to derive. In simple use cases, you can think of lenses as automatically applying filters such as: `where: x.org_id == ^org_id` to the outputs of all input projections automatically. 27 | 28 | When you want to actually incrementally rematerialize a view, you create a `Cinema.Lens.t()` (or a simple keyword list for simple filters), and pass that into the `Cinema.project/3` function like so: 29 | 30 | ```elixir 31 | iex> Cinema.project(MyApp.Projections.AccountsReceivable, [org_id: 123, date: ~D[2022-01-01]]) 32 | [ 33 | %MyApp.Projections.AccountsReceivable{ 34 | org_id: 123, 35 | date: ~D[2022-01-01], 36 | ... 37 | }, 38 | ... 39 | ] 40 | ``` 41 | 42 | Projections generally define their own `Ecto.Schema` internally and can also be queried directly -- note that this will not rematerialize any dependencies or rows in the table you're querying: 43 | 44 | ```elixir 45 | iex> MyApp.Repo.all(MyApp.Projections.AccountsReceivable) 46 | [ 47 | %MyApp.Projections.AccountsReceivable{ 48 | org_id: 123, 49 | date: ~D[2022-01-01], 50 | ... 51 | }, 52 | ... 53 | ] 54 | ``` 55 | 56 | Projections can include other projections as inputs, and Cinema will automatically rematerialize those projections as needed. For example, if `AccountsReceivable` depends on `Invoices`, Cinema will automatically rematerialize `Invoices` before rematerializing `AccountsReceivable`. 57 | 58 | Projection graphs usually begin with "virtual" projections that have no inputs or `derivation/2` callback, instead only outputting either an `Ecto.Query` or stream which is passed directly through any `Cinema.Lens.t()` and into the next projection in the graph. 59 | 60 | Cinema does this by building a DAG of all projections and their dependencies. Cinema will likewise try to run any projections in parallel where possible. A minimal example of a projection looks like the following: 61 | 62 | ```elixir 63 | defmodule MyApp.Projections.Accounts do 64 | use Cinema.Projection, virtual?: true 65 | 66 | @impl Cinema.Projection 67 | def inputs, do: [] 68 | 69 | @impl Cinema.Projection 70 | def output, do: from(a in "accounts", select: a.id) 71 | end 72 | 73 | defmodule MyApp.Projections.AccountsReceivable do 74 | use Cinema.Projection, 75 | conflict_target: [:account_id], 76 | required_fields: [:account_id], 77 | on_conflict: :replace_all, 78 | read_repo: MyApp.Repo.Replica, 79 | write_repo: MyApp.Repo, 80 | timeout: :timer.minutes(5), 81 | 82 | alias Cinema.Projection 83 | alias MyApp.Projections.Accounts 84 | 85 | @primary_key false 86 | schema "accounts_receivable" do 87 | field(account_id:, :id) 88 | field(total, :integer) 89 | 90 | timestamps() 91 | end 92 | 93 | @impl Cinema.Projection 94 | def inputs, do: [Accounts] 95 | 96 | @impl Cinema.Projection 97 | def output, do: from(a in "accounts_receivable", select: a) 98 | 99 | @impl Cinema.Projection 100 | def derivation({Accounts, stream}, lens) do 101 | Projection.dematerialize(lens) 102 | 103 | stream 104 | |> Stream.chunk_every(2000) 105 | |> Stream.map(&from x in MyApp.Invoice, where: x.account_id in ^&1, select: %{account_id: x.account_id, total: sum(x.total)}) 106 | |> Stream.map(&Projection.materialize/1) 107 | |> Stream.run() 108 | end 109 | end 110 | ``` 111 | 112 | ### Configuration 113 | 114 | Currently, Cinema lets you configure the following options: 115 | 116 | - `:engine` - The runtime to use for executing projection graphs. Defaults to `Cinema.Engine.Task`. 117 | - `:async` - Whether to run projections asynchronously. Defaults to `true`. 118 | 119 | Additional configuration options can be implemented on a projection-by-projection basis, please see the docs for the `Cinema.Projection` behaviour for more information. 120 | 121 | ## License 122 | 123 | Cinema is released under the [MIT License](LICENSE.md). 124 | -------------------------------------------------------------------------------- /lib/cinema/engine/oban_pro_workflow.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(Oban.Pro.Workflow) do 2 | defmodule Cinema.Engine.Oban.Pro.Workflow do 3 | @moduledoc """ 4 | The `Cinema.Engine.Oban.Pro.Workflow` module provides an implementation of the `Cinema.Engine` behaviour 5 | which uses the `Oban.Pro.Workflow` module to execute projections. 6 | 7 | See `Oban.Testing` for testing information. 8 | 9 | See the `Cinema.Engine` module for more details re: defining engines. 10 | """ 11 | 12 | @behaviour Cinema.Engine 13 | 14 | alias Cinema.Engine 15 | alias Cinema.Projection 16 | alias Cinema.Projection.Lens 17 | alias Oban.Pro.Workflow 18 | 19 | defmodule Worker do 20 | @moduledoc false 21 | use Oban.Pro.Worker, queue: :default 22 | 23 | alias Cinema.Engine 24 | alias Cinema.Projection 25 | 26 | @impl Oban.Pro.Worker 27 | def process(%{args: %{"projection" => encoded_projection, "lens" => encoded_lens}}) do 28 | lens = encoded_lens |> Base.decode64!() |> :erlang.binary_to_term() 29 | projection = encoded_projection |> Base.decode64!() |> :erlang.binary_to_term() 30 | 31 | :ok = Engine.do_exec(projection, lens) 32 | end 33 | end 34 | 35 | # TODO: support custom timeouts 36 | # @timeout :timer.minutes(1) 37 | 38 | @doc """ 39 | Executes the given `Projection.t()` using the Task engine. Returns `{:ok, projection}` if the projection 40 | was successfully executed, otherwise returns `{:error, projection}`. 41 | 42 | You can fetch the output of the `Projection.t()` (assuming it was successfully executed) by passing the 43 | returned `Projection.t()` to `Cinema.Projection.fetch/1` function. 44 | 45 | ## Options 46 | 47 | * `:timeout` - The timeout to use when executing the projection. Defaults to `1 minute`. 48 | * `:skip_dependencies` - Defaults to `false`. If `true`, skips executing the dependencies of the given projection. 49 | this is very useful for testing purposes where you use `ExMachina` to "mock" the results of all input projections. 50 | 51 | """ 52 | @impl Cinema.Engine 53 | def exec(%Projection{} = projection, opts \\ []) do 54 | # TODO: support custom timeouts 55 | # timeout = opts[:timeout] || @timeout 56 | priority = opts[:priority] || nil 57 | queue = opts[:queue] || :default 58 | oban = opts[:oban] || Oban 59 | 60 | # Mainly for testing purposes, you can set `relative_to` to offset all time calculations 61 | # to fall relative to a certain date or datetime. 62 | # 63 | # Can also be used to schedule a job that failed for a previous day, if that happens for 64 | # some reason. 65 | relative_to = Keyword.get(opts, :relative_to, DateTime.utc_now()) 66 | 67 | # If `immediately: false`, then we schedule the job for 00:00 UTC, which is also the default. 68 | # Pass `immediately: true` to schedule the job for the current time. 69 | 70 | scheduled_at = 71 | if Keyword.get(opts, :immediately) do 72 | relative_to 73 | else 74 | %{relative_to | hour: 1, minute: 0, second: 0, microsecond: 0} 75 | end 76 | 77 | encoded_lens = projection.lens |> :erlang.term_to_binary() |> Base.encode64() 78 | 79 | normalized_exec_graph = 80 | Enum.map(projection.exec_graph, &List.wrap/1) 81 | 82 | workflow_dependencies = 83 | Enum.zip(normalized_exec_graph, [[] | normalized_exec_graph]) 84 | 85 | workflow = 86 | Enum.reduce(workflow_dependencies, Workflow.new(), fn {projections, deps}, workflow -> 87 | Enum.reduce(projections, workflow, fn projection, workflow -> 88 | encoded_projection = projection |> :erlang.term_to_binary() |> Base.encode64() 89 | 90 | Workflow.add( 91 | workflow, 92 | projection, 93 | Worker.new(%{projection: encoded_projection, lens: encoded_lens}, 94 | scheduled_at: scheduled_at, 95 | priority: priority, 96 | queue: queue 97 | ), 98 | deps: deps 99 | ) 100 | end) 101 | end) 102 | 103 | %Oban.Job{} = 104 | job = 105 | workflow 106 | |> Oban.insert_all() 107 | |> List.first() 108 | |> Map.get(:meta, %{}) 109 | |> Map.get("workflow_id") 110 | |> then(&Workflow.get_job(oban, &1, projection.terminal_projection)) 111 | 112 | {:ok, %Projection{projection | meta: %{job: job}}} 113 | end 114 | 115 | @doc """ 116 | Fetches the output of a `Projection.t()` which has been executed using `Cinema.Engine.exec/2`. 117 | """ 118 | @impl Cinema.Engine 119 | def fetch(%Projection{} = projection) do 120 | case projection.meta[:job] do 121 | nil -> 122 | {:error, "The given projection was not executed using the `#{__MODULE__}` engine."} 123 | 124 | job -> 125 | job = projection.read_repo.reload!(job, force: true) 126 | 127 | cond do 128 | job.state == "completed" -> 129 | {:ok, Lens.apply(projection.lens, projection.terminal_projection, as_stream: true)} 130 | 131 | job.state in ["ready", "scheduled", "executing"] -> 132 | retry_counter = Process.get({__MODULE__, job.id, :retry}, 0) 133 | 134 | if retry_counter <= 10 do 135 | Process.put({__MODULE__, job.id, :retry}, retry_counter + 1) 136 | 137 | exp_backoff = :timer.seconds(1 * (retry_counter * 2)) 138 | Process.sleep(exp_backoff) 139 | 140 | fetch(projection) 141 | else 142 | {:error, "The workflow failed to complete within the given timeout."} 143 | end 144 | 145 | true -> 146 | {:error, "The workflow failed with status: `#{job.state}`."} 147 | end 148 | end 149 | end 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /.credo.exs: -------------------------------------------------------------------------------- 1 | alias Credo.Check 2 | 3 | %{ 4 | configs: [ 5 | %{ 6 | name: "default", 7 | files: %{ 8 | included: [ 9 | "lib/", 10 | "src/", 11 | "web/", 12 | "apps/*/lib/", 13 | "apps/*/src/", 14 | "apps/*/web/" 15 | ], 16 | excluded: [~r"/tests/", ~r"/_build/", ~r"/deps/", ~r"/node_modules/"] 17 | }, 18 | plugins: [], 19 | requires: [], 20 | strict: true, 21 | parse_timeout: 5000, 22 | color: true, 23 | checks: %{ 24 | enabled: [ 25 | ## Consistency Checks ------------------------------------------------ 26 | {Check.Consistency.ExceptionNames, []}, 27 | {Check.Consistency.LineEndings, []}, 28 | {Check.Consistency.ParameterPatternMatching, []}, 29 | {Check.Consistency.SpaceAroundOperators, []}, 30 | {Check.Consistency.SpaceInParentheses, []}, 31 | {Check.Consistency.TabsOrSpaces, []}, 32 | 33 | ## Design Checks ----------------------------------------------------- 34 | {Check.Design.AliasUsage, if_nested_deeper_than: 2}, 35 | 36 | ## Readability Checks ------------------------------------------------ 37 | {Check.Readability.AliasOrder, []}, 38 | {Check.Readability.FunctionNames, []}, 39 | {Check.Readability.LargeNumbers, []}, 40 | {Check.Readability.MaxLineLength, priority: :low, max_length: 120}, 41 | {Check.Readability.ModuleAttributeNames, []}, 42 | {Check.Readability.ModuleDoc, []}, 43 | {Check.Readability.ModuleNames, []}, 44 | {Check.Readability.ParenthesesInCondition, []}, 45 | {Check.Readability.ParenthesesOnZeroArityDefs, []}, 46 | {Check.Readability.PipeIntoAnonymousFunctions, []}, 47 | {Check.Readability.PredicateFunctionNames, []}, 48 | {Check.Readability.PreferImplicitTry, []}, 49 | {Check.Readability.RedundantBlankLines, []}, 50 | {Check.Readability.Semicolons, []}, 51 | {Check.Readability.SpaceAfterCommas, []}, 52 | {Check.Readability.StringSigils, []}, 53 | {Check.Readability.TrailingBlankLine, []}, 54 | {Check.Readability.TrailingWhiteSpace, []}, 55 | {Check.Readability.UnnecessaryAliasExpansion, []}, 56 | {Check.Readability.VariableNames, []}, 57 | {Check.Readability.WithSingleClause, []}, 58 | 59 | ## Refactoring Opportunities ----------------------------------------- 60 | {Check.Refactor.Apply, []}, 61 | {Check.Refactor.CondStatements, []}, 62 | {Check.Refactor.CyclomaticComplexity, [max_complexity: 12]}, 63 | {Check.Refactor.FunctionArity, []}, 64 | {Check.Refactor.LongQuoteBlocks, []}, 65 | {Check.Refactor.MatchInCondition, []}, 66 | {Check.Refactor.MapJoin, []}, 67 | {Check.Refactor.NegatedConditionsInUnless, []}, 68 | {Check.Refactor.NegatedConditionsWithElse, []}, 69 | {Check.Refactor.Nesting, [max_nesting: 4]}, 70 | {Check.Refactor.UnlessWithElse, []}, 71 | {Check.Refactor.WithClauses, []}, 72 | {Check.Refactor.FilterFilter, []}, 73 | {Check.Refactor.RejectReject, []}, 74 | {Check.Refactor.RedundantWithClauseResult, []}, 75 | 76 | ## Warnings ---------------------------------------------------------- 77 | {Check.Warning.ApplicationConfigInModuleAttribute, []}, 78 | {Check.Warning.BoolOperationOnSameValues, []}, 79 | {Check.Warning.ExpensiveEmptyEnumCheck, []}, 80 | {Check.Warning.IExPry, []}, 81 | {Check.Warning.IoInspect, []}, 82 | {Check.Warning.OperationOnSameValues, []}, 83 | {Check.Warning.OperationWithConstantResult, []}, 84 | {Check.Warning.RaiseInsideRescue, []}, 85 | {Check.Warning.SpecWithStruct, []}, 86 | {Check.Warning.WrongTestFileExtension, []}, 87 | {Check.Warning.UnusedEnumOperation, []}, 88 | {Check.Warning.UnusedFileOperation, []}, 89 | {Check.Warning.UnusedKeywordOperation, []}, 90 | {Check.Warning.UnusedListOperation, []}, 91 | {Check.Warning.UnusedPathOperation, []}, 92 | {Check.Warning.UnusedRegexOperation, []}, 93 | {Check.Warning.UnusedStringOperation, []}, 94 | {Check.Warning.UnusedTupleOperation, []}, 95 | {Check.Warning.UnsafeExec, []}, 96 | 97 | ## Checks which should always be on for consistency-sake IMO --------- 98 | {Check.Consistency.MultiAliasImportRequireUse, []}, 99 | {Check.Consistency.UnusedVariableNames, force: :meaningful}, 100 | {Check.Design.DuplicatedCode, []}, 101 | {Check.Design.SkipTestWithoutComment, []}, 102 | {Check.Readability.ImplTrue, []}, 103 | {Check.Readability.MultiAlias, []}, 104 | {Check.Readability.NestedFunctionCalls, []}, 105 | {Check.Readability.SeparateAliasRequire, []}, 106 | {Check.Readability.SingleFunctionToBlockPipe, []}, 107 | {Check.Readability.SinglePipe, []}, 108 | {Check.Readability.StrictModuleLayout, []}, 109 | {Check.Readability.WithCustomTaggedTuple, []}, 110 | {Check.Refactor.ABCSize, [max_size: 70]}, 111 | {Check.Refactor.DoubleBooleanNegation, []}, 112 | {Check.Refactor.FilterReject, []}, 113 | {Check.Refactor.MapMap, []}, 114 | {Check.Refactor.NegatedIsNil, []}, 115 | {Check.Refactor.PipeChainStart, []}, 116 | {Check.Refactor.RejectFilter, []}, 117 | {Check.Refactor.VariableRebinding, []}, 118 | {Check.Warning.LeakyEnvironment, []}, 119 | {Check.Warning.MapGetUnsafePass, []}, 120 | {Check.Warning.MixEnv, []}, 121 | {Check.Warning.UnsafeToAtom, []}, 122 | 123 | ## Causes Issues with Phoenix ---------------------------------------- 124 | {Check.Readability.Specs, []}, 125 | {Check.Refactor.ModuleDependencies, [max_deps: 19]}, 126 | 127 | ## Optional (move to `disabled` based on app domain) ----------------- 128 | {Check.Refactor.IoPuts, []} 129 | ], 130 | disabled: [ 131 | ## Checks which are overly limiting ---------------------------------- 132 | {Check.Design.TagTODO, exit_status: 2}, 133 | {Check.Design.TagFIXME, []}, 134 | {Check.Readability.BlockPipe, []}, 135 | {Check.Readability.AliasAs, []}, 136 | {Check.Refactor.AppendSingleItem, []}, 137 | 138 | ## Incompatible with modern versions of Elixir ----------------------- 139 | {Check.Refactor.MapInto, []}, 140 | {Check.Warning.LazyLogging, []} 141 | ] 142 | } 143 | } 144 | ] 145 | } 146 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "acceptor_pool": {:hex, :acceptor_pool, "1.0.0", "43c20d2acae35f0c2bcd64f9d2bde267e459f0f3fd23dab26485bf518c281b21", [:rebar3], [], "hexpm", "0cbcd83fdc8b9ad2eee2067ef8b91a14858a5883cb7cd800e6fcd5803e158788"}, 3 | "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, 4 | "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"}, 5 | "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"}, 6 | "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, 7 | "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, 8 | "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, 9 | "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, 10 | "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, 11 | "earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"}, 12 | "ecto": {:hex, :ecto, "3.12.3", "1a9111560731f6c3606924c81c870a68a34c819f6d4f03822f370ea31a582208", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9efd91506ae722f95e48dc49e70d0cb632ede3b7a23896252a60a14ac6d59165"}, 13 | "ecto_sql": {:hex, :ecto_sql, "3.12.0", "73cea17edfa54bde76ee8561b30d29ea08f630959685006d9c6e7d1e59113b7d", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dc9e4d206f274f3947e96142a8fdc5f69a2a6a9abb4649ef5c882323b6d512f0"}, 14 | "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, 15 | "ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"}, 16 | "ex_machina": {:hex, :ex_machina, "2.8.0", "a0e847b5712065055ec3255840e2c78ef9366634d62390839d4880483be38abe", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "79fe1a9c64c0c1c1fab6c4fa5d871682cb90de5885320c187d117004627a7729"}, 17 | "excoveralls": {:hex, :excoveralls, "0.18.3", "bca47a24d69a3179951f51f1db6d3ed63bca9017f476fe520eb78602d45f7756", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "746f404fcd09d5029f1b211739afb8fb8575d775b21f6a3908e7ce3e640724c6"}, 18 | "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"}, 19 | "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"}, 20 | "grpcbox": {:hex, :grpcbox, "0.17.1", "6e040ab3ef16fe699ffb513b0ef8e2e896da7b18931a1ef817143037c454bcce", [:rebar3], [{:acceptor_pool, "~> 1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~> 0.15.1", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~> 0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~> 0.9.1", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "4a3b5d7111daabc569dc9cbd9b202a3237d81c80bf97212fbc676832cb0ceb17"}, 21 | "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"}, 22 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 23 | "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, 24 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, 25 | "makeup_erlang": {:hex, :makeup_erlang, "1.0.1", "c7f58c120b2b5aa5fd80d540a89fdf866ed42f1f3994e4fe189abebeab610839", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "8a89a1eeccc2d798d6ea15496a6e4870b75e014d1af514b1b71fa33134f57814"}, 26 | "mix_test_watch": {:hex, :mix_test_watch, "1.2.0", "1f9acd9e1104f62f280e30fc2243ae5e6d8ddc2f7f4dc9bceb454b9a41c82b42", [:mix], [{:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}], "hexpm", "278dc955c20b3fb9a3168b5c2493c2e5cffad133548d307e0a50c7f2cfbf34f6"}, 27 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, 28 | "opentelemetry": {:hex, :opentelemetry, "1.4.0", "f928923ed80adb5eb7894bac22e9a198478e6a8f04020ae1d6f289fdcad0b498", [:rebar3], [{:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "50b32ce127413e5d87b092b4d210a3449ea80cd8224090fe68d73d576a3faa15"}, 29 | "opentelemetry_api": {:hex, :opentelemetry_api, "1.3.1", "83b4713593f80562d9643c4ab0b6f80f3c5fa4c6d0632c43e11b2ccb6b04dfa7", [:mix, :rebar3], [{:opentelemetry_semantic_conventions, "~> 0.2", [hex: :opentelemetry_semantic_conventions, repo: "hexpm", optional: false]}], "hexpm", "9e8a5cc38671e3ac61be48abe5f6b3afdbbb50a1dc08b7950c56f169611505c1"}, 30 | "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.7.0", "dec4e90c0667cf11a3642f7fe71982dbc0c6bfbb8725a0b13766830718cf0d98", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.4.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.3.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "d0f25f6439ec43f2561537c3fabbe177b38547cddaa3a692cbb8f4770dbefc1e"}, 31 | "opentelemetry_process_propagator": {:hex, :opentelemetry_process_propagator, "0.2.2", "85244a49f0c32ae1e2f3d58c477c265bd6125ee3480ade82b0fa9324b85ed3f0", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "04db13302a34bea8350a13ed9d49c22dfd32c4bc590d8aa88b6b4b7e4f346c61"}, 32 | "opentelemetry_semantic_conventions": {:hex, :opentelemetry_semantic_conventions, "0.2.0", "b67fe459c2938fcab341cb0951c44860c62347c005ace1b50f8402576f241435", [:mix, :rebar3], [], "hexpm", "d61fa1f5639ee8668d74b527e6806e0503efc55a42db7b5f39939d84c07d6895"}, 33 | "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.2", "410ab4d76b0921f42dbccbe5a7c831b8125282850be649ee1f70050d3961118a", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "641ab469deb181957ac6d59bce6e1321d5fe2a56df444fc9c19afcad623ab253"}, 34 | "postgrex": {:hex, :postgrex, "0.19.1", "73b498508b69aded53907fe48a1fee811be34cc720e69ef4ccd568c8715495ea", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "8bac7885a18f381e091ec6caf41bda7bb8c77912bb0e9285212829afe5d8a8f8"}, 35 | "sibyl": {:hex, :sibyl, "0.1.9", "58c85d4253687555dbda18386f7d51b60801a870ad99a2e4a91ea4a8a1717b84", [:mix], [{:decorator, "~> 1.2", [hex: :decorator, repo: "hexpm", optional: false]}, {:jason, "~> 1.3", [hex: :jason, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.1", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.1", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:opentelemetry_exporter, "~> 1.1", [hex: :opentelemetry_exporter, repo: "hexpm", optional: false]}, {:opentelemetry_process_propagator, "~> 0.1.0 or ~> 0.2.0", [hex: :opentelemetry_process_propagator, repo: "hexpm", optional: false]}, {:opentelemetry_telemetry, "~> 1.0", [hex: :opentelemetry_telemetry, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "498ebfef8aaceb952e27333a755b8b1f28051e2edad87e3c05cd05c2c2b6a7f7"}, 36 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, 37 | "styler": {:hex, :styler, "0.11.9", "2595393b94e660cd6e8b582876337cc50ff047d184ccbed42fdad2bfd5d78af5", [:mix], [], "hexpm", "8b7806ba1fdc94d0a75127c56875f91db89b75117fcc67572661010c13e1f259"}, 38 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 39 | "tls_certificate_check": {:hex, :tls_certificate_check, "1.24.0", "d00e2887551ff8cdae4d0340d90d9fcbc4943c7b5f49d32ed4bc23aff4db9a44", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "90b25a58ee433d91c17f036d4d354bf8859a089bfda60e68a86f8eecae45ef1b"}, 40 | } 41 | -------------------------------------------------------------------------------- /lib/cinema/projection.ex: -------------------------------------------------------------------------------- 1 | defmodule Cinema.Projection do 2 | @moduledoc """ 3 | This module is responsible to defining "projections", which are similar to "views" built into 4 | most RDBMS systems. 5 | 6 | ## Key Concepts 7 | 8 | A projection is simply a module which tells `Cinema` three things: 9 | 10 | 1) What inputs (usually other projections) are required to execute a projection. 11 | 2) What outputs (usually a `Stream`-compatible term) are produced by executing a projection. 12 | 3) Most importantly, the instructions required to derive data for the projection. 13 | 14 | It is important to call out that a `Projection` is different from defining a `View` in a traditional 15 | RDBMS system. A `View` is a stored query that is executed on-demand, while a `Projection` is a 16 | series of instructions that can be executed to derive a result, which is optionally stored in a table 17 | for querying. 18 | 19 | While `View`s can be materialized, at least in PostgreSQL, you're usually forced to refresh the 20 | entire view whenever you want to update any stored data. `Projection`s on the other hand are designed 21 | to allow some form of incremental materialization, where only data relevent to the `Cinema.Lens` 22 | is required to be refreshed. 23 | 24 | Currently, Projections need to be manually called to be refreshed; though future work may include 25 | automatic triggers for refreshing Projections when their inputs change. 26 | 27 | Please see `Cinema.project/2` for more details on how projections are executed. 28 | 29 | ## Materialized vs Virtual Projections 30 | 31 | Projections currently come in two flavours: virtual and materialized, defaulting to materialized 32 | for most cases. 33 | 34 | A materialized projection requires users to define an `Ecto.Schema` schema and associated database 35 | migration to store the projection's output. These projections can then be indexed and queried like any 36 | other schema. 37 | 38 | Virtual projections on the other hand are not stored in the database, and are instead derived on-the-fly 39 | by executing the projection's instructions (if any) and immediately returning the Projection's output. 40 | 41 | Generally, virtual projections are used for kickstarting larger `Cinema` data pipelines from 42 | your application's existing database tables, while materialized projections are used for storing 43 | intermediate results for later querying. 44 | 45 | You can control whether a projection is materialized or virtual by setting the `:materialized` option 46 | to `true` or `false` respectively. It is set to `true` by default. 47 | 48 | ## Dematerialization 49 | 50 | Materialized Projections are automatically dematerialized prior to materialization, which means that 51 | the projection's output is deleted before the projection is executed. This is useful for ensuring that 52 | the projection's output is always up-to-date, and can be used to ensure that the projection's output 53 | is idempotent. 54 | 55 | Only data which is in "scope" of a given Projection and Lens is dematerialized, so generally speaking, 56 | assuming `(dematerialize(input, lens); materialize(input, lens)) == materialize(input, lens)`, you 57 | don't have to worry about dematerialization causing data loss. 58 | 59 | If you have a projection that is not idempotent, you can disable dematerialization by setting the 60 | `:dematerialize` option to `false`, though you should be aware that you may want to manually manage 61 | the projection's output to ensure that it is up-to-date and any outdated data is cleaned up. 62 | 63 | ## Side Effects 64 | 65 | Projections can also, optionally, trigger side-effects by writing to the database or other external 66 | systems. This can be done by executing the desires side-effect causing code within a Projection's 67 | `derive/2` callback. 68 | 69 | Generally in such projections, it is inadvisable to dematerialize the projection, as the side-effects 70 | may not be idempotent, and may rely on the state of the Projection's output as a log of what has 71 | already been processed. 72 | 73 | If this is the cases, dematerialization can be disabled by setting the `:dematerialize` option to `false`. 74 | 75 | ## Telemetry 76 | 77 | All Projections emit Telemetry events when they are executed, which can be used to monitor said projections. 78 | 79 | The `Cinema` Telemetry events are as follows: TBA 80 | """ 81 | 82 | use Sibyl 83 | 84 | alias Cinema.Projection.Lens 85 | alias Cinema.Projection.Rank 86 | alias Cinema.Utils 87 | 88 | # Opts can contain: 89 | # - :required - a list of required fields that must be defined in the pattern 90 | # - :read_from - the Ecto repo to use for querying, could be `MyApp.Repo`, `MyApp.Repo.replica()`, etc 91 | # - :write_to - the Ecto repo to use for writing, could be `MyApp.Repo`, etc 92 | defmacro __using__(opts \\ []) do 93 | quote do 94 | @behaviour unquote(__MODULE__) 95 | 96 | @after_compile unquote(__MODULE__) 97 | use Ecto.Schema 98 | 99 | import Ecto.Query 100 | 101 | # TODO: use something that gives us queryable functionality 102 | 103 | require unquote(__MODULE__) 104 | 105 | @doc false 106 | def opts, do: unquote(opts) 107 | 108 | @impl unquote(__MODULE__) 109 | def derivation({_input, _stream}, _lens) do 110 | raise ArgumentError, message: "Not Implemented." 111 | end 112 | 113 | @impl unquote(__MODULE__) 114 | def fields do 115 | if unquote(__MODULE__).virtual?(__MODULE__) do 116 | [] 117 | else 118 | apply(__MODULE__, :__schema__, [:fields]) 119 | end 120 | end 121 | 122 | defoverridable derivation: 2 123 | end 124 | end 125 | 126 | @type implementation :: module() 127 | @type input :: implementation() 128 | @type output :: Ecto.Queryable.t() | [map()] | term() 129 | @type t :: %__MODULE__{} 130 | 131 | @callback fields() :: [atom()] 132 | @callback inputs() :: [implementation()] 133 | @callback output() :: output() 134 | 135 | @callback derivation({input(), output()}, Lens.t()) :: term() 136 | 137 | defstruct [ 138 | :terminal_projection, 139 | :exec_graph, 140 | :valid?, 141 | :engine, 142 | :read_repo, 143 | :write_repo, 144 | meta: %{}, 145 | lens: %Lens{} 146 | ] 147 | 148 | @doc """ 149 | Builds a new instance of the given projection, with the given lens. Does not execute the projection. 150 | See `Cinema.Engine` for functions that operate on `Projection.t()`s. 151 | 152 | Takes an optional `lens` argument, which is a `Cinema.Lens.t()` struct that can be used to 153 | scope the projection's inputs and outputs. 154 | 155 | Also takes an optional `opts` argument, which is a `Keyword.t()` list of options that can be used to 156 | configure the projection's behavior. 157 | 158 | ## Options 159 | 160 | * `:application` - The application to use when fetching the list of all projections. Defaults to `nil`. 161 | 162 | """ 163 | @spec build!(implementation, Lens.t(), Keyword.t()) :: t() 164 | def build!(projection, lens \\ %Lens{}, opts \\ []) do 165 | unless implemented?(projection) do 166 | raise ArgumentError, 167 | message: """ 168 | The given projection does not implement the `Cinema.Projection` behaviour. 169 | """ 170 | end 171 | 172 | application = opts[:application] 173 | opts = projection.opts() 174 | 175 | write_repo = opts[:write_repo] || opts[:repo] 176 | read_repo = opts[:read_repo] || write_repo 177 | 178 | %__MODULE__{ 179 | lens: lens, 180 | terminal_projection: projection, 181 | write_repo: write_repo, 182 | read_repo: read_repo, 183 | exec_graph: dependencies(application, projection) 184 | } 185 | end 186 | 187 | @doc false 188 | @spec __after_compile__(Macro.Env.t(), binary()) :: term() 189 | def __after_compile__(env, _bytecode) do 190 | implementation = env.module 191 | 192 | unless virtual?(implementation) do 193 | required_keys = MapSet.new(implementation.opts[:required] || []) 194 | defined_keys = implementation |> struct([]) |> Map.keys() |> MapSet.new() 195 | 196 | # TODO: implement custom exception 197 | unless MapSet.subset?(required_keys, defined_keys) do 198 | raise ArgumentError, 199 | message: """ 200 | `#{inspect(__MODULE__)}`s has been configured to required the following fields: `#{inspect(MapSet.to_list(required_keys))}` 201 | """ 202 | end 203 | end 204 | end 205 | 206 | @spec materialize(rows_or_queryable :: term()) :: term() 207 | defmacro materialize(rows_or_queryable) do 208 | caller = __CALLER__.module 209 | {function, arity} = __CALLER__.function 210 | 211 | # TODO: custom exception 212 | unless function == :derivation && arity == 2 do 213 | raise ArgumentError, 214 | message: """ 215 | `#{inspect(__MODULE__)}.materialize/1` can only be called from within a `#{__MODULE__}`'s `derivation/2` callback. 216 | """ 217 | end 218 | 219 | quote bind_quoted: [caller: caller, rows_or_queryable: rows_or_queryable] do 220 | opts = __MODULE__.opts() 221 | fields = __MODULE__.fields() 222 | 223 | write_repo = opts[:write_repo] || opts[:repo] 224 | read_repo = opts[:read_repo] || write_repo 225 | 226 | {_count, rows} = 227 | cond do 228 | is_list(rows_or_queryable) -> 229 | rows_or_queryable 230 | |> Enum.map(&Map.take(&1, fields)) 231 | |> then(&write_repo.insert_all(caller, &1, opts)) 232 | 233 | is_struct(rows_or_queryable, Ecto.Query) and read_repo != write_repo -> 234 | # TODO: utilize sentinel values to avoid re-inserting the same data over and over 235 | rows = 236 | rows_or_queryable |> read_repo.all(opts) |> Enum.map(&Utils.sanitize_timestamps/1) 237 | 238 | postgres_max_parameters = 65_535 239 | paramters_per_row = rows |> List.first(%{}) |> map_size() |> max(1) 240 | max_rows_per_batch = (postgres_max_parameters / paramters_per_row) |> floor() |> max(1) 241 | 242 | rows 243 | |> Enum.chunk_every(max_rows_per_batch) 244 | |> Enum.reduce({0, []}, fn batch, {count, acc} -> 245 | {count_inc, acc_inc} = write_repo.insert_all(caller, batch, opts) 246 | {count + count_inc, (acc || []) ++ rows} 247 | end) 248 | 249 | is_struct(rows_or_queryable, Ecto.Query) and read_repo == write_repo -> 250 | write_repo.insert_all(caller, rows_or_queryable, opts) 251 | end 252 | 253 | rows 254 | end 255 | end 256 | 257 | @spec dematerialize(Lens.t()) :: term() 258 | defmacro dematerialize(lens) do 259 | caller = __CALLER__.module 260 | {function, arity} = __CALLER__.function 261 | 262 | # TODO: custom exception 263 | unless function == :derivation && arity == 2 do 264 | raise ArgumentError, 265 | message: """ 266 | `#{inspect(__MODULE__)}.dematerialize/1` can only be called from within a `#{__MODULE__}`'s `derivation/2` callback. 267 | """ 268 | end 269 | 270 | quote bind_quoted: [caller: caller, lens: lens] do 271 | opts = caller.opts() 272 | write_repo = opts[:write_repo] || opts[:repo] 273 | 274 | unless is_struct(lens, Lens) do 275 | raise ArgumentError, 276 | message: """ 277 | `#{inspect(__MODULE__)}.dematerialize/1` expects an argument of type `Cinema.Lens.t()`. 278 | """ 279 | end 280 | 281 | unless Keyword.get(caller.opts(), :dematerialize, true) do 282 | raise ArgumentError, 283 | message: """ 284 | `#{inspect(__MODULE__)}` has not been configured to be dematerialized. 285 | """ 286 | end 287 | 288 | __MODULE__ 289 | |> Ecto.Query.where(^lens.filters) 290 | |> write_repo.delete_all() 291 | end 292 | end 293 | 294 | @doc """ 295 | Returns `true` if the given module is a virtual projection (does not define a schema), otherwise returns `false`. 296 | 297 | See examples: 298 | 299 | ```elixir 300 | iex> Cinema.Projection.virtual?(Enum) 301 | false 302 | 303 | iex> defmodule MyApp.VirtualProjection do 304 | ...> use Cinema.Projection 305 | ...> 306 | ...> @impl Cinema.Projection 307 | ...> def inputs, do: [] 308 | ...> 309 | ...> @impl Cinema.Projection 310 | ...> def ouput, do: [] 311 | ...> end 312 | iex> Cinema.Projection.virtual?(MyApp.VirtualProjection) 313 | true 314 | ``` 315 | """ 316 | @spec virtual?(implementation :: module()) :: boolean() 317 | def virtual?(implementation) do 318 | implemented?(implementation) and not function_exported?(implementation, :__schema__, 1) 319 | end 320 | 321 | @doc """ 322 | Returns `true` if the given module implements the `#{__MODULE__}` behaviour, otherwise returns `false`. 323 | 324 | See examples: 325 | 326 | ```elixir 327 | iex> Cinema.Projection.implemented?(Enum) 328 | false 329 | 330 | iex> defmodule MyApp.SomeProjection do 331 | ...> use Cinema.Projection 332 | ...> 333 | ...> @impl Cinema.Projection 334 | ...> def inputs, do: [] 335 | ...> 336 | ...> @impl Cinema.Projection 337 | ...> def ouput, do: [] 338 | ...> end 339 | iex> Cinema.Projection.implemented?(MyApp.SomeProjection) 340 | true 341 | ``` 342 | """ 343 | @spec implemented?(module()) :: boolean() 344 | def implemented?(module) do 345 | Utils.implemented?(module, __MODULE__) 346 | end 347 | 348 | @doc "Lists all defined projections defined in the given application or loaded in the VM." 349 | @spec list(application :: atom() | nil) :: [module()] 350 | def list(application \\ nil) do 351 | static_modules = 352 | case :application.get_key(application, :modules) do 353 | {:ok, static_modules} -> 354 | static_modules 355 | 356 | _otherwise -> 357 | [] 358 | end 359 | 360 | dynamic_modules = 361 | :code.all_loaded() |> Enum.map(&elem(&1, 0)) |> Enum.reject(&(&1 in static_modules)) 362 | 363 | for module <- static_modules ++ dynamic_modules, 364 | match?({:module, _module}, :code.ensure_loaded(module)), 365 | implemented?(module) do 366 | module 367 | end 368 | end 369 | 370 | @doc "Returns the dependency graph needed to execute the given projection." 371 | @spec dependencies(module()) :: [module()] 372 | @spec dependencies(application :: atom(), module()) :: [module()] 373 | def dependencies(application \\ nil, implementation) do 374 | application 375 | |> list() 376 | |> Enum.map(&{&1, &1.inputs()}) 377 | |> Rank.derive!(implementation) 378 | end 379 | end 380 | --------------------------------------------------------------------------------