├── test ├── e2e │ ├── .gitignore │ ├── .tool-versions │ └── .template.env ├── realtime_web │ ├── views │ │ ├── page_view_test.exs │ │ ├── layout_view_test.exs │ │ └── error_view_test.exs │ ├── live │ │ ├── inspector_live │ │ │ └── index_test.exs │ │ ├── page_live │ │ │ └── index_test.exs │ │ ├── status_live │ │ │ └── index_test.exs │ │ └── tenants_live │ │ │ └── index_test.exs │ ├── controllers │ │ ├── page_controller_test.exs │ │ ├── openapi_controller_test.exs │ │ └── live_dasboard_test.exs │ ├── channels │ │ ├── tenant_rate_limiters_test.exs │ │ └── auth │ │ │ └── channels_authorization_test.exs │ ├── integration │ │ └── tracing_test.exs │ └── plugs │ │ └── rate_limiter_test.exs ├── realtime │ ├── helpers_test.exs │ ├── oid_test.exs │ ├── adapters │ │ └── postgres │ │ │ └── protocol_test.exs │ ├── monitoring │ │ ├── prom_ex_test.exs │ │ ├── erl_sys_mon_test.exs │ │ ├── distributed_metrics_test.exs │ │ └── latency_test.exs │ ├── logs_test.exs │ ├── telemetry │ │ └── logger_test.exs │ ├── signal_handler_test.exs │ ├── tenants │ │ ├── migrations_test.exs │ │ ├── rebalancer_test.exs │ │ └── connect │ │ │ └── register_process_test.exs │ └── metrics_cleaner_test.exs ├── support │ ├── replication_test_handler.ex │ ├── tracing.ex │ ├── channel_case.ex │ ├── joken_current_time_mock.ex │ ├── rate_counter_helper.ex │ ├── cleanup.ex │ ├── containers │ │ └── container.ex │ ├── conn_case.ex │ ├── tenant_connection.ex │ ├── data_case.ex │ └── metrics_helper.ex └── api_jwt_secret_test.exs ├── .tool-versions ├── lib ├── realtime.ex ├── realtime_web │ ├── views │ │ ├── layout_view.ex │ │ ├── error_view.ex │ │ ├── changeset_view.ex │ │ ├── tenant_view.ex │ │ └── error_helpers.ex │ ├── controllers │ │ ├── ping_controller.ex │ │ ├── page_controller.ex │ │ └── broadcast_controller.ex │ ├── channels │ │ ├── presence.ex │ │ ├── payloads │ │ │ ├── presence.ex │ │ │ ├── broadcast │ │ │ │ └── replay.ex │ │ │ ├── postgres_change.ex │ │ │ ├── broadcast.ex │ │ │ ├── config.ex │ │ │ └── join.ex │ │ ├── realtime_channel │ │ │ └── assign.ex │ │ ├── tenant_rate_limiters.ex │ │ └── auth │ │ │ └── channels_authorization.ex │ ├── live │ │ ├── page_live │ │ │ ├── index.ex │ │ │ └── index.html.heex │ │ ├── time_live.ex │ │ ├── status_live │ │ │ ├── index.html.heex │ │ │ └── index.ex │ │ ├── ping_live.ex │ │ └── tenants_live │ │ │ └── index.html.heex │ ├── templates │ │ └── layout │ │ │ └── root.html.heex │ ├── gettext.ex │ ├── plugs │ │ ├── rate_limiter.ex │ │ ├── baggage_request_id.ex │ │ ├── assign_tenant.ex │ │ └── auth_tenant.ex │ ├── api_spec.ex │ ├── dashboard │ │ └── process_dump.ex │ ├── socket │ │ └── user_broadcast.ex │ └── telemetry.ex ├── realtime │ ├── tenants │ │ ├── repo │ │ │ └── migrations │ │ │ │ ├── 20231204144025_enable_channels_rls.ex │ │ │ │ ├── 20240919163305_change_messages_id_type.ex │ │ │ │ ├── 20211122062447_grant_realtime_usage_to_authenticated_role.ex │ │ │ │ ├── 20250506224012_subscription_index_bridging_disabled.ex │ │ │ │ ├── 20241019105805_uuid_auto_generation.ex │ │ │ │ ├── 20250523164012_run_subscription_index_bridging_disabled.ex │ │ │ │ ├── 20240109165339_add_update_grant_to_channels.ex │ │ │ │ ├── 20240805133720_logged_messages_table.ex │ │ │ │ ├── 20240108234812_add_channels_column_for_write_check.ex │ │ │ │ ├── 20240801235015_unlogged_messages_table.ex │ │ │ │ ├── 20250107150512_realtime_subscription_unlogged.ex │ │ │ │ ├── 20231018144023_create_channels.ex │ │ │ │ ├── 20250905041441_create_messages_replay_index.ex │ │ │ │ ├── 20240418121054_remove_check_columns.ex │ │ │ │ ├── 20231204144024_create_rls_helper_functions.ex │ │ │ │ ├── 20250110162412_realtime_subscription_logged.ex │ │ │ │ ├── 20241108114728_messages_using_uuid.ex │ │ │ │ ├── 20211116213355_create_realtime_cast_function.ex │ │ │ │ ├── 20250123174212_remove_unused_publications.ex │ │ │ │ ├── 20240311171622_add_insert_and_delete_grant_to_channels.ex │ │ │ │ ├── 20241130184212_recreate_entity_index_using_btree.ex │ │ │ │ ├── 20231204144023_set_required_grants.ex │ │ │ │ ├── 20240227174441_add_broadcast_permissions_table.ex │ │ │ │ ├── 20250714121412_broadcast_send_error_logging.ex │ │ │ │ ├── 20240321100241_add_presences_permissions_table.ex │ │ │ │ ├── 20211116213934_create_realtime_is_visible_through_filters_function.ex │ │ │ │ ├── 20211116051442_create_realtime_check_equality_op_function.ex │ │ │ │ ├── 20241220123912_realtime_send_handle_exceptions_remove_partition_creation.ex │ │ │ │ ├── 20241121104152_fix_send_function_.ex │ │ │ │ ├── 20241224161212_realtime_send_sets_config.ex │ │ │ │ ├── 20250128220012_realtime_send_sets_topic_config.ex │ │ │ │ ├── 20211202204605_update_realtime_build_prepared_statement_sql_function_for_compatibility_with_all_types.ex │ │ │ │ ├── 20241220035512_fix_send_function_partition_creation.ex │ │ │ │ ├── 20211116050929_create_realtime_quote_wal2json_function.ex │ │ │ │ ├── 20220712093339_recreate_realtime_build_prepared_statement_sql_function.ex │ │ │ │ ├── 20240401105812_create_realtime_admin_and_move_ownership.ex │ │ │ │ ├── 20251103001201_broadcast_send_include_payload_id.ex │ │ │ │ ├── 20240523004032_redefine_authorization_tables.ex │ │ │ │ ├── 20211116024918_create_realtime_subscription_table.ex │ │ │ │ ├── 20220908172859_null_passes_filters_recreate_is_visible_through_filters.ex │ │ │ │ └── 20211116212300_create_realtime_build_prepared_statement_sql_function.ex │ │ ├── authorization │ │ │ ├── policies │ │ │ │ ├── presence_policies.ex │ │ │ │ └── broadcast_policies.ex │ │ │ └── policies.ex │ │ ├── connect │ │ │ ├── get_tenant.ex │ │ │ ├── check_connection.ex │ │ │ ├── register_process.ex │ │ │ └── piper.ex │ │ ├── janitor │ │ │ └── maintenance_task.ex │ │ ├── rebalancer.ex │ │ └── cache.ex │ ├── telemetry │ │ ├── telemetry.ex │ │ └── logger.ex │ ├── rate_counter │ │ └── dynamic_supervisor.ex │ ├── repo.ex │ ├── monitoring │ │ ├── prom_ex │ │ │ └── plugins │ │ │ │ ├── channels.ex │ │ │ │ └── tenants.ex │ │ ├── os_metrics.ex │ │ └── erl_sys_mon.ex │ ├── context_cache.ex │ ├── syn │ │ └── postgres_cdc.ex │ ├── adapters │ │ ├── postgres │ │ │ └── protocol │ │ │ │ ├── keep_alive.ex │ │ │ │ └── write.ex │ │ └── changes.ex │ ├── signal_handler.ex │ ├── helpers.ex │ ├── release.ex │ ├── crypto.ex │ ├── api │ │ └── message.ex │ ├── gen_counter │ │ └── gen_counter.ex │ ├── metrics_cleaner.ex │ └── logs.ex └── extensions │ ├── extensions.ex │ └── postgres_cdc_rls │ ├── db_settings.ex │ ├── supervisor.ex │ ├── worker_supervisor.ex │ └── message_dispatcher.ex ├── rel ├── overlays │ ├── bin │ │ ├── server.bat │ │ ├── migrate.bat │ │ ├── server │ │ └── migrate │ └── config.example.yml ├── env.bat.eex ├── vm.args.eex └── env.sh.eex ├── assets ├── css │ └── app.css ├── package.json └── tailwind.config.js ├── priv ├── repo │ ├── migrations │ │ ├── .formatter.exs │ │ ├── 20240902173232_add_extension_external_id_index.exs │ │ ├── 20220527210857_add_external_id_uniq_index.exs │ │ ├── 20240306114423_add_tenant_jwt_jwks.exs │ │ ├── 20251218000543_ensure_jwt_secret_is_text.exs │ │ ├── 20221223010058_drop_tenants_uniq_external_id_index.exs │ │ ├── 20240625211759_remove_enable_authorization_flag.exs │ │ ├── 20231024094642_add_tenant_suspend_flag.exs │ │ ├── 20240418082835_add_authorization_flag.exs │ │ ├── 20250424203323_add_migrations_ran_to_tenant.exs │ │ ├── 20250613072131_add_tenant_broadcast_adapter.exs │ │ ├── 20240704172020_add_notify_private_alpha.exs │ │ ├── 20220815211129_new_max_events_per_second_default.exs │ │ ├── 20221102172703_rename_pg_type.exs │ │ ├── 20241106103258_add_private_only_flag_column_to_tenant.exs │ │ ├── 20230810220924_alter_extensions_table_columns_to_text.exs │ │ ├── 20220815215024_set_current_max_events_per_second.exs │ │ ├── 20250711044927_change_default_broadcast_adapter_to_gen_rpc.exs │ │ ├── 20250926223044_set_default_presence_value.exs │ │ ├── 20250811121559_add_max_presence_events_per_second.exs │ │ ├── 20220410212326_add_tenant_max_eps.exs │ │ ├── 20221018173709_add_cdc_default.exs │ │ ├── 20230810220907_alter_tenants_table_columns_to_text.exs │ │ ├── 20220506102948_rename_poll_interval_to_poll_interval_ms.exs │ │ ├── 20251204170944_nullable_jwt_secrets.exs │ │ ├── 20230110180046_add_limits_fields_to_tenants.exs │ │ ├── 20210706140551_create_tenant.exs │ │ ├── 20220818141501_change_limits_defaults.exs │ │ └── 20220329161857_add_extensions_table.exs │ ├── seeds_before_migration.exs │ └── seeds.exs └── static │ ├── worker.js │ ├── robots.txt │ └── favicon.svg ├── .releaserc ├── .sobelow-conf ├── .formatter.exs ├── bench ├── gen_counter.exs └── secrets.exs ├── .github └── workflows │ ├── docker-build.yml │ ├── integration_tests.yml │ ├── version_updated.yml │ ├── mirror.yml │ ├── prod_linter.yml │ └── tests.yml ├── docker-compose.dbs.yml ├── dev └── postgres │ └── 00-supabase-schema.sql ├── .credo.exs ├── .gitignore ├── coveralls.json ├── deploy └── fly │ ├── qa.toml │ ├── prod.toml │ └── staging.toml ├── .dockerignore ├── docker-compose.yml └── config └── test.exs /test/e2e/.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | -------------------------------------------------------------------------------- /test/e2e/.tool-versions: -------------------------------------------------------------------------------- 1 | deno latest 2 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | elixir 1.18.4-otp-27 2 | nodejs 24 3 | erlang 27 4 | -------------------------------------------------------------------------------- /lib/realtime.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime do 2 | @moduledoc false 3 | end 4 | -------------------------------------------------------------------------------- /rel/overlays/bin/server.bat: -------------------------------------------------------------------------------- 1 | set PHX_SERVER=true 2 | call "%~dp0\realtime" start 3 | -------------------------------------------------------------------------------- /test/e2e/.template.env: -------------------------------------------------------------------------------- 1 | PROJECT_URL= 2 | PROJECT_ANON_TOKEN= 3 | PROJECT_JWT_SECRET= -------------------------------------------------------------------------------- /assets/css/app.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; -------------------------------------------------------------------------------- /rel/overlays/bin/migrate.bat: -------------------------------------------------------------------------------- 1 | call "%~dp0\realtime" eval Realtime.Release.migrate 2 | -------------------------------------------------------------------------------- /assets/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "@supabase/supabase-js": "^2.85.0" 4 | } 5 | } -------------------------------------------------------------------------------- /priv/repo/migrations/.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [:ecto_sql], 3 | inputs: ["*.exs"] 4 | ] 5 | -------------------------------------------------------------------------------- /rel/overlays/bin/server: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cd -P -- "$(dirname -- "$0")" 3 | PHX_SERVER=true exec ./realtime start 4 | -------------------------------------------------------------------------------- /lib/realtime_web/views/layout_view.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.LayoutView do 2 | use RealtimeWeb, :view 3 | end 4 | -------------------------------------------------------------------------------- /rel/overlays/bin/migrate: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cd -P -- "$(dirname -- "$0")" 3 | exec ./realtime eval Realtime.Release.migrate 4 | -------------------------------------------------------------------------------- /test/realtime_web/views/page_view_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PageViewTest do 2 | use RealtimeWeb.ConnCase 3 | end 4 | -------------------------------------------------------------------------------- /test/realtime/helpers_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.HelpersTest do 2 | use Realtime.DataCase 3 | doctest Realtime.Helpers 4 | end 5 | -------------------------------------------------------------------------------- /priv/repo/seeds_before_migration.exs: -------------------------------------------------------------------------------- 1 | import Ecto.Adapters.SQL, only: [query: 3] 2 | 3 | [ 4 | "create schema if not exists realtime" 5 | ] 6 | |> Enum.each(&query(Realtime.Repo, &1, [])) 7 | -------------------------------------------------------------------------------- /priv/static/worker.js: -------------------------------------------------------------------------------- 1 | addEventListener("message", (e) => { 2 | if (e.data.event === "start") { 3 | setInterval(() => postMessage({ event: "keepAlive" }), e.data.interval); 4 | } 5 | }); 6 | -------------------------------------------------------------------------------- /test/realtime/oid_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.OidTest do 2 | use ExUnit.Case, async: true 3 | import Realtime.Adapters.Postgres.OidDatabase 4 | doctest Realtime.Adapters.Postgres.OidDatabase 5 | end 6 | -------------------------------------------------------------------------------- /lib/realtime_web/controllers/ping_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PingController do 2 | use RealtimeWeb, :controller 3 | 4 | def ping(conn, _params) do 5 | json(conn, %{message: "Success"}) 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /.releaserc: -------------------------------------------------------------------------------- 1 | { 2 | "branches": [ 3 | "main" 4 | ], 5 | "plugins": [ 6 | "@semantic-release/commit-analyzer", 7 | "@semantic-release/release-notes-generator", 8 | "@semantic-release/github" 9 | ] 10 | } -------------------------------------------------------------------------------- /priv/static/robots.txt: -------------------------------------------------------------------------------- 1 | # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file 2 | # 3 | # To ban all spiders from the entire site uncomment the next two lines: 4 | # User-agent: * 5 | # Disallow: / 6 | -------------------------------------------------------------------------------- /priv/repo/migrations/20240902173232_add_extension_external_id_index.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddExtensionExternalIdIndex do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create_if_not_exists index("extensions", [:tenant_external_id]) 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /.sobelow-conf: -------------------------------------------------------------------------------- 1 | [ 2 | verbose: true, 3 | private: false, 4 | skip: false, 5 | router: nil, 6 | exit: :low, 7 | format: "txt", 8 | out: nil, 9 | threshold: :medium, 10 | ignore: ["Config.CSP", "Config.HTTPS"], 11 | ignore_files: [], 12 | version: false 13 | ] 14 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220527210857_add_external_id_uniq_index.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddExternalIdUniqIndex do 2 | use Ecto.Migration 3 | 4 | def change do 5 | execute("alter table tenants add constraint uniq_external_id unique (external_id)") 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /priv/repo/migrations/20240306114423_add_tenant_jwt_jwks.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AdddTenantJwtJwksColumn do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add :jwt_jwks, :map, default: nil 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20251218000543_ensure_jwt_secret_is_text.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.EnsureJwtSecretIsText do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | modify :jwt_secret, :text, null: true 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [:ecto, :ecto_sql, :phoenix, :open_api_spex], 3 | subdirectories: ["priv/*/migrations"], 4 | plugins: [Phoenix.LiveView.HTMLFormatter], 5 | inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/*seeds*.exs"], 6 | line_length: 120 7 | ] 8 | -------------------------------------------------------------------------------- /priv/repo/migrations/20221223010058_drop_tenants_uniq_external_id_index.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.DropTenantsUniqExternalIdIndex do 2 | use Ecto.Migration 3 | 4 | def change do 5 | execute("ALTER TABLE IF EXISTS tenants DROP CONSTRAINT IF EXISTS uniq_external_id") 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /priv/repo/migrations/20240625211759_remove_enable_authorization_flag.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.RemoveEnableAuthorizationFlag do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | remove :enable_authorization 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20231024094642_add_tenant_suspend_flag.exs: -------------------------------------------------------------------------------- 1 | defmodule :"Elixir.Realtime.Repo.Migrations.Add-tenant-suspend-flag" do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add :suspend, :boolean, default: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20240418082835_add_authorization_flag.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddAuthorizationFlag do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add :enable_authorization, :boolean, default: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20250424203323_add_migrations_ran_to_tenant.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddMigrationsRanToTenant do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add(:migrations_ran, :integer, default: 0) 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20231204144025_enable_channels_rls.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.EnableChannelsRls do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute("ALTER TABLE realtime.channels ENABLE row level security") 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20250613072131_add_tenant_broadcast_adapter.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddTenantBroadcastAdapter do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add :broadcast_adapter, :string, default: "phoenix" 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20240704172020_add_notify_private_alpha.exs: -------------------------------------------------------------------------------- 1 | defmodule :"Elixir.Realtime.Repo.Migrations.Add-notify-private-alpha" do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add :notify_private_alpha, :boolean, default: false 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240919163305_change_messages_id_type.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.ChangeMessagesIdType do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | alter table(:messages) do 7 | add_if_not_exists :uuid, :uuid 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/realtime_web/views/error_view.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ErrorView do 2 | use RealtimeWeb, :view 3 | 4 | def render("error.json", %{conn: %{assigns: %{message: message}}}), do: %{message: message} 5 | 6 | def template_not_found(template, _assigns), do: Phoenix.Controller.status_message_from_template(template) 7 | end 8 | -------------------------------------------------------------------------------- /test/realtime_web/views/layout_view_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.LayoutViewTest do 2 | use RealtimeWeb.ConnCase 3 | 4 | # When testing helpers, you may want to import Phoenix.HTML and 5 | # use functions such as safe_to_string() to convert the helper 6 | # result into an HTML string. 7 | # import Phoenix.HTML 8 | end 9 | -------------------------------------------------------------------------------- /bench/gen_counter.exs: -------------------------------------------------------------------------------- 1 | alias Realtime.GenCounter 2 | 3 | counter = :counters.new(1, [:write_concurrency]) 4 | _gen_counter = GenCounter.new(:any_term) 5 | 6 | Benchee.run( 7 | %{ 8 | ":counters.add" => fn -> :counters.add(counter, 1, 1) end, 9 | "GenCounter.add" => fn -> GenCounter.add(:any_term) end 10 | } 11 | ) 12 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220815211129_new_max_events_per_second_default.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.NewMaxEventsPerSecondDefault do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table("tenants") do 6 | modify(:max_events_per_second, :integer, null: false, default: 1_000) 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20221102172703_rename_pg_type.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.RenamePgType do 2 | use Ecto.Migration 3 | 4 | def up do 5 | execute("update extensions set type = 'postgres_cdc_rls'") 6 | end 7 | 8 | def down do 9 | execute("update extensions set type = 'postgres'") 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /priv/repo/migrations/20241106103258_add_private_only_flag_column_to_tenant.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddPrivateOnlyFlagColumnToTenant do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add(:private_only, :boolean, default: false, null: false) 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/realtime_web/controllers/page_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PageController do 2 | use RealtimeWeb, :controller 3 | 4 | def index(conn, _params) do 5 | render(conn, "index.html") 6 | end 7 | 8 | def healthcheck(conn, _params) do 9 | conn 10 | |> put_status(:ok) 11 | |> text("ok") 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211122062447_grant_realtime_usage_to_authenticated_role.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.GrantRealtimeUsageToAuthenticatedRole do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute("grant usage on schema realtime to authenticated;") 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /priv/repo/migrations/20230810220924_alter_extensions_table_columns_to_text.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AlterExtensionsTableColumnsToText do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:extensions) do 6 | modify :type, :text 7 | modify :tenant_external_id, :text 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /rel/env.bat.eex: -------------------------------------------------------------------------------- 1 | @echo off 2 | rem Set the release to work across nodes. If using the long name format like 3 | rem the one below (my_app@127.0.0.1), you need to also uncomment the 4 | rem RELEASE_DISTRIBUTION variable below. Must be "sname", "name" or "none". 5 | rem set RELEASE_DISTRIBUTION=name 6 | rem set RELEASE_NODE=<%= @release.name %>@127.0.0.1 7 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250506224012_subscription_index_bridging_disabled.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.SubscriptionIndexBridgingDisabled do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | """ 7 | alter table realtime.subscription reset (index_bridging); 8 | """ 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241019105805_uuid_auto_generation.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.UuidAutoGeneration do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | alter table(:messages) do 7 | modify :uuid, :uuid, null: false, default: fragment("gen_random_uuid()") 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220815215024_set_current_max_events_per_second.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.SetCurrentMaxEventsPerSecond do 2 | use Ecto.Migration 3 | 4 | def change do 5 | execute( 6 | "update tenants set max_events_per_second = 1000", 7 | "update tenants set max_events_per_second = 10000" 8 | ) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250523164012_run_subscription_index_bridging_disabled.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RunSubscriptionIndexBridgingDisabled do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | execute(""" 7 | alter table realtime.subscription reset (index_bridging); 8 | """) 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240109165339_add_update_grant_to_channels.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.AddUpdateGrantToChannels do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | GRANT UPDATE ON realtime.channels TO postgres, anon, authenticated, service_role 9 | """) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240805133720_logged_messages_table.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.LoggedMessagesTable do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | execute """ 7 | -- Commented to have oriole compatability 8 | -- ALTER TABLE realtime.messages SET LOGGED; 9 | """ 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240108234812_add_channels_column_for_write_check.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.AddChannelsColumnForWriteCheck do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | alter table(:channels, prefix: "realtime") do 8 | add :check, :boolean, default: false 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240801235015_unlogged_messages_table.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.UnloggedMessagesTable do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | execute """ 7 | -- Commented to have oriole compatability 8 | -- ALTER TABLE realtime.messages SET UNLOGGED; 9 | """ 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /priv/repo/migrations/20250711044927_change_default_broadcast_adapter_to_gen_rpc.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.ChangeDefaultBroadcastAdapterToGenRpc do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table("tenants") do 6 | modify :broadcast_adapter, :string, default: "gen_rpc", from: {:string, default: "phoenix"} 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/realtime/telemetry/telemetry.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Telemetry do 2 | @moduledoc """ 3 | Telemetry wrapper 4 | """ 5 | 6 | @doc """ 7 | Dispatches Telemetry events. 8 | """ 9 | 10 | @spec execute([atom, ...], map, map) :: :ok 11 | def execute(event, measurements, metadata \\ %{}) do 12 | :telemetry.execute(event, measurements, metadata) 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /priv/repo/migrations/20250926223044_set_default_presence_value.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.SetDefaultPresenceValue do 2 | use Ecto.Migration 3 | @disable_ddl_transaction true 4 | @disable_migration_lock true 5 | def change do 6 | alter table(:tenants) do 7 | modify :max_presence_events_per_second, :integer, default: 1000 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250107150512_realtime_subscription_unlogged.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RealtimeSubscriptionUnlogged do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | execute(""" 7 | -- Commented to have oriole compatability 8 | -- ALTER TABLE realtime.subscription SET UNLOGGED; 9 | """) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /priv/repo/migrations/20250811121559_add_max_presence_events_per_second.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddMaxPresenceEventsPerSecond do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | add :max_presence_events_per_second, :integer, default: 10000 7 | add :max_payload_size_in_kb, :integer, default: 3000 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /rel/overlays/config.example.yml: -------------------------------------------------------------------------------- 1 | endpoint_port: 4000 2 | db_repo: 3 | - hostname: "127.0.0.1" 4 | username: "postgres" 5 | password: "postgres" 6 | database: "postgres" 7 | pool_size: 3 8 | port: 5432 9 | cluster: 10 | - cookie: "cookie_config" 11 | service: "realtime-dns" 12 | application_name: "realtime" 13 | polling_interval: 5000 14 | # debug: false 15 | -------------------------------------------------------------------------------- /test/realtime_web/live/inspector_live/index_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.InspectorLive.IndexTest do 2 | use RealtimeWeb.ConnCase 3 | import Phoenix.LiveViewTest 4 | 5 | describe "Inspector LiveView" do 6 | test "renders inspector page", %{conn: conn} do 7 | {:ok, _view, html} = live(conn, ~p"/inspector") 8 | 9 | assert html =~ "Realtime Inspector" 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /test/realtime_web/live/page_live/index_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PageLive.IndexTest do 2 | use RealtimeWeb.ConnCase 3 | import Phoenix.LiveViewTest 4 | 5 | describe "Index LiveView" do 6 | test "renders page successfully", %{conn: conn} do 7 | {:ok, _view, html} = live(conn, "/") 8 | 9 | assert html =~ "Supabase Realtime: Multiplayer Edition" 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220410212326_add_tenant_max_eps.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddTenantMaxEps do 2 | use Ecto.Migration 3 | 4 | def up do 5 | alter table("tenants") do 6 | add(:max_events_per_second, :integer, default: 10_000) 7 | end 8 | end 9 | 10 | def down do 11 | alter table("tenants") do 12 | remove(:max_events_per_second) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /priv/repo/migrations/20221018173709_add_cdc_default.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddCdcDefault do 2 | use Ecto.Migration 3 | 4 | def up do 5 | alter table("tenants") do 6 | add(:postgres_cdc_default, :string, default: "postgres_cdc_rls") 7 | end 8 | end 9 | 10 | def down do 11 | alter table("tenants") do 12 | remove(:postgres_cdc_default) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20231018144023_create_channels.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateChannels do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | create table(:channels, prefix: "realtime") do 8 | add(:name, :string, null: false) 9 | timestamps() 10 | end 11 | 12 | create unique_index(:channels, [:name], prefix: "realtime") 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /priv/repo/migrations/20230810220907_alter_tenants_table_columns_to_text.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AlterTenantsTableColumnsToText do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | modify :name, :text 7 | modify :external_id, :text 8 | modify :jwt_secret, :text 9 | modify :postgres_cdc_default, :text, default: "postgres_cdc_rls" 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250905041441_create_messages_replay_index.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateMessagesReplayIndex do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | create_if_not_exists index(:messages, [{:desc, :inserted_at}, :topic], 8 | where: "extension = 'broadcast' and private IS TRUE" 9 | ) 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v4 15 | 16 | - name: Set up Docker Buildx 17 | uses: docker/setup-buildx-action@v3 18 | 19 | - name: Build Docker image 20 | run: docker build . 21 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220506102948_rename_poll_interval_to_poll_interval_ms.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.RenamePollIntervalToPollIntervalMs do 2 | use Ecto.Migration 3 | import Realtime.Api, only: [rename_settings_field: 2] 4 | 5 | def up do 6 | rename_settings_field("poll_interval", "poll_interval_ms") 7 | end 8 | 9 | def down do 10 | rename_settings_field("poll_interval_ms", "poll_interval") 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /priv/repo/migrations/20251204170944_nullable_jwt_secrets.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.NullableJwtSecrets do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table(:tenants) do 6 | modify :jwt_secret, :text, null: true 7 | end 8 | 9 | create constraint(:tenants, :jwt_secret_or_jwt_jwks_required, 10 | check: "jwt_secret IS NOT NULL OR jwt_jwks IS NOT NULL" 11 | ) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240418121054_remove_check_columns.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RemoveCheckColumns do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | alter table(:channels) do 8 | remove :check 9 | end 10 | 11 | alter table(:broadcasts) do 12 | remove :check 13 | end 14 | 15 | alter table(:presences) do 16 | remove :check 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20231204144024_create_rls_helper_functions.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRlsHelperFunctions do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | create or replace function realtime.channel_name() returns text as $$ 9 | select nullif(current_setting('realtime.channel_name', true), '')::text; 10 | $$ language sql stable; 11 | """) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/presence.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Presence do 2 | @moduledoc """ 3 | Provides presence tracking to channels and processes. 4 | 5 | See the [`Phoenix.Presence`](http://hexdocs.pm/phoenix/Phoenix.Presence.html) 6 | docs for more details. 7 | """ 8 | use Phoenix.Presence, 9 | otp_app: :realtime, 10 | pubsub_server: Realtime.PubSub, 11 | dispatcher: RealtimeWeb.RealtimeChannel.MessageDispatcher, 12 | pool_size: 10 13 | end 14 | -------------------------------------------------------------------------------- /priv/repo/migrations/20230110180046_add_limits_fields_to_tenants.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddLimitsFieldsToTenants do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table("tenants") do 6 | add(:max_bytes_per_second, :integer, default: 100_000, null: false) 7 | add(:max_channels_per_client, :integer, default: 100, null: false) 8 | add(:max_joins_per_second, :integer, default: 500, null: false) 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/realtime_web/controllers/page_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PageControllerTest do 2 | use RealtimeWeb.ConnCase 3 | 4 | test "GET / renders index page", %{conn: conn} do 5 | conn = get(conn, "/") 6 | assert html_response(conn, 200) =~ " Supabase Realtime: Multiplayer Edition" 7 | end 8 | 9 | test "GET /healthcheck returns ok status", %{conn: conn} do 10 | conn = get(conn, "/healthcheck") 11 | assert text_response(conn, 200) == "ok" 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250110162412_realtime_subscription_logged.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RealtimeSubscriptionLogged do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # PG Updates doesn't allow us to use UNLOGGED tables due to the fact that Sequences on PG14 still need to be logged 6 | def change do 7 | execute(""" 8 | -- Commented to have oriole compatability 9 | -- ALTER TABLE realtime.subscription SET LOGGED; 10 | """) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /test/realtime/adapters/postgres/protocol_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Adapters.Postgres.ProtocolTest do 2 | use ExUnit.Case, async: true 3 | alias Realtime.Adapters.Postgres.Protocol 4 | 5 | test "defguard is_write/1" do 6 | require Protocol 7 | assert Protocol.is_write("w") 8 | refute Protocol.is_write("k") 9 | end 10 | 11 | test "defguard is_keep_alive/1" do 12 | require Protocol 13 | assert Protocol.is_keep_alive("k") 14 | refute Protocol.is_keep_alive("w") 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /test/realtime_web/views/error_view_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ErrorViewTest do 2 | use RealtimeWeb.ConnCase, async: true 3 | 4 | # Bring render/3 and render_to_string/3 for testing custom views 5 | import Phoenix.View 6 | 7 | test "renders 404.html" do 8 | assert render_to_string(RealtimeWeb.ErrorView, "404.html", []) == "Not Found" 9 | end 10 | 11 | test "renders 500.html" do 12 | assert render_to_string(RealtimeWeb.ErrorView, "500.html", []) == "Internal Server Error" 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/realtime/rate_counter/dynamic_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.RateCounter.DynamicSupervisor do 2 | @moduledoc """ 3 | Dynamic Supervisor to spin up `RateCounter`s as needed. 4 | """ 5 | 6 | use DynamicSupervisor 7 | 8 | @spec start_link(list()) :: {:error, any} | {:ok, pid} 9 | def start_link(args) do 10 | DynamicSupervisor.start_link(__MODULE__, args, name: __MODULE__) 11 | end 12 | 13 | @impl true 14 | def init(_args) do 15 | DynamicSupervisor.init(strategy: :one_for_one) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/realtime/tenants/authorization/policies/presence_policies.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Authorization.Policies.PresencePolicies do 2 | @moduledoc """ 3 | PresencePolicies structure that holds the required authorization information for a given connection within the scope of a tracking / receiving presence messages 4 | """ 5 | require Logger 6 | 7 | defstruct read: nil, write: nil 8 | 9 | @type t :: %__MODULE__{ 10 | read: boolean() | nil, 11 | write: boolean() | nil 12 | } 13 | end 14 | -------------------------------------------------------------------------------- /lib/realtime_web/live/page_live/index.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PageLive.Index do 2 | use RealtimeWeb, :live_view 3 | 4 | @impl true 5 | def mount(_params, _session, socket) do 6 | {:ok, socket} 7 | end 8 | 9 | @impl true 10 | def handle_params(params, _url, socket) do 11 | {:noreply, apply_action(socket, socket.assigns.live_action, params)} 12 | end 13 | 14 | defp apply_action(socket, :index, _params) do 15 | socket 16 | |> assign(:page_title, "Home - Supabase Realtime") 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/realtime/tenants/authorization/policies/broadcast_policies.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Authorization.Policies.BroadcastPolicies do 2 | @moduledoc """ 3 | BroadcastPolicies structure that holds the required authorization information for a given connection within the scope of a sending / receiving broadcasts messages 4 | """ 5 | require Logger 6 | 7 | defstruct read: nil, write: nil 8 | 9 | @type t :: %__MODULE__{ 10 | read: boolean() | nil, 11 | write: boolean() | nil 12 | } 13 | end 14 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241108114728_messages_using_uuid.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.MessagesUsingUuid do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | alter table(:messages) do 7 | remove(:id) 8 | remove(:uuid) 9 | add(:id, :uuid, null: false, default: fragment("gen_random_uuid()")) 10 | end 11 | 12 | execute("ALTER TABLE realtime.messages ADD PRIMARY KEY (id, inserted_at)") 13 | execute("DROP SEQUENCE realtime.messages_id_seq") 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/realtime/repo.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo do 2 | use Ecto.Repo, 3 | otp_app: :realtime, 4 | adapter: Ecto.Adapters.Postgres 5 | 6 | def with_dynamic_repo(config, callback) do 7 | default_dynamic_repo = get_dynamic_repo() 8 | {:ok, repo} = [name: nil, pool_size: 2] |> Keyword.merge(config) |> Realtime.Repo.start_link() 9 | 10 | try do 11 | put_dynamic_repo(repo) 12 | callback.(repo) 13 | after 14 | put_dynamic_repo(default_dynamic_repo) 15 | Supervisor.stop(repo) 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /priv/repo/migrations/20210706140551_create_tenant.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.CreateTenants do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:tenants, primary_key: false) do 6 | add(:id, :binary_id, primary_key: true) 7 | add(:name, :string) 8 | add(:external_id, :string) 9 | add(:jwt_secret, :string, size: 500) 10 | add(:max_concurrent_users, :integer, default: 10_000) 11 | timestamps() 12 | end 13 | 14 | create(index(:tenants, [:external_id], unique: true)) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/realtime_web/views/changeset_view.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ChangesetView do 2 | use RealtimeWeb, :view 3 | 4 | @doc """ 5 | Traverses and translates changeset errors. 6 | 7 | See `Ecto.Changeset.traverse_errors/2` and 8 | `RealtimeWeb.ErrorHelpers.translate_error/1` for more details. 9 | """ 10 | def translate_errors(changeset) do 11 | Ecto.Changeset.traverse_errors(changeset, &translate_error/1) 12 | end 13 | 14 | def render("error.json", %{changeset: changeset}) do 15 | %{errors: translate_errors(changeset)} 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/extensions/extensions.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Extensions do 2 | @moduledoc """ 3 | This module provides functions to get extension settings. 4 | """ 5 | def db_settings(type) do 6 | db_settings = 7 | Application.get_env(:realtime, :extensions) 8 | |> Enum.reduce(nil, fn 9 | {_, %{key: ^type, db_settings: db_settings}}, _ -> db_settings 10 | _, acc -> acc 11 | end) 12 | 13 | %{ 14 | default: apply(db_settings, :default, []), 15 | required: apply(db_settings, :required, []) 16 | } 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/payloads/presence.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Channels.Payloads.Presence do 2 | @moduledoc """ 3 | Validate presence field of the join payload. 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | alias RealtimeWeb.Channels.Payloads.Join 8 | 9 | embedded_schema do 10 | field :enabled, :boolean, default: true 11 | field :key, :any, default: UUID.uuid1(), virtual: true 12 | end 13 | 14 | def changeset(presence, attrs) do 15 | cast(presence, attrs, [:enabled, :key], message: &Join.error_message/2) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /test/realtime_web/controllers/openapi_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Controllers.OpenapiControllerTest do 2 | use RealtimeWeb.ConnCase 3 | 4 | describe "openapi" do 5 | test "returns the openapi spec", %{conn: conn} do 6 | conn = get(conn, ~p"/api/openapi") 7 | assert json_response(conn, 200) 8 | end 9 | end 10 | 11 | describe "swaggerui" do 12 | test "returns the swaggerui", %{conn: conn} do 13 | conn = get(conn, ~p"/swaggerui") 14 | assert html_response(conn, 200) =~ "Swagger UI" 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/payloads/broadcast/replay.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Channels.Payloads.Broadcast.Replay do 2 | @moduledoc """ 3 | Validate broadcast replay field of the join payload. 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | alias RealtimeWeb.Channels.Payloads.Join 8 | 9 | embedded_schema do 10 | field :limit, :integer, default: 10 11 | field :since, :integer, default: 0 12 | end 13 | 14 | def changeset(broadcast, attrs) do 15 | cast(broadcast, attrs, [:limit, :since], message: &Join.error_message/2) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220818141501_change_limits_defaults.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.ChangeLimitsDefaults do 2 | use Ecto.Migration 3 | 4 | def change do 5 | alter table("tenants") do 6 | modify(:max_events_per_second, :integer, 7 | null: false, 8 | default: 100, 9 | from: {:integer, null: false, default: 100} 10 | ) 11 | 12 | modify(:max_concurrent_users, :integer, 13 | null: false, 14 | default: 200, 15 | from: {:integer, null: false, default: 200} 16 | ) 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211116213355_create_realtime_cast_function.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeCastFunction do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute("create function realtime.cast(val text, type_ regtype) 8 | returns jsonb 9 | immutable 10 | language plpgsql 11 | as $$ 12 | declare 13 | res jsonb; 14 | begin 15 | execute format('select to_jsonb(%L::'|| type_::text || ')', val) into res; 16 | return res; 17 | end 18 | $$;") 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/realtime/tenants/connect/get_tenant.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Connect.GetTenant do 2 | @moduledoc """ 3 | Get tenant database connection. 4 | """ 5 | 6 | alias Realtime.Api.Tenant 7 | alias Realtime.Tenants 8 | @behaviour Realtime.Tenants.Connect.Piper 9 | 10 | @impl Realtime.Tenants.Connect.Piper 11 | def run(acc) do 12 | %{tenant_id: tenant_id} = acc 13 | 14 | case Tenants.Cache.get_tenant_by_external_id(tenant_id) do 15 | %Tenant{} = tenant -> {:ok, Map.put(acc, :tenant, tenant)} 16 | _ -> {:error, :tenant_not_found} 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime/monitoring/prom_ex/plugins/channels.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.PromEx.Plugins.Channels do 2 | @moduledoc """ 3 | Realtime channels monitoring plugin for PromEx 4 | """ 5 | use PromEx.Plugin 6 | require Logger 7 | 8 | @impl true 9 | def event_metrics(_opts) do 10 | Event.build(:realtime, [ 11 | counter( 12 | [:realtime, :channel, :error], 13 | event_name: [:realtime, :channel, :error], 14 | measurement: :code, 15 | tags: [:code], 16 | description: "Count of errors in the Realtime channels initialization" 17 | ) 18 | ]) 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250123174212_remove_unused_publications.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RemoveUnusedPublications do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | execute(""" 7 | DO $$ 8 | DECLARE 9 | r RECORD; 10 | BEGIN 11 | FOR r IN 12 | SELECT pubname FROM pg_publication WHERE pubname LIKE 'realtime_messages%' or pubname LIKE 'supabase_realtime_messages%' 13 | LOOP 14 | EXECUTE 'DROP PUBLICATION IF EXISTS ' || quote_ident(r.pubname) || ';' ; 15 | END LOOP; 16 | END $$; 17 | """) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime/tenants/connect/check_connection.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Connect.CheckConnection do 2 | @moduledoc """ 3 | Check tenant database connection. 4 | """ 5 | 6 | @behaviour Realtime.Tenants.Connect.Piper 7 | @impl true 8 | def run(acc) do 9 | %{tenant: tenant} = acc 10 | 11 | case Realtime.Database.check_tenant_connection(tenant) do 12 | {:ok, conn} -> 13 | db_conn_reference = Process.monitor(conn) 14 | {:ok, %{acc | db_conn_pid: conn, db_conn_reference: db_conn_reference}} 15 | 16 | {:error, error} -> 17 | {:error, error} 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/payloads/postgres_change.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Channels.Payloads.PostgresChange do 2 | @moduledoc """ 3 | Validate postgres_changes field of the join payload. 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | alias RealtimeWeb.Channels.Payloads.Join 8 | 9 | embedded_schema do 10 | field :event, :string 11 | field :schema, :string 12 | field :table, :string 13 | field :filter, :string 14 | end 15 | 16 | def changeset(postgres_change, attrs) do 17 | cast(postgres_change, attrs, [:event, :schema, :table, :filter], message: &Join.error_message/2) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /priv/repo/migrations/20220329161857_add_extensions_table.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Repo.Migrations.AddExtensionsTable do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:extensions, primary_key: false) do 6 | add(:id, :binary_id, primary_key: true) 7 | add(:type, :string) 8 | add(:settings, :map) 9 | 10 | add( 11 | :tenant_external_id, 12 | references(:tenants, on_delete: :delete_all, type: :string, column: :external_id) 13 | ) 14 | 15 | timestamps() 16 | end 17 | 18 | create(index(:extensions, [:tenant_external_id, :type], unique: true)) 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/realtime/context_cache.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.ContextCache do 2 | @moduledoc """ 3 | Read through cache for hot database paths. 4 | """ 5 | 6 | require Logger 7 | 8 | def apply_fun(context, {fun, arity}, args) do 9 | cache = cache_name(context) 10 | cache_key = {{fun, arity}, args} 11 | 12 | case Cachex.fetch(cache, cache_key, fn {{_fun, _arity}, args} -> {:commit, {:cached, apply(context, fun, args)}} end) do 13 | {:commit, {:cached, value}} -> value 14 | {:ok, {:cached, value}} -> value 15 | end 16 | end 17 | 18 | defp cache_name(context) do 19 | Module.concat(context, Cache) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240311171622_add_insert_and_delete_grant_to_channels.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.AddInsertAndDeleteGrantToChannels do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | GRANT INSERT, DELETE ON realtime.channels TO postgres, anon, authenticated, service_role 9 | """) 10 | 11 | execute(""" 12 | GRANT INSERT ON realtime.broadcasts TO postgres, anon, authenticated, service_role 13 | """) 14 | 15 | execute(""" 16 | GRANT USAGE ON SEQUENCE realtime.broadcasts_id_seq TO postgres, anon, authenticated, service_role 17 | """) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime_web/live/time_live.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.TimeLive do 2 | use RealtimeWeb, :live_view 3 | 4 | def mount(_params, _session, socket) do 5 | {:ok, assign_time(socket)} 6 | end 7 | 8 | def render(assigns) do 9 | ~H""" 10 | <%= @server_time %> 11 | """ 12 | end 13 | 14 | def handle_info(:time, socket) do 15 | {:noreply, assign_time(socket)} 16 | end 17 | 18 | defp assign_time(socket) do 19 | timer = if Mix.env() == :dev, do: 60_000, else: 100 20 | Process.send_after(self(), :time, timer) 21 | now = DateTime.utc_now() |> DateTime.to_string() 22 | 23 | socket 24 | |> assign(:server_time, now) 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241130184212_recreate_entity_index_using_btree.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RecreateEntityIndexUsingBtree do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | def change do 6 | execute("drop index if exists \"realtime\".\"ix_realtime_subscription_entity\"") 7 | 8 | execute(""" 9 | do $$ 10 | begin 11 | create index concurrently if not exists ix_realtime_subscription_entity on realtime.subscription using btree (entity); 12 | exception 13 | when others then 14 | create index if not exists ix_realtime_subscription_entity on realtime.subscription using btree (entity); 15 | end$$; 16 | """) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/payloads/broadcast.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Channels.Payloads.Broadcast do 2 | @moduledoc """ 3 | Validate broadcast field of the join payload. 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | alias RealtimeWeb.Channels.Payloads.Join 8 | 9 | embedded_schema do 10 | field :ack, :boolean, default: false 11 | field :self, :boolean, default: false 12 | embeds_one :replay, RealtimeWeb.Channels.Payloads.Broadcast.Replay 13 | end 14 | 15 | def changeset(broadcast, attrs) do 16 | cast(broadcast, attrs, [:ack, :self], message: &Join.error_message/2) 17 | |> cast_embed(:replay, invalid_message: "unable to parse, expected a map") 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime_web/live/status_live/index.html.heex: -------------------------------------------------------------------------------- 1 | <.h1>Supabase Realtime: Multiplayer Edition 2 | 3 | <.h2>Cluster Status 4 | 5 |

Understand the latency between nodes across the Realtime cluster.

6 | 7 |
8 |
9 |
10 |
From: <%= p.payload.from_region %> - <%= p.payload.from_node %>
11 |
To: <%= p.payload.region %> - <%= p.payload.node %>
12 |
<%= p.payload.latency %> ms
13 |
<%= p.payload.timestamp %>
14 |
15 |
16 |
17 | -------------------------------------------------------------------------------- /rel/vm.args.eex: -------------------------------------------------------------------------------- 1 | ## Customize flags given to the VM: http://erlang.org/doc/man/erl.html 2 | ## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here 3 | 4 | ## Number of dirty schedulers doing IO work (file, sockets, and others) 5 | ##+SDio 5 6 | 7 | ## Increase number of concurrent ports/sockets 8 | ##+Q 65536 9 | 10 | ## Tweak GC to run more often 11 | ##-env ERL_FULLSWEEP_AFTER 10 12 | 13 | ## Limit process heap for all procs to 2500 MB. The number here is the number of words 14 | +hmax <%= div(2_500_000_000, :erlang.system_info(:wordsize)) %> 15 | 16 | ## Set distribution buffer busy limit (default is 1024) 17 | +zdbbl 100000 18 | 19 | ## Disable Busy Wait 20 | +sbwt none 21 | +sbwtdio none 22 | +sbwtdcpu none 23 | -------------------------------------------------------------------------------- /test/support/replication_test_handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Replication.TestHandler do 2 | @behaviour PostgresReplication.Handler 3 | import PostgresReplication.Protocol 4 | alias PostgresReplication.Protocol.KeepAlive 5 | 6 | @impl true 7 | def call(message, _metadata) when is_write(message) do 8 | :noreply 9 | end 10 | 11 | def call(message, _metadata) when is_keep_alive(message) do 12 | reply = 13 | case parse(message) do 14 | %KeepAlive{reply: :now, wal_end: wal_end} -> 15 | wal_end = wal_end + 1 16 | standby(wal_end, wal_end, wal_end, :now) 17 | 18 | _ -> 19 | hold() 20 | end 21 | 22 | {:reply, reply} 23 | end 24 | 25 | def call(_, _), do: :noreply 26 | end 27 | -------------------------------------------------------------------------------- /docker-compose.dbs.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | db: 5 | image: supabase/postgres:14.1.0.105 6 | container_name: realtime-db 7 | ports: 8 | - "5432:5432" 9 | volumes: 10 | - ./dev/postgres:/docker-entrypoint-initdb.d/ 11 | command: postgres -c config_file=/etc/postgresql/postgresql.conf 12 | environment: 13 | POSTGRES_HOST: /var/run/postgresql 14 | POSTGRES_PASSWORD: postgres 15 | tenant_db: 16 | image: supabase/postgres:14.1.0.105 17 | container_name: tenant-db 18 | ports: 19 | - "5433:5432" 20 | command: postgres -c config_file=/etc/postgresql/postgresql.conf 21 | environment: 22 | POSTGRES_HOST: /var/run/postgresql 23 | POSTGRES_PASSWORD: postgres 24 | -------------------------------------------------------------------------------- /lib/realtime/tenants/janitor/maintenance_task.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Janitor.MaintenanceTask do 2 | @moduledoc """ 3 | Perform maintenance on the messages table. 4 | * Delete old messages 5 | * Create new partitions 6 | """ 7 | 8 | @spec run(String.t()) :: :ok | {:error, any} 9 | def run(tenant_external_id) do 10 | with %Realtime.Api.Tenant{} = tenant <- Realtime.Tenants.Cache.get_tenant_by_external_id(tenant_external_id), 11 | {:ok, conn} <- Realtime.Database.connect(tenant, "realtime_janitor"), 12 | :ok <- Realtime.Messages.delete_old_messages(conn), 13 | :ok <- Realtime.Tenants.Migrations.create_partitions(conn) do 14 | GenServer.stop(conn) 15 | :ok 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /dev/postgres/00-supabase-schema.sql: -------------------------------------------------------------------------------- 1 | create role anon nologin noinherit; 2 | create role authenticated nologin noinherit; 3 | create role service_role nologin noinherit bypassrls; 4 | 5 | grant usage on schema public to anon, authenticated, service_role; 6 | 7 | alter default privileges in schema public grant all on tables to anon, authenticated, service_role; 8 | alter default privileges in schema public grant all on functions to anon, authenticated, service_role; 9 | alter default privileges in schema public grant all on sequences to anon, authenticated, service_role; 10 | 11 | create schema if not exists _realtime; 12 | create schema if not exists realtime; 13 | 14 | create publication supabase_realtime with (publish = 'insert, update, delete'); 15 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20231204144023_set_required_grants.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.SetRequiredGrants do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | GRANT USAGE ON SCHEMA realtime TO postgres, anon, authenticated, service_role 9 | """) 10 | 11 | execute(""" 12 | GRANT SELECT ON ALL TABLES IN SCHEMA realtime TO postgres, anon, authenticated, service_role 13 | """) 14 | 15 | execute(""" 16 | GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA realtime TO postgres, anon, authenticated, service_role 17 | """) 18 | 19 | execute(""" 20 | GRANT USAGE ON ALL SEQUENCES IN SCHEMA realtime TO postgres, anon, authenticated, service_role 21 | """) 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/support/tracing.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tracing do 2 | defmacro __using__(_opts) do 3 | quote do 4 | # Use Record module to extract fields of the Span record from the opentelemetry dependency. 5 | require Record 6 | @span_fields Record.extract(:span, from: "deps/opentelemetry/include/otel_span.hrl") 7 | @status_fields Record.extract(:status, from: "deps/opentelemetry_api/include/opentelemetry.hrl") 8 | @attributes_fields Record.extract(:attributes, from: "deps/opentelemetry_api/src/otel_attributes.erl") 9 | # Define macros for otel stuff 10 | Record.defrecordp(:span, @span_fields) 11 | Record.defrecordp(:status, @status_fields) 12 | Record.defrecordp(:attributes, @attributes_fields) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/extensions/postgres_cdc_rls/db_settings.ex: -------------------------------------------------------------------------------- 1 | defmodule Extensions.PostgresCdcRls.DbSettings do 2 | @moduledoc """ 3 | Schema callbacks for CDC RLS implementation. 4 | """ 5 | 6 | def default do 7 | %{ 8 | "poll_interval_ms" => 100, 9 | "poll_max_changes" => 100, 10 | "poll_max_record_bytes" => 1_048_576, 11 | "publication" => "supabase_realtime", 12 | "slot_name" => "supabase_realtime_replication_slot" 13 | } 14 | end 15 | 16 | def required do 17 | [ 18 | {"region", &is_binary/1, false}, 19 | {"db_host", &is_binary/1, true}, 20 | {"db_name", &is_binary/1, true}, 21 | {"db_user", &is_binary/1, true}, 22 | {"db_port", &is_binary/1, true}, 23 | {"db_password", &is_binary/1, true} 24 | ] 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/realtime_web/templates/layout/root.html.heex: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | <%= csrf_meta_tag() %> 9 | <.live_title> 10 | <%= assigns[:page_title] || "Supabase Realtime" %> 11 | 12 | 13 | 15 | 16 | 17 | <%= @inner_content %> 18 | 19 | 20 | -------------------------------------------------------------------------------- /lib/realtime/monitoring/os_metrics.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.OsMetrics do 2 | @moduledoc """ 3 | This module provides functions to get CPU and RAM usage. 4 | """ 5 | 6 | @spec ram_usage() :: float() 7 | def ram_usage do 8 | mem = :memsup.get_system_memory_data() 9 | free_mem = if Mix.env() in [:dev, :test], do: mem[:free_memory], else: mem[:available_memory] 10 | 100 - free_mem / mem[:total_memory] * 100 11 | end 12 | 13 | @spec cpu_la() :: %{avg1: float(), avg5: float(), avg15: float()} 14 | def cpu_la do 15 | %{ 16 | avg1: :cpu_sup.avg1() / 256, 17 | avg5: :cpu_sup.avg5() / 256, 18 | avg15: :cpu_sup.avg15() / 256 19 | } 20 | end 21 | 22 | @spec cpu_util() :: float() | {:error, term()} 23 | def cpu_util do 24 | :cpu_sup.util() 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/realtime/tenants/connect/register_process.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Connect.RegisterProcess do 2 | @moduledoc """ 3 | Registers the database process in :syn 4 | """ 5 | alias Realtime.Tenants.Connect 6 | @behaviour Realtime.Tenants.Connect.Piper 7 | 8 | @impl true 9 | def run(acc) do 10 | %{tenant_id: tenant_id, db_conn_pid: conn} = acc 11 | 12 | with {:ok, _} <- :syn.update_registry(Connect, tenant_id, fn _pid, meta -> %{meta | conn: conn} end), 13 | {:ok, _} <- Registry.register(Connect.Registry, tenant_id, %{}) do 14 | {:ok, acc} 15 | else 16 | {:error, :undefined} -> {:error, :process_not_found} 17 | {:error, {:already_registered, _}} -> {:error, :already_registered} 18 | {:error, reason} -> {:error, reason} 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240227174441_add_broadcast_permissions_table.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.AddBroadcastsPoliciesTable do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | create table(:broadcasts) do 8 | add :channel_id, references(:channels, on_delete: :delete_all), null: false 9 | add :check, :boolean, default: false, null: false 10 | timestamps() 11 | end 12 | 13 | create unique_index(:broadcasts, :channel_id) 14 | 15 | execute("ALTER TABLE realtime.broadcasts ENABLE row level security") 16 | execute("GRANT SELECT ON realtime.broadcasts TO postgres, anon, authenticated, service_role") 17 | execute("GRANT UPDATE ON realtime.broadcasts TO postgres, anon, authenticated, service_role") 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/realtime/syn/postgres_cdc.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Syn.PostgresCdc do 2 | @moduledoc """ 3 | Scope for the PostgresCdc module. 4 | """ 5 | 6 | @doc """ 7 | Returns the scope for a given tenant id. 8 | """ 9 | @spec scope(String.t()) :: atom() 10 | def scope(tenant_id) do 11 | shards = Application.fetch_env!(:realtime, :postgres_cdc_scope_shards) 12 | shard = :erlang.phash2(tenant_id, shards) 13 | :"realtime_postgres_cdc_#{shard}" 14 | end 15 | 16 | def scopes() do 17 | shards = Application.fetch_env!(:realtime, :postgres_cdc_scope_shards) 18 | Enum.map(0..(shards - 1), fn shard -> :"realtime_postgres_cdc_#{shard}" end) 19 | end 20 | 21 | def syn_topic_prefix(), do: "realtime_postgres_cdc_" 22 | def syn_topic(tenant_id), do: "#{syn_topic_prefix()}#{tenant_id}" 23 | end 24 | -------------------------------------------------------------------------------- /lib/realtime_web/gettext.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Gettext do 2 | @moduledoc """ 3 | A module providing Internationalization with a gettext-based API. 4 | 5 | By using [Gettext](https://hexdocs.pm/gettext), 6 | your module gains a set of macros for translations, for example: 7 | 8 | import RealtimeWeb.Gettext 9 | 10 | # Simple translation 11 | gettext("Here is the string to translate") 12 | 13 | # Plural translation 14 | ngettext("Here is the string to translate", 15 | "Here are the strings to translate", 16 | 3) 17 | 18 | # Domain-based translation 19 | dgettext("errors", "Here is the error message to translate") 20 | 21 | See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage. 22 | """ 23 | use Gettext.Backend, otp_app: :realtime 24 | end 25 | -------------------------------------------------------------------------------- /test/api_jwt_secret_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ApiJwtSecretTest do 2 | use RealtimeWeb.ConnCase, async: false 3 | 4 | test "no api key", %{conn: conn} do 5 | previous = Application.get_env(:realtime, :api_jwt_secret) 6 | Application.put_env(:realtime, :api_jwt_secret, nil) 7 | on_exit(fn -> Application.put_env(:realtime, :api_jwt_secret, previous) end) 8 | 9 | conn = get(conn, Routes.tenant_path(conn, :index)) 10 | assert conn.status == 403 11 | end 12 | 13 | test "api key is right", %{conn: conn} do 14 | api_jwt_secret = Application.get_env(:realtime, :api_jwt_secret) 15 | jwt = generate_jwt_token(api_jwt_secret) 16 | conn = Plug.Conn.put_req_header(conn, "authorization", "Bearer " <> jwt) 17 | conn = get(conn, Routes.tenant_path(conn, :index)) 18 | assert conn.status == 200 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /.credo.exs: -------------------------------------------------------------------------------- 1 | %{ 2 | configs: [ 3 | %{ 4 | name: "default", 5 | files: %{ 6 | included: ["lib/", "src/", "web/", "apps/"], 7 | excluded: [] 8 | }, 9 | plugins: [], 10 | requires: [], 11 | strict: false, 12 | parse_timeout: 5000, 13 | color: true, 14 | checks: %{ 15 | disabled: [ 16 | {Credo.Check.Design.TagTODO, []}, 17 | {Credo.Check.Consistency.ExceptionNames, []}, 18 | {Credo.Check.Refactor.Nesting, []}, 19 | {Credo.Check.Refactor.CyclomaticComplexity, []}, 20 | {Credo.Check.Readability.WithSingleClause, []}, 21 | {Credo.Check.Readability.AliasOrder, []}, 22 | {Credo.Check.Readability.StringSigils, []}, 23 | {Credo.Check.Refactor.Apply, []} 24 | ] 25 | } 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/integration_tests.yml: -------------------------------------------------------------------------------- 1 | name: Integration Tests 2 | on: 3 | pull_request: 4 | paths: 5 | - "lib/**" 6 | - "test/**" 7 | - "config/**" 8 | - "priv/**" 9 | - "assets/**" 10 | - "rel/**" 11 | - "mix.exs" 12 | - "Dockerfile" 13 | - "run.sh" 14 | - "docker-compose.test.yml" 15 | 16 | push: 17 | branches: 18 | - main 19 | 20 | concurrency: 21 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 22 | cancel-in-progress: true 23 | 24 | jobs: 25 | tests: 26 | name: Tests 27 | runs-on: blacksmith-8vcpu-ubuntu-2404 28 | 29 | steps: 30 | - uses: actions/checkout@v2 31 | - name: Run integration test 32 | run: docker compose -f docker-compose.tests.yml up --abort-on-container-exit --exit-code-from test-runner 33 | 34 | -------------------------------------------------------------------------------- /test/realtime/monitoring/prom_ex_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.PromExTest do 2 | use ExUnit.Case, async: true 3 | doctest Realtime.PromEx 4 | alias Realtime.PromEx 5 | 6 | describe "get_metrics/0" do 7 | test "builds metrics in prometheus format which includes host region and id" do 8 | metrics = PromEx.get_metrics() |> IO.iodata_to_binary() 9 | 10 | assert String.contains?( 11 | metrics, 12 | "# HELP beam_system_schedulers_online_info The number of scheduler threads that are online." 13 | ) 14 | 15 | assert String.contains?(metrics, "# TYPE beam_system_schedulers_online_info gauge") 16 | 17 | assert String.contains?( 18 | metrics, 19 | "beam_system_schedulers_online_info{host=\"nohost\",id=\"nohost\",region=\"us-east-1\"}" 20 | ) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /bench/secrets.exs: -------------------------------------------------------------------------------- 1 | alias RealtimeWeb.ChannelsAuthorization 2 | alias Realtime.Helpers, as: H 3 | 4 | jwt = 5 | "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImxvY2FsaG9zdCIsInJvbGUiOiJhbm9uIiwiaWF0IjoxNjU4NjAwNzkxLCJleHAiOjE5NzQxNzY3OTF9.Iki--9QilZ7vySEUJHj0a1T8BDHkR7rmdWStXImCZfk" 6 | 7 | jwt_secret = "d3v_HtNXEpT+zfsyy1LE1WPGmNKLWRfw/rpjnVtCEEM2cSFV2s+kUh5OKX7TPYmG" 8 | 9 | secret_key = "1234567890123456" 10 | string_to_encrypt = "supabase_realtime" 11 | string_to_decrypt = "A5mS7ggkPXm0FaKKoZtrsYNlZA3qZxFe9XA9w2YYqgU=" 12 | 13 | Benchee.run(%{ 14 | "authorize_jwt" => fn -> 15 | {:ok, _} = ChannelsAuthorization.authorize_conn(jwt, jwt_secret, nil) 16 | end, 17 | "encrypt_string" => fn -> 18 | H.encrypt!(string_to_encrypt, secret_key) 19 | end, 20 | "decrypt_string" => fn -> 21 | H.decrypt!(string_to_decrypt, secret_key) 22 | end 23 | }) 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | realtime-*.tar 24 | 25 | # Ignore assets that are produced by build tools. 26 | /priv/static/assets/ 27 | 28 | # Ignore Dialyzer .plt 29 | /priv/plts/* 30 | node_modules 31 | .supabase 32 | config/prod.secret.exs 33 | demo/.env 34 | .lexical 35 | .vscode -------------------------------------------------------------------------------- /lib/realtime/tenants/connect/piper.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Connect.Piper do 2 | @moduledoc """ 3 | Pipes different commands to execute specific actions during the connection process. 4 | """ 5 | require Logger 6 | @callback run(any()) :: {:ok, any()} | {:error, any()} 7 | 8 | def run(pipers, init) do 9 | Enum.reduce_while(pipers, {:ok, init}, fn piper, {:ok, acc} -> 10 | case :timer.tc(fn -> piper.run(acc) end, :millisecond) do 11 | {exec_time, {:ok, result}} -> 12 | Logger.info("#{inspect(piper)} executed in #{exec_time} ms") 13 | {:cont, {:ok, result}} 14 | 15 | {exec_time, {:error, error}} -> 16 | Logger.error("#{inspect(piper)} failed in #{exec_time} ms") 17 | {:halt, {:error, error}} 18 | 19 | _ -> 20 | raise ArgumentError, "must return {:ok, _} or {:error, _}" 21 | end 22 | end) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /rel/env.sh.eex: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Set the release to work across nodes. If using the long name format like 4 | # the one below (my_app@127.0.0.1), you need to also uncomment the 5 | # RELEASE_DISTRIBUTION variable below. Must be "sname", "name" or "none". 6 | 7 | # for Fly.io 8 | ip=$(grep fly-local-6pn /etc/hosts | cut -f 1) 9 | 10 | if [ "$AWS_EXECUTION_ENV" = "AWS_ECS_FARGATE" ]; then 11 | # for AWS ECS Fargate 12 | ip=$(hostname -I | awk '{print $3}') 13 | elif [ -n "${POD_IP}" ]; then 14 | # for kubernetes 15 | ip=${POD_IP} 16 | fi 17 | 18 | # default to localhost 19 | if [ -z $ip ]; then 20 | ip=127.0.0.1 21 | fi 22 | 23 | # assign the value of NODE_NAME if it exists, else assign the value of FLY_APP_NAME, 24 | # and if that doesn't exist either, assign "realtime" to node_name 25 | node_name="${NODE_NAME:=${APP_NAME:=realtime}}" 26 | 27 | export RELEASE_DISTRIBUTION=name 28 | export RELEASE_NODE=$node_name@$ip 29 | -------------------------------------------------------------------------------- /lib/realtime/adapters/postgres/protocol/keep_alive.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Adapters.Postgres.Protocol.KeepAlive do 2 | @moduledoc """ 3 | Primary keepalive message (B) 4 | Byte1('k') 5 | Identifies the message as a sender keepalive. 6 | 7 | Int64 8 | The current end of WAL on the server. 9 | 10 | Int64 11 | The server's system clock at the time of transmission, as microseconds since midnight on 2000-01-01. 12 | 13 | Byte1 14 | 1 means that the client should reply to this message as soon as possible, to avoid a timeout disconnect. 0 otherwise. 15 | 16 | The receiving process can send replies back to the sender at any time, using one of the following message formats (also in the payload of a CopyData message): 17 | """ 18 | @type t :: %__MODULE__{ 19 | wal_end: integer(), 20 | clock: integer(), 21 | reply: :now | :await 22 | } 23 | defstruct [:wal_end, :clock, :reply] 24 | end 25 | -------------------------------------------------------------------------------- /lib/realtime/adapters/postgres/protocol/write.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Adapters.Postgres.Protocol.Write do 2 | @moduledoc """ 3 | XLogData (B) 4 | Byte1('w') 5 | Identifies the message as WAL data. 6 | 7 | Int64 8 | The starting point of the WAL data in this message. 9 | 10 | Int64 11 | The current end of WAL on the server. 12 | 13 | Int64 14 | The server's system clock at the time of transmission, as microseconds since midnight on 2000-01-01. 15 | 16 | Byten 17 | A section of the WAL data stream. 18 | 19 | A single WAL record is never split across two XLogData messages. When a WAL record crosses a WAL page boundary, and is therefore already split using continuation records, it can be split at the page boundary. In other words, the first main WAL record and its continuation records can be sent in different XLogData messages. 20 | """ 21 | defstruct [:server_wal_start, :server_wal_end, :server_system_clock, :message] 22 | end 23 | -------------------------------------------------------------------------------- /lib/realtime_web/live/page_live/index.html.heex: -------------------------------------------------------------------------------- 1 | <.h1>Supabase Realtime: Multiplayer Edition 2 |

Listen to PostgreSQL changes in real-time over WebSockets

3 |

Presence and Broadcast for multiplayer features

4 | 5 |
6 |

Tools

7 | 11 |
12 | 13 |
14 |

Links

15 | 21 |
22 | -------------------------------------------------------------------------------- /.github/workflows/version_updated.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | branches: 4 | - "main" 5 | paths: 6 | - "lib/**" 7 | - "config/**" 8 | - "priv/**" 9 | - "assets/**" 10 | - "rel/**" 11 | - "mix.exs" 12 | - "Dockerfile" 13 | - "run.sh" 14 | 15 | permissions: 16 | contents: read 17 | 18 | name: Default Checks 19 | 20 | jobs: 21 | versions_updated: 22 | name: Versions Updated 23 | runs-on: blacksmith-4vcpu-ubuntu-2404 24 | steps: 25 | - name: Checkout code 26 | uses: actions/checkout@v3 27 | 28 | - name: Verify Versions Updated 29 | uses: step-security/changed-files@v45 30 | id: verify_changed_files 31 | with: 32 | files: | 33 | mix.exs 34 | 35 | - name: Fail Unless Versions Updated 36 | id: fail_unless_changed 37 | if: steps.verify_changed_files.outputs.any_changed == 'false' 38 | run: | 39 | echo "::error ::Please update the mix.exs version" 40 | exit 1 41 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250714121412_broadcast_send_error_logging.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.BroadcastSendErrorLogging do 2 | @moduledoc false 3 | use Ecto.Migration 4 | # Removes pg_notification to use postgres logging instead 5 | def change do 6 | execute(""" 7 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true ) RETURNS void 8 | AS $$ 9 | BEGIN 10 | BEGIN 11 | -- Set the topic configuration 12 | EXECUTE format('SET LOCAL realtime.topic TO %L', topic); 13 | 14 | -- Attempt to insert the message 15 | INSERT INTO realtime.messages (payload, event, topic, private, extension) 16 | VALUES (payload, event, topic, private, 'broadcast'); 17 | EXCEPTION 18 | WHEN OTHERS THEN 19 | -- Capture and notify the error 20 | RAISE WARNING 'ErrorSendingBroadcastMessage: %', SQLERRM; 21 | END; 22 | END; 23 | $$ 24 | LANGUAGE plpgsql; 25 | """) 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/realtime/signal_handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.SignalHandler do 2 | @moduledoc false 3 | @behaviour :gen_event 4 | require Logger 5 | 6 | @spec shutdown_in_progress? :: :ok | {:error, :shutdown_in_progress} 7 | def shutdown_in_progress? do 8 | case !!Application.get_env(:realtime, :shutdown_in_progress) do 9 | true -> {:error, :shutdown_in_progress} 10 | false -> :ok 11 | end 12 | end 13 | 14 | @impl true 15 | def init({%{handler_mod: _} = args, :ok}) do 16 | {:ok, args} 17 | end 18 | 19 | @impl true 20 | def handle_event(signal, %{handler_mod: handler_mod} = state) do 21 | Logger.error("#{__MODULE__}: #{inspect(signal)} received") 22 | 23 | if signal == :sigterm do 24 | Application.put_env(:realtime, :shutdown_in_progress, true) 25 | end 26 | 27 | handler_mod.handle_event(signal, state) 28 | end 29 | 30 | @impl true 31 | defdelegate handle_info(info, state), to: :erl_signal_handler 32 | 33 | @impl true 34 | defdelegate handle_call(request, state), to: :erl_signal_handler 35 | end 36 | -------------------------------------------------------------------------------- /test/realtime/monitoring/erl_sys_mon_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Monitoring.ErlSysMonTest do 2 | use ExUnit.Case, async: true 3 | import ExUnit.CaptureLog 4 | alias Realtime.ErlSysMon 5 | 6 | describe "system monitoring" do 7 | test "logs system monitor events" do 8 | start_supervised!({ErlSysMon, config: [{:long_message_queue, {1, 100}}]}) 9 | 10 | log = 11 | capture_log(fn -> 12 | Task.async(fn -> 13 | Process.register(self(), TestProcess) 14 | Enum.map(1..1000, &send(self(), &1)) 15 | # Wait for ErlSysMon to notice 16 | Process.sleep(4000) 17 | end) 18 | |> Task.await() 19 | end) 20 | 21 | assert log =~ "Realtime.ErlSysMon message:" 22 | assert log =~ "$initial_call\", {Realtime.Monitoring.ErlSysMonTest" 23 | assert log =~ "ancestors\", [#{inspect(self())}]" 24 | assert log =~ "registered_name: TestProcess" 25 | assert log =~ "message_queue_len: " 26 | assert log =~ "total_heap_size: " 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240321100241_add_presences_permissions_table.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.AddPresencesPoliciesTable do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | create table(:presences) do 8 | add :channel_id, references(:channels, on_delete: :delete_all), null: false 9 | add :check, :boolean, default: false, null: false 10 | timestamps() 11 | end 12 | 13 | create unique_index(:presences, :channel_id) 14 | 15 | execute("ALTER TABLE realtime.presences ENABLE row level security") 16 | execute("GRANT SELECT ON realtime.presences TO postgres, anon, authenticated, service_role") 17 | execute("GRANT UPDATE ON realtime.presences TO postgres, anon, authenticated, service_role") 18 | 19 | execute(""" 20 | GRANT INSERT ON realtime.presences TO postgres, anon, authenticated, service_role 21 | """) 22 | 23 | execute(""" 24 | GRANT USAGE ON SEQUENCE realtime.presences_id_seq TO postgres, anon, authenticated, service_role 25 | """) 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/realtime_web/live/status_live/index_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.StatusLive.IndexTest do 2 | use RealtimeWeb.ConnCase 3 | import Phoenix.LiveViewTest 4 | 5 | alias Realtime.Latency.Payload 6 | alias Realtime.Nodes 7 | alias RealtimeWeb.Endpoint 8 | 9 | describe "Status LiveView" do 10 | test "renders status page", %{conn: conn} do 11 | {:ok, _view, html} = live(conn, ~p"/status") 12 | 13 | assert html =~ "Realtime Status" 14 | end 15 | 16 | test "receives broadcast from PubSub", %{conn: conn} do 17 | {:ok, view, _html} = live(conn, ~p"/status") 18 | 19 | payload = %Payload{ 20 | from_node: Nodes.short_node_id_from_name(:"pink@127.0.0.1"), 21 | node: Nodes.short_node_id_from_name(:"orange@127.0.0.1"), 22 | latency: "42ms", 23 | timestamp: DateTime.utc_now() 24 | } 25 | 26 | Endpoint.broadcast("admin:cluster", "ping", payload) 27 | 28 | html = render(view) 29 | assert html =~ "42ms" 30 | assert html =~ "pink@127.0.0.1_orange@127.0.0.1" 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/realtime_web/live/ping_live.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.PingLive do 2 | use RealtimeWeb, :live_view 3 | 4 | def mount(_params, _session, socket) do 5 | ping() 6 | {:ok, assign(socket, :ping, "0.0 ms")} 7 | end 8 | 9 | def render(assigns) do 10 | ~H""" 11 | <%= @ping %> 12 | """ 13 | end 14 | 15 | def handle_info(:ping, socket) do 16 | socket = socket |> push_event("ping", %{ping: DateTime.utc_now() |> DateTime.to_iso8601()}) 17 | 18 | {:noreply, socket} 19 | end 20 | 21 | def handle_event("pong", %{"ping" => ping}, socket) do 22 | {:ok, datetime, 0} = DateTime.from_iso8601(ping) 23 | 24 | pong = 25 | (DateTime.diff(DateTime.utc_now(), datetime, :microsecond) / 1000) 26 | |> Float.round(1) 27 | |> Float.to_string() 28 | 29 | ping() 30 | {:noreply, assign(socket, :ping, pong <> " ms")} 31 | end 32 | 33 | defp ping do 34 | timer = if Mix.env() == :dev, do: 60_000, else: 1_000 35 | Process.send_after(self(), :ping, timer) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/realtime/logs_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.LogsTest do 2 | use ExUnit.Case 3 | 4 | describe "Jason.Encoder implementation" do 5 | test "encodes DBConnection.ConnectionError" do 6 | error = %DBConnection.ConnectionError{ 7 | message: "connection lost", 8 | reason: :timeout, 9 | severity: :error 10 | } 11 | 12 | encoded = Jason.encode!(error) 13 | assert encoded =~ "message: \"connection lost\"" 14 | assert encoded =~ "reason: :timeout" 15 | assert encoded =~ "severity: :error" 16 | end 17 | 18 | test "encodes Postgrex.Error" do 19 | error = %Postgrex.Error{ 20 | message: "relation not found", 21 | postgres: %{ 22 | code: "42P01", 23 | schema: "public", 24 | table: "users" 25 | } 26 | } 27 | 28 | encoded = Jason.encode!(error) 29 | assert encoded =~ "message: \"relation not found\"" 30 | assert encoded =~ "schema: \"public\"" 31 | assert encoded =~ "table: \"users\"" 32 | assert encoded =~ "code: \"42P01\"" 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/realtime_web/channels/tenant_rate_limiters_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.TenantRateLimitersTest do 2 | use Realtime.DataCase, async: true 3 | 4 | use Mimic 5 | alias RealtimeWeb.TenantRateLimiters 6 | alias Realtime.Api.Tenant 7 | 8 | setup do 9 | tenant = %Tenant{external_id: random_string(), max_concurrent_users: 1, max_joins_per_second: 1} 10 | 11 | %{tenant: tenant} 12 | end 13 | 14 | describe "check_tenant/1" do 15 | test "rate is not exceeded", %{tenant: tenant} do 16 | assert TenantRateLimiters.check_tenant(tenant) == :ok 17 | end 18 | 19 | test "max concurrent users is exceeded", %{tenant: tenant} do 20 | Realtime.UsersCounter.add(self(), tenant.external_id) 21 | 22 | assert TenantRateLimiters.check_tenant(tenant) == {:error, :too_many_connections} 23 | end 24 | 25 | test "max joins is exceeded", %{tenant: tenant} do 26 | expect(Realtime.RateCounter, :get, fn _ -> {:ok, %{limit: %{triggered: true}}} end) 27 | 28 | assert TenantRateLimiters.check_tenant(tenant) == {:error, :too_many_joins} 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/realtime/telemetry/logger_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Telemetry.LoggerTest do 2 | use ExUnit.Case 3 | import ExUnit.CaptureLog 4 | alias Realtime.Telemetry.Logger, as: TelemetryLogger 5 | 6 | setup do 7 | level = Logger.level() 8 | Logger.configure(level: :info) 9 | on_exit(fn -> Logger.configure(level: level) end) 10 | end 11 | 12 | describe "logger backend initialization" do 13 | test "logs on telemetry event" do 14 | start_link_supervised!({TelemetryLogger, handler_id: "telemetry-logger-test"}) 15 | 16 | assert capture_log(fn -> 17 | :telemetry.execute([:realtime, :connections], %{count: 1}, %{tenant: "tenant"}) 18 | end) =~ "Billing metrics: [:realtime, :connections]" 19 | end 20 | 21 | test "ignores events without tenant" do 22 | start_link_supervised!({TelemetryLogger, handler_id: "telemetry-logger-test"}) 23 | 24 | refute capture_log(fn -> 25 | :telemetry.execute([:realtime, :connections], %{count: 1}, %{}) 26 | end) =~ "Billing metrics: [:realtime, :connections]" 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/realtime/monitoring/distributed_metrics_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.DistributedMetricsTest do 2 | # Async false due to Clustered usage 3 | use ExUnit.Case, async: false 4 | 5 | alias Realtime.DistributedMetrics 6 | 7 | setup_all do 8 | {:ok, node} = Clustered.start() 9 | %{node: node} 10 | end 11 | 12 | describe "info/0 while connected" do 13 | test "per node metric", %{node: node} do 14 | assert %{ 15 | ^node => %{ 16 | pid: _pid, 17 | port: _port, 18 | queue_size: {:ok, _}, 19 | state: :up, 20 | inet_stats: [ 21 | recv_oct: _, 22 | recv_cnt: _, 23 | recv_max: _, 24 | recv_avg: _, 25 | recv_dvi: _, 26 | send_oct: _, 27 | send_cnt: _, 28 | send_max: _, 29 | send_avg: _, 30 | send_pend: _ 31 | ] 32 | } 33 | } = DistributedMetrics.info() 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/extensions/postgres_cdc_rls/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Extensions.PostgresCdcRls.Supervisor do 2 | @moduledoc """ 3 | Supervisor to spin up the Postgres CDC RLS tree. 4 | """ 5 | use Supervisor 6 | 7 | alias Extensions.PostgresCdcRls 8 | 9 | @spec start_link :: :ignore | {:error, any} | {:ok, pid} 10 | def start_link do 11 | Supervisor.start_link(__MODULE__, [], name: __MODULE__) 12 | end 13 | 14 | @impl true 15 | def init(_args) do 16 | load_migrations_modules() 17 | 18 | :syn.add_node_to_scopes(Realtime.Syn.PostgresCdc.scopes()) 19 | 20 | children = [ 21 | { 22 | PartitionSupervisor, 23 | partitions: 20, child_spec: DynamicSupervisor, strategy: :one_for_one, name: PostgresCdcRls.DynamicSupervisor 24 | } 25 | ] 26 | 27 | Supervisor.init(children, strategy: :one_for_one) 28 | end 29 | 30 | defp load_migrations_modules do 31 | {:ok, modules} = :application.get_key(:realtime, :modules) 32 | 33 | modules 34 | |> Enum.filter(&String.starts_with?(to_string(&1), "Elixir.Realtime.Tenants.Migrations")) 35 | |> Enum.each(&Code.ensure_loaded!/1) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /priv/static/favicon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211116213934_create_realtime_is_visible_through_filters_function.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeIsVisibleThroughFiltersFunction do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute( 8 | "create function realtime.is_visible_through_filters(columns realtime.wal_column[], filters realtime.user_defined_filter[]) 9 | returns bool 10 | language sql 11 | immutable 12 | as $$ 13 | /* 14 | Should the record be visible (true) or filtered out (false) after *filters* are applied 15 | */ 16 | select 17 | -- Default to allowed when no filters present 18 | coalesce( 19 | sum( 20 | realtime.check_equality_op( 21 | op:=f.op, 22 | type_:=col.type::regtype, 23 | -- cast jsonb to text 24 | val_1:=col.value #>> '{}', 25 | val_2:=f.value 26 | )::int 27 | ) = count(1), 28 | true 29 | ) 30 | from 31 | unnest(filters) f 32 | join unnest(columns) col 33 | on f.column_name = col.name; 34 | $$;" 35 | ) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /.github/workflows/mirror.yml: -------------------------------------------------------------------------------- 1 | name: Mirror Image 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | version: 7 | description: "Image tag" 8 | required: true 9 | type: string 10 | 11 | jobs: 12 | mirror: 13 | runs-on: blacksmith-4vcpu-ubuntu-2404 14 | permissions: 15 | contents: read 16 | packages: write 17 | id-token: write 18 | steps: 19 | - name: configure aws credentials 20 | uses: aws-actions/configure-aws-credentials@v1 21 | with: 22 | role-to-assume: ${{ secrets.PROD_AWS_ROLE }} 23 | aws-region: us-east-1 24 | - uses: docker/login-action@v2 25 | with: 26 | registry: public.ecr.aws 27 | - uses: docker/login-action@v2 28 | with: 29 | registry: ghcr.io 30 | username: ${{ github.actor }} 31 | password: ${{ secrets.GITHUB_TOKEN }} 32 | - uses: akhilerm/tag-push-action@v2.1.0 33 | with: 34 | src: docker.io/supabase/realtime:${{ inputs.version }} 35 | dst: | 36 | public.ecr.aws/supabase/realtime:${{ inputs.version }} 37 | ghcr.io/supabase/realtime:${{ inputs.version }} 38 | -------------------------------------------------------------------------------- /lib/realtime/helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Helpers do 2 | @moduledoc """ 3 | This module includes helper functions for different contexts that can't be union in one module. 4 | """ 5 | require Logger 6 | 7 | @spec cancel_timer(reference() | nil) :: non_neg_integer() | false | :ok | nil 8 | def cancel_timer(nil), do: nil 9 | def cancel_timer(ref), do: Process.cancel_timer(ref) 10 | 11 | @doc """ 12 | Takes the first N items from the queue and returns the list of items and the new queue. 13 | 14 | ## Examples 15 | 16 | iex> q = :queue.new() 17 | iex> q = :queue.in(1, q) 18 | iex> q = :queue.in(2, q) 19 | iex> q = :queue.in(3, q) 20 | iex> Realtime.Helpers.queue_take(q, 2) 21 | {[2, 1], {[], [3]}} 22 | """ 23 | 24 | @spec queue_take(:queue.queue(), non_neg_integer()) :: {list(), :queue.queue()} 25 | def queue_take(q, count) do 26 | Enum.reduce_while(1..count, {[], q}, fn _, {items, queue} -> 27 | case :queue.out(queue) do 28 | {{:value, item}, new_q} -> 29 | {:cont, {[item | items], new_q}} 30 | 31 | {:empty, new_q} -> 32 | {:halt, {items, new_q}} 33 | end 34 | end) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/realtime/release.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Release do 2 | @moduledoc """ 3 | Used for executing DB release tasks when run in production without Mix 4 | installed. 5 | """ 6 | @app :realtime 7 | 8 | def migrate do 9 | load_app() 10 | 11 | for repo <- repos() do 12 | {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :up, all: true)) 13 | end 14 | end 15 | 16 | def rollback(repo, version) do 17 | load_app() 18 | {:ok, _, _} = Ecto.Migrator.with_repo(repo, &Ecto.Migrator.run(&1, :down, to: version)) 19 | end 20 | 21 | def seeds(repo) do 22 | load_app() 23 | {:ok, _} = Application.ensure_all_started(:realtime) 24 | 25 | {:ok, {:ok, _}, _} = 26 | Ecto.Migrator.with_repo(repo, fn _repo -> 27 | seeds_file = "#{:code.priv_dir(@app)}/repo/seeds.exs" 28 | 29 | if File.regular?(seeds_file) do 30 | {:ok, Code.eval_file(seeds_file)} 31 | else 32 | {:error, "Seeds file not found."} 33 | end 34 | end) 35 | end 36 | 37 | defp repos do 38 | Application.fetch_env!(@app, :ecto_repos) 39 | end 40 | 41 | defp load_app do 42 | Application.load(@app) 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/realtime/crypto.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Crypto do 2 | @moduledoc """ 3 | Encrypt and decrypt operations required by Realtime. It uses the secret set on Application.get_env(:realtime, :db_enc_key) 4 | """ 5 | 6 | @doc """ 7 | Encrypts the given text 8 | """ 9 | @spec encrypt!(binary()) :: binary() 10 | def encrypt!(text) do 11 | secret_key = Application.get_env(:realtime, :db_enc_key) 12 | 13 | :aes_128_ecb 14 | |> :crypto.crypto_one_time(secret_key, pad(text), true) 15 | |> Base.encode64() 16 | end 17 | 18 | @doc """ 19 | Decrypts the given base64 encoded text 20 | """ 21 | @spec decrypt!(binary()) :: binary() 22 | def decrypt!(base64_text) do 23 | secret_key = Application.get_env(:realtime, :db_enc_key) 24 | crypto_text = Base.decode64!(base64_text) 25 | 26 | :aes_128_ecb 27 | |> :crypto.crypto_one_time(secret_key, crypto_text, false) 28 | |> unpad() 29 | end 30 | 31 | defp pad(data) do 32 | to_add = 16 - rem(byte_size(data), 16) 33 | data <> :binary.copy(<>, to_add) 34 | end 35 | 36 | defp unpad(data) do 37 | to_remove = :binary.last(data) 38 | :binary.part(data, 0, byte_size(data) - to_remove) 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/realtime_web/plugs/rate_limiter.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Plugs.RateLimiter do 2 | @moduledoc """ 3 | Rate limits tenants. 4 | """ 5 | import Plug.Conn 6 | import Phoenix.Controller, only: [json: 2] 7 | require Logger 8 | 9 | alias Realtime.Api.Tenant 10 | 11 | def init(opts) do 12 | opts 13 | end 14 | 15 | def call( 16 | %{ 17 | assigns: %{ 18 | tenant: %Tenant{ 19 | events_per_second_rolling: avg, 20 | events_per_second_now: _current, 21 | max_events_per_second: max 22 | } 23 | } 24 | } = conn, 25 | _opts 26 | ) do 27 | avg = trunc(avg) 28 | 29 | conn = 30 | conn 31 | |> put_resp_header("x-rate-rolling", Integer.to_string(avg)) 32 | |> put_resp_header("x-rate-limit", Integer.to_string(max)) 33 | |> put_resp_header("x-rate-limit-remaining", Integer.to_string(max - avg)) 34 | 35 | if avg >= max do 36 | conn 37 | |> put_status(429) 38 | |> json(%{message: "Too many requests"}) 39 | |> halt() 40 | else 41 | conn 42 | end 43 | end 44 | 45 | def call(conn, _opts) do 46 | conn 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211116051442_create_realtime_check_equality_op_function.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeCheckEqualityOpFunction do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute("create function realtime.check_equality_op( 8 | op realtime.equality_op, 9 | type_ regtype, 10 | val_1 text, 11 | val_2 text 12 | ) 13 | returns bool 14 | immutable 15 | language plpgsql 16 | as $$ 17 | /* 18 | Casts *val_1* and *val_2* as type *type_* and check the *op* condition for truthiness 19 | */ 20 | declare 21 | op_symbol text = ( 22 | case 23 | when op = 'eq' then '=' 24 | when op = 'neq' then '!=' 25 | when op = 'lt' then '<' 26 | when op = 'lte' then '<=' 27 | when op = 'gt' then '>' 28 | when op = 'gte' then '>=' 29 | else 'UNKNOWN OP' 30 | end 31 | ); 32 | res boolean; 33 | begin 34 | execute format('select %L::'|| type_::text || ' ' || op_symbol || ' %L::'|| type_::text, val_1, val_2) into res; 35 | return res; 36 | end; 37 | $$;") 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/realtime/tenants/authorization/policies.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Authorization.Policies do 2 | @moduledoc """ 3 | Policies structure that holds the required authorization information for a given connection. 4 | 5 | Currently there are two types of policies: 6 | * Realtime.Tenants.Authorization.Policies.BroadcastPolicies - Used to store the access to Broadcast feature on a given Topic 7 | * Realtime.Tenants.Authorization.Policies.PresencePolicies - Used to store the access to Presence feature on a given Topic 8 | """ 9 | 10 | alias Realtime.Tenants.Authorization.Policies.BroadcastPolicies 11 | alias Realtime.Tenants.Authorization.Policies.PresencePolicies 12 | 13 | defstruct broadcast: %BroadcastPolicies{}, 14 | presence: %PresencePolicies{} 15 | 16 | @type t :: %__MODULE__{ 17 | broadcast: BroadcastPolicies.t(), 18 | presence: PresencePolicies.t() 19 | } 20 | 21 | @doc """ 22 | Updates the Policies struct sub key with the given value. 23 | """ 24 | @spec update_policies(t(), atom, atom, boolean) :: t() 25 | def update_policies(policies, key, sub_key, value) do 26 | Map.update!(policies, key, fn map -> Map.put(map, sub_key, value) end) 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /coveralls.json: -------------------------------------------------------------------------------- 1 | { 2 | "skip_files": [ 3 | "lib/realtime_web/api_spec.ex", 4 | "lib/realtime_web/channels/presence.ex", 5 | "lib/realtime_web/controllers/page_controller.ex", 6 | "lib/realtime_web/dashboard/", 7 | "lib/realtime_web/endpoint.ex", 8 | "lib/realtime_web/gettext.ex", 9 | "lib/realtime_web/live/", 10 | "lib/realtime_web/open_api_schemas.ex", 11 | "lib/realtime_web/telemetry.ex", 12 | "lib/realtime_web/views/", 13 | "lib/realtime.ex", 14 | "lib/realtime/adapters/changes.ex", 15 | "lib/realtime/adapters/postgres/decoder.ex", 16 | "lib/realtime/adapters/postgres/oid_database.ex", 17 | "lib/realtime/adapters/postgres/protocol/", 18 | "lib/realtime/application.ex", 19 | "lib/realtime/monitoring/prom_ex/plugins/phoenix.ex", 20 | "lib/realtime/operations.ex", 21 | "lib/realtime/release.ex", 22 | "lib/realtime/tenants/authorization/policies/broadcast_policies.ex", 23 | "lib/realtime/tenants/authorization/policies/presence_policies.ex", 24 | "lib/realtime/tenants/repo/migrations/", 25 | "/lib/realtime/tenants/cache_supervisor.ex", 26 | "test/" 27 | ] 28 | } -------------------------------------------------------------------------------- /lib/realtime_web/api_spec.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ApiSpec do 2 | @moduledoc false 3 | 4 | alias OpenApiSpex.Components 5 | alias OpenApiSpex.Info 6 | alias OpenApiSpex.OpenApi 7 | alias OpenApiSpex.Paths 8 | alias OpenApiSpex.SecurityScheme 9 | alias OpenApiSpex.Server 10 | alias OpenApiSpex.ServerVariable 11 | 12 | alias RealtimeWeb.Router 13 | 14 | @behaviour OpenApi 15 | 16 | @impl OpenApi 17 | def spec do 18 | url = 19 | case Mix.env() do 20 | :prod -> "https://{tenant}.supabase.co/realtime/v1" 21 | _ -> "http://{tenant}.localhost:4000/" 22 | end 23 | 24 | %OpenApi{ 25 | servers: [ 26 | %Server{ 27 | url: url, 28 | variables: %{"tenant" => %ServerVariable{default: "tenant"}} 29 | } 30 | ], 31 | info: %Info{ 32 | title: to_string(Application.spec(:realtime, :description)), 33 | version: to_string(Application.spec(:realtime, :vsn)) 34 | }, 35 | paths: Paths.from_router(Router), 36 | components: %Components{ 37 | securitySchemes: %{"authorization" => %SecurityScheme{type: "http", scheme: "bearer"}} 38 | } 39 | } 40 | |> OpenApiSpex.resolve_schema_modules() 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/realtime_web/plugs/baggage_request_id.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Plugs.BaggageRequestId do 2 | @moduledoc """ 3 | Populates request ID based on trace baggage. 4 | It looks for the specified `baggage_key` (default to 'request-id'). 5 | 6 | Otherwise generates a request ID using `Plug.RequestId` 7 | """ 8 | 9 | def baggage_key, do: Application.get_env(:realtime, :request_id_baggage_key, "request-id") 10 | 11 | require Logger 12 | alias Plug.Conn 13 | @behaviour Plug 14 | 15 | @impl true 16 | @doc false 17 | def init(opts) do 18 | Keyword.get(opts, :baggage_key, "request-id") 19 | end 20 | 21 | @impl true 22 | @doc false 23 | @spec call(Conn.t(), String.t()) :: Conn.t() 24 | def call(conn, baggage_key) do 25 | :otel_propagator_text_map.extract(conn.req_headers) 26 | 27 | with %{^baggage_key => {request_id, _}} <- :otel_baggage.get_all(), 28 | true <- valid_request_id?(request_id) do 29 | Logger.metadata(request_id: request_id) 30 | Conn.put_resp_header(conn, "x-request-id", request_id) 31 | else 32 | _ -> 33 | opts = Plug.RequestId.init([]) 34 | Plug.RequestId.call(conn, opts) 35 | end 36 | end 37 | 38 | defp valid_request_id?(s), do: byte_size(s) in 10..200 39 | end 40 | -------------------------------------------------------------------------------- /deploy/fly/qa.toml: -------------------------------------------------------------------------------- 1 | app = "realtime-qa" 2 | kill_signal = "SIGTERM" 3 | kill_timeout = 5 4 | processes = [] 5 | 6 | [deploy] 7 | release_command = "/app/bin/migrate" 8 | strategy = "rolling" 9 | 10 | [env] 11 | DNS_NODES = "realtime-qa.internal" 12 | ERL_CRASH_DUMP = "/data/erl_crash.dump" 13 | ERL_CRASH_DUMP_SECONDS = 30 14 | 15 | [experimental] 16 | allowed_public_ports = [] 17 | auto_rollback = true 18 | 19 | [[services]] 20 | internal_port = 4000 21 | processes = ["app"] 22 | protocol = "tcp" 23 | script_checks = [] 24 | [services.concurrency] 25 | hard_limit = 100000 26 | soft_limit = 100000 27 | type = "connections" 28 | 29 | [[services.ports]] 30 | force_https = true 31 | handlers = ["http"] 32 | port = 80 33 | 34 | [[services.ports]] 35 | handlers = ["tls", "http"] 36 | port = 443 37 | 38 | [[services.tcp_checks]] 39 | grace_period = "30s" 40 | interval = "15s" 41 | restart_limit = 6 42 | timeout = "2s" 43 | 44 | [[services.http_checks]] 45 | interval = 10000 46 | grace_period = "5s" 47 | method = "get" 48 | path = "/" 49 | protocol = "http" 50 | restart_limit = 0 51 | timeout = 2000 52 | tls_skip_verify = false 53 | [services.http_checks.headers] 54 | -------------------------------------------------------------------------------- /lib/realtime_web/views/tenant_view.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.TenantView do 2 | use RealtimeWeb, :view 3 | alias RealtimeWeb.TenantView 4 | 5 | def render("index.json", %{tenants: tenants}) do 6 | %{data: render_many(tenants, TenantView, "tenant.json")} 7 | end 8 | 9 | def render("show.json", %{tenant: tenant}) do 10 | %{data: render_one(tenant, TenantView, "tenant.json")} 11 | end 12 | 13 | def render("not_found.json", %{tenant: nil}) do 14 | %{error: "not found"} 15 | end 16 | 17 | def render("tenant.json", %{tenant: tenant}) do 18 | %{ 19 | id: tenant.id, 20 | external_id: tenant.external_id, 21 | name: tenant.name, 22 | max_concurrent_users: tenant.max_concurrent_users, 23 | max_channels_per_client: tenant.max_channels_per_client, 24 | max_events_per_second: tenant.max_events_per_second, 25 | max_joins_per_second: tenant.max_joins_per_second, 26 | inserted_at: tenant.inserted_at, 27 | extensions: 28 | Enum.map(tenant.extensions, fn extension -> 29 | Map.update(extension, :settings, %{}, fn settings -> 30 | Map.drop(settings, ["db_password"]) 31 | end) 32 | end), 33 | private_only: tenant.private_only 34 | } 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/realtime_web/live/tenants_live/index_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.TenantsLive.IndexTest do 2 | use RealtimeWeb.ConnCase 3 | import Phoenix.LiveViewTest 4 | 5 | describe "TenantsLive Index" do 6 | setup do 7 | user = random_string() 8 | password = random_string() 9 | 10 | System.put_env("DASHBOARD_USER", user) 11 | System.put_env("DASHBOARD_PASSWORD", password) 12 | 13 | on_exit(fn -> 14 | System.delete_env("DASHBOARD_USER") 15 | System.delete_env("DASHBOARD_PASSWORD") 16 | end) 17 | 18 | %{user: user, password: password} 19 | end 20 | 21 | test "renders tenant view", %{conn: conn, user: user, password: password} do 22 | {:ok, _view, html} = 23 | conn |> using_basic_auth(user, password) |> live(~p"/admin/tenants") 24 | 25 | assert html =~ "Listing all Supabase Realtime tenants." 26 | end 27 | 28 | test "returns 401 if no credentials", %{conn: conn} do 29 | assert conn |> get(~p"/admin/tenants") |> response(401) 30 | end 31 | end 32 | 33 | defp using_basic_auth(conn, username, password) do 34 | header_content = "Basic " <> Base.encode64("#{username}:#{password}") 35 | put_req_header(conn, "authorization", header_content) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/payloads/config.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Channels.Payloads.Config do 2 | @moduledoc """ 3 | Validate config field of the join payload. 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | alias RealtimeWeb.Channels.Payloads.Join 8 | alias RealtimeWeb.Channels.Payloads.Broadcast 9 | alias RealtimeWeb.Channels.Payloads.Presence 10 | alias RealtimeWeb.Channels.Payloads.PostgresChange 11 | 12 | embedded_schema do 13 | embeds_one :broadcast, Broadcast 14 | embeds_one :presence, Presence 15 | embeds_many :postgres_changes, PostgresChange 16 | field :private, :boolean, default: false 17 | end 18 | 19 | def changeset(config, attrs) do 20 | attrs = 21 | attrs 22 | |> Enum.map(fn 23 | {k, v} when is_list(v) -> {k, Enum.filter(v, fn v -> v != nil end)} 24 | {k, v} -> {k, v} 25 | end) 26 | |> Map.new() 27 | 28 | config 29 | |> cast(attrs, [:private], message: &Join.error_message/2) 30 | |> cast_embed(:broadcast, invalid_message: "unable to parse, expected a map") 31 | |> cast_embed(:presence, invalid_message: "unable to parse, expected a map") 32 | |> cast_embed(:postgres_changes, invalid_message: "unable to parse, expected an array of maps") 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/support/channel_case.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ChannelCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | channel tests. 5 | 6 | Such tests rely on `Phoenix.ChannelTest` and also 7 | import other functionality to make it easier 8 | to build common data structures and query the data layer. 9 | 10 | Finally, if the test case interacts with the database, 11 | we enable the SQL sandbox, so changes done to the database 12 | are reverted at the end of every test. If you are using 13 | PostgreSQL, you can even run database tests asynchronously 14 | by setting `use RealtimeWeb.ChannelCase, async: true`, although 15 | this option is not recommended for other databases. 16 | """ 17 | 18 | use ExUnit.CaseTemplate 19 | alias Ecto.Adapters.SQL.Sandbox 20 | 21 | using do 22 | quote do 23 | # Import conveniences for testing with channels 24 | import Phoenix.ChannelTest 25 | import Generators 26 | import TenantConnection 27 | # The default endpoint for testing 28 | @endpoint RealtimeWeb.Endpoint 29 | end 30 | end 31 | 32 | setup tags do 33 | pid = Sandbox.start_owner!(Realtime.Repo, shared: not tags[:async]) 34 | on_exit(fn -> Sandbox.stop_owner(pid) end) 35 | :ok 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/support/joken_current_time_mock.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Joken.CurrentTime.Mock do 2 | @moduledoc """ 3 | 4 | Mock implementation of Joken current time with time freezing. 5 | 6 | This is a copy of Joken.CurrentTime.Mock. 7 | 8 | """ 9 | 10 | use Agent 11 | 12 | def start_link do 13 | Agent.start_link( 14 | fn -> 15 | %{is_frozen: false, frozen_value: nil} 16 | end, 17 | name: Joken 18 | ) 19 | end 20 | 21 | def child_spec(_args) do 22 | %{ 23 | id: __MODULE__, 24 | start: {__MODULE__, :start_link, []} 25 | } 26 | end 27 | 28 | def current_time do 29 | state = Agent.get(Joken, fn state -> state end) 30 | 31 | if state[:is_frozen] do 32 | state[:frozen_value] 33 | else 34 | :os.system_time(:second) 35 | end 36 | end 37 | 38 | def freeze do 39 | freeze(:os.system_time(:second)) 40 | end 41 | 42 | def freeze(timestamp) do 43 | Agent.update(Joken, fn _state -> 44 | %{is_frozen: true, frozen_value: timestamp} 45 | end) 46 | end 47 | 48 | def unique_name_per_process do 49 | binary_pid = 50 | self() 51 | |> :erlang.pid_to_list() 52 | |> :erlang.iolist_to_binary() 53 | 54 | "{__MODULE__}_#{binary_pid}" |> String.to_atom() 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241220123912_realtime_send_handle_exceptions_remove_partition_creation.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RealtimeSendHandleExceptionsRemovePartitionCreation do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # We missed the schema prefix of `realtime.` in the create table partition statement 6 | def change do 7 | execute(""" 8 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true ) RETURNS void 9 | AS $$ 10 | BEGIN 11 | BEGIN 12 | -- Attempt to insert the message 13 | INSERT INTO realtime.messages (payload, event, topic, private, extension) 14 | VALUES (payload, event, topic, private, 'broadcast'); 15 | EXCEPTION 16 | WHEN OTHERS THEN 17 | -- Capture and notify the error 18 | PERFORM pg_notify( 19 | 'realtime:system', 20 | jsonb_build_object( 21 | 'error', SQLERRM, 22 | 'function', 'realtime.send', 23 | 'event', event, 24 | 'topic', topic, 25 | 'private', private 26 | )::text 27 | ); 28 | END; 29 | END; 30 | $$ 31 | LANGUAGE plpgsql; 32 | """) 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/realtime/telemetry/logger.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Telemetry.Logger do 2 | @moduledoc """ 3 | We can log less frequent Telemetry events to get data into BigQuery. 4 | """ 5 | 6 | require Logger 7 | 8 | use GenServer 9 | 10 | @events [ 11 | [:realtime, :connections], 12 | [:realtime, :rate_counter, :channel, :events], 13 | [:realtime, :rate_counter, :channel, :joins], 14 | [:realtime, :rate_counter, :channel, :db_events], 15 | [:realtime, :rate_counter, :channel, :presence_events] 16 | ] 17 | 18 | def start_link(args) do 19 | GenServer.start_link(__MODULE__, args) 20 | end 21 | 22 | def init(handler_id: handler_id) do 23 | :telemetry.attach_many(handler_id, @events, &__MODULE__.handle_event/4, []) 24 | 25 | {:ok, []} 26 | end 27 | 28 | @doc """ 29 | Logs billing metrics for a tenant aggregated and emitted by a PromEx metric poller. 30 | """ 31 | def handle_event(event, measurements, %{tenant: tenant}, _config) do 32 | meta = %{project: tenant, measurements: measurements} 33 | Logger.info(["Billing metrics: ", inspect(event)], meta) 34 | :ok 35 | end 36 | 37 | def handle_event(_event, _measurements, _metadata, _config) do 38 | :ok 39 | end 40 | 41 | def handle_info(_msg, state) do 42 | {:noreply, state} 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/realtime_web/dashboard/process_dump.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Dashboard.ProcessDump do 2 | @moduledoc """ 3 | Live Dashboard page to dump the current processes tree 4 | """ 5 | use Phoenix.LiveDashboard.PageBuilder 6 | 7 | @impl true 8 | def menu_link(_, _) do 9 | {:ok, "Process Dump"} 10 | end 11 | 12 | @impl true 13 | def mount(_, _, socket) do 14 | ts = :os.system_time(:millisecond) 15 | name = "process_dump_#{ts}" 16 | content = dump_processes(name) 17 | {:ok, socket |> assign(content: content) |> assign(name: name)} 18 | end 19 | 20 | @impl true 21 | def render(assigns) do 22 | ~H""" 23 |
24 |

Process Dump

25 | 26 | Download 27 | 28 |
After you untar the file, you can use `File.read!("filename") |> :erlang.binary_to_term` to check the contents 29 |
30 | """ 31 | end 32 | 33 | defp dump_processes(name) do 34 | term = Process.list() |> Enum.map(&Process.info/1) |> :erlang.term_to_binary() 35 | path = "/tmp/#{name}" 36 | File.write!(path, term) 37 | System.cmd("tar", ["-czf", "#{path}.tar.gz", path]) 38 | "#{path}.tar.gz" |> File.read!() |> Base.encode64() 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/realtime/signal_handler_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.SignalHandlerTest do 2 | use ExUnit.Case 3 | import ExUnit.CaptureLog 4 | alias Realtime.SignalHandler 5 | 6 | defmodule FakeHandler do 7 | def handle_event(:sigterm, _state), do: send(self(), :ok) 8 | end 9 | 10 | setup do 11 | on_exit(fn -> 12 | Application.put_env(:realtime, :shutdown_in_progress, false) 13 | end) 14 | end 15 | 16 | describe "signal handling" do 17 | test "sends signal to handler_mod" do 18 | {:ok, state} = SignalHandler.init({%{handler_mod: FakeHandler}, :ok}) 19 | 20 | assert capture_log(fn -> SignalHandler.handle_event(:sigterm, state) end) =~ 21 | "SignalHandler: :sigterm received" 22 | 23 | assert_receive :ok 24 | end 25 | end 26 | 27 | describe "shutdown_in_progress?/1" do 28 | test "shutdown_in_progress? returns error when shutdown is in progress" do 29 | Application.put_env(:realtime, :shutdown_in_progress, true) 30 | assert SignalHandler.shutdown_in_progress?() == {:error, :shutdown_in_progress} 31 | end 32 | 33 | test "shutdown_in_progress? returns ok when no shutdown in progress" do 34 | Application.put_env(:realtime, :shutdown_in_progress, false) 35 | assert SignalHandler.shutdown_in_progress?() == :ok 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241121104152_fix_send_function_.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.FixSendFunction do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # We missed the schema prefix of `realtime.` in the create table partition statement 6 | def change do 7 | execute(""" 8 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true) 9 | RETURNS void 10 | AS $$ 11 | DECLARE 12 | partition_name text; 13 | BEGIN 14 | partition_name := 'messages_' || to_char(NOW(), 'YYYY_MM_DD'); 15 | 16 | IF NOT EXISTS ( 17 | SELECT 1 18 | FROM pg_class c 19 | JOIN pg_namespace n ON n.oid = c.relnamespace 20 | WHERE n.nspname = 'realtime' 21 | AND c.relname = partition_name 22 | ) THEN 23 | EXECUTE format( 24 | 'CREATE TABLE realtime.%I PARTITION OF realtime.messages FOR VALUES FROM (%L) TO (%L)', 25 | partition_name, 26 | NOW(), 27 | (NOW() + interval '1 day')::timestamp 28 | ); 29 | END IF; 30 | 31 | INSERT INTO realtime.messages (payload, event, topic, private, extension) 32 | VALUES (payload, event, topic, private, 'broadcast'); 33 | END; 34 | $$ 35 | LANGUAGE plpgsql; 36 | """) 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241224161212_realtime_send_sets_config.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RealtimeSendSetsConfig do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # We missed the schema prefix of `realtime.` in the create table partition statement 6 | def change do 7 | execute(""" 8 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true ) RETURNS void 9 | AS $$ 10 | BEGIN 11 | BEGIN 12 | -- Set the topic configuration 13 | SET LOCAL realtime.topic TO topic; 14 | 15 | -- Attempt to insert the message 16 | INSERT INTO realtime.messages (payload, event, topic, private, extension) 17 | VALUES (payload, event, topic, private, 'broadcast'); 18 | EXCEPTION 19 | WHEN OTHERS THEN 20 | -- Capture and notify the error 21 | PERFORM pg_notify( 22 | 'realtime:system', 23 | jsonb_build_object( 24 | 'error', SQLERRM, 25 | 'function', 'realtime.send', 26 | 'event', event, 27 | 'topic', topic, 28 | 'private', private 29 | )::text 30 | ); 31 | END; 32 | END; 33 | $$ 34 | LANGUAGE plpgsql; 35 | """) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /deploy/fly/prod.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for realtime-prod on 2023-08-08T09:07:09-07:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = "realtime-prod" 7 | primary_region = "sea" 8 | kill_signal = "SIGTERM" 9 | kill_timeout = "5s" 10 | 11 | [experimental] 12 | auto_rollback = true 13 | 14 | [deploy] 15 | release_command = "/app/bin/migrate" 16 | strategy = "rolling" 17 | 18 | [env] 19 | DNS_NODES = "realtime-prod.internal" 20 | ERL_CRASH_DUMP = "/data/erl_crash.dump" 21 | ERL_CRASH_DUMP_SECONDS = "30" 22 | 23 | 24 | [[services]] 25 | protocol = "tcp" 26 | internal_port = 4000 27 | processes = ["app"] 28 | 29 | [[services.ports]] 30 | port = 80 31 | handlers = ["http"] 32 | force_https = true 33 | 34 | [[services.ports]] 35 | port = 443 36 | handlers = ["tls", "http"] 37 | [services.concurrency] 38 | type = "connections" 39 | hard_limit = 100000 40 | soft_limit = 100000 41 | 42 | [[services.tcp_checks]] 43 | interval = "15s" 44 | timeout = "2s" 45 | grace_period = "30s" 46 | 47 | [[services.http_checks]] 48 | interval = "10s" 49 | timeout = "2s" 50 | grace_period = "5s" 51 | method = "get" 52 | path = "/" 53 | protocol = "http" 54 | tls_skip_verify = false 55 | -------------------------------------------------------------------------------- /test/support/rate_counter_helper.ex: -------------------------------------------------------------------------------- 1 | defmodule RateCounterHelper do 2 | alias Realtime.RateCounter 3 | 4 | @spec stop(term()) :: :ok 5 | def stop(tenant_id) do 6 | keys = 7 | Registry.select(Realtime.Registry.Unique, [ 8 | {{{:"$1", :_, {:_, :_, :"$2"}}, :"$3", :_}, [{:==, :"$1", RateCounter}, {:==, :"$2", tenant_id}], [:"$_"]} 9 | ]) 10 | 11 | Enum.each(keys, fn {{_, _, key}, {pid, _}} -> 12 | if Process.alive?(pid), do: GenServer.stop(pid) 13 | Realtime.GenCounter.delete(key) 14 | Cachex.del!(RateCounter, key) 15 | end) 16 | 17 | :ok 18 | end 19 | 20 | @spec tick!(RateCounter.Args.t()) :: RateCounter.t() 21 | def tick!(args) do 22 | [{pid, _}] = Registry.lookup(Realtime.Registry.Unique, {RateCounter, :rate_counter, args.id}) 23 | send(pid, :tick) 24 | {:ok, :sys.get_state(pid)} 25 | end 26 | 27 | def tick_tenant_rate_counters!(tenant_id) do 28 | keys = 29 | Registry.select(Realtime.Registry.Unique, [ 30 | {{{:"$1", :_, {:_, :_, :"$2"}}, :"$3", :_}, [{:==, :"$1", RateCounter}, {:==, :"$2", tenant_id}], [:"$_"]} 31 | ]) 32 | 33 | Enum.each(keys, fn {{_, _, _key}, {pid, _}} -> 34 | send(pid, :tick) 35 | # do a get_state to wait for the tick to be processed 36 | :sys.get_state(pid) 37 | end) 38 | 39 | :ok 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /assets/tailwind.config.js: -------------------------------------------------------------------------------- 1 | const plugin = require("tailwindcss/plugin") 2 | const colors = require('tailwindcss/colors') 3 | 4 | module.exports = { 5 | content: [ 6 | './js/**/*.js', 7 | '../lib/*_web.ex', 8 | '../lib/*_web/**/*.*ex', 9 | ], 10 | theme: { 11 | colors: { 12 | transparent: 'transparent', 13 | current: 'currentColor', 14 | black: colors.black, 15 | white: colors.white, 16 | gray: colors.gray, 17 | emerald: colors.emerald, 18 | indigo: colors.indigo, 19 | yellow: colors.yellow, 20 | green: colors.green 21 | }, 22 | fontFamily: { 23 | sans: ['custom-font', 'Helvetica Neue', 'Helvetica', 'Arial', 'sans-serif'], 24 | mono: ['Source Code Pro', 'Menlo', 'monospace'], 25 | }, 26 | }, 27 | plugins: [ 28 | require("@tailwindcss/forms"), 29 | require('@tailwindcss/typography'), 30 | plugin(({addVariant}) => addVariant("phx-no-feedback", [".phx-no-feedback&", ".phx-no-feedback &"])), 31 | plugin(({addVariant}) => addVariant("phx-click-loading", [".phx-click-loading&", ".phx-click-loading &"])), 32 | plugin(({addVariant}) => addVariant("phx-submit-loading", [".phx-submit-loading&", ".phx-submit-loading &"])), 33 | plugin(({addVariant}) => addVariant("phx-change-loading", [".phx-change-loading&", ".phx-change-loading &"])) 34 | ] 35 | }; 36 | -------------------------------------------------------------------------------- /test/realtime/monitoring/latency_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.LatencyTest do 2 | # async: false due to the usage of Clustered mode that interacts with this tests and breaks their expectations 3 | use Realtime.DataCase, async: false 4 | alias Realtime.Latency 5 | 6 | describe "ping/3" do 7 | setup do 8 | Node.stop() 9 | :ok 10 | end 11 | 12 | @tag skip: "Clustered tests creating flakiness, requires time to analyse" 13 | test "emulate a healthy remote node" do 14 | assert [{%Task{}, {:ok, %{response: {:ok, {:pong, "not_set"}}}}}] = Latency.ping() 15 | end 16 | 17 | @tag skip: "Clustered tests creating flakiness, requires time to analyse" 18 | test "emulate a slow but healthy remote node" do 19 | assert [{%Task{}, {:ok, %{response: {:ok, {:pong, "not_set"}}}}}] = Latency.ping(5_000, 10_000, 30_000) 20 | end 21 | 22 | @tag skip: "Clustered tests creating flakiness, requires time to analyse" 23 | test "emulate an unhealthy remote node" do 24 | assert [{%Task{}, {:ok, %{response: {:badrpc, :timeout}}}}] = Latency.ping(5_000, 1_000) 25 | end 26 | 27 | @tag skip: "Clustered tests creating flakiness, requires time to analyse" 28 | test "no response from our Task for a remote node at all" do 29 | assert [{%Task{}, nil}] = Latency.ping(10_000, 5_000, 2_000) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /test/support/cleanup.ex: -------------------------------------------------------------------------------- 1 | defmodule Cleanup do 2 | alias Realtime.Tenants.Connect 3 | def ensure_no_replication_slot(attempts \\ 5) 4 | def ensure_no_replication_slot(0), do: raise("Replication slot teardown failed") 5 | @table_name :"syn_registry_by_name_Elixir.Realtime.Tenants.Connect" 6 | 7 | def ensure_no_replication_slot(attempts) do 8 | {:ok, conn} = 9 | Postgrex.start_link( 10 | hostname: "localhost", 11 | port: 5433, 12 | database: "postgres", 13 | username: "supabase_admin", 14 | password: "postgres" 15 | ) 16 | 17 | # Stop lingering connections 18 | Enum.each(:ets.tab2list(@table_name), fn {tenant_id, _, _, _, _, _} -> 19 | Connect.shutdown(tenant_id) 20 | end) 21 | 22 | # Ensure no replication slots are active 23 | case Postgrex.query(conn, "SELECT active_pid, slot_name FROM pg_replication_slots", []) do 24 | {:ok, %{rows: []}} -> 25 | :ok 26 | 27 | {:ok, %{rows: rows}} -> 28 | Enum.each(rows, fn [pid, slot_name] -> 29 | Postgrex.query(conn, "select pg_terminate_backend($1) ", [pid]) 30 | Postgrex.query(conn, "select pg_drop_replication_slot($1)", [slot_name]) 31 | end) 32 | 33 | {:error, _} -> 34 | Process.sleep(1000) 35 | ensure_no_replication_slot(attempts - 1) 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /test/realtime/tenants/migrations_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.MigrationsTest do 2 | alias Realtime.Tenants.Cache 3 | # Can't use async: true because Cachex does not work well with Ecto Sandbox 4 | use Realtime.DataCase, async: false 5 | 6 | alias Realtime.Tenants.Migrations 7 | 8 | describe "run_migrations/1" do 9 | test "migrations for a given tenant only run once" do 10 | tenant = Containers.checkout_tenant() 11 | 12 | res = 13 | for _ <- 0..10 do 14 | Task.async(fn -> Migrations.run_migrations(tenant) end) 15 | end 16 | |> Task.await_many() 17 | |> Enum.uniq() 18 | 19 | assert [:ok] = res 20 | end 21 | 22 | test "migrations run if tenant has migrations_ran set to 0" do 23 | tenant = Containers.checkout_tenant() 24 | 25 | assert Migrations.run_migrations(tenant) == :ok 26 | # Sleeping waiting for Cache to be invalided 27 | Process.sleep(100) 28 | assert Cache.get_tenant_by_external_id(tenant.external_id).migrations_ran == Enum.count(Migrations.migrations()) 29 | end 30 | 31 | test "migrations do not run if tenant has migrations_ran at the count of all migrations" do 32 | tenant = tenant_fixture(%{migrations_ran: Enum.count(Migrations.migrations())}) 33 | assert Migrations.run_migrations(tenant) == :noop 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20250128220012_realtime_send_sets_topic_config.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RealtimeSendSetsTopicConfig do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # We missed the schema prefix of `realtime.` in the create table partition statement 6 | def change do 7 | execute(""" 8 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true ) RETURNS void 9 | AS $$ 10 | BEGIN 11 | BEGIN 12 | -- Set the topic configuration 13 | EXECUTE format('SET LOCAL realtime.topic TO %L', topic); 14 | 15 | -- Attempt to insert the message 16 | INSERT INTO realtime.messages (payload, event, topic, private, extension) 17 | VALUES (payload, event, topic, private, 'broadcast'); 18 | EXCEPTION 19 | WHEN OTHERS THEN 20 | -- Capture and notify the error 21 | PERFORM pg_notify( 22 | 'realtime:system', 23 | jsonb_build_object( 24 | 'error', SQLERRM, 25 | 'function', 'realtime.send', 26 | 'event', event, 27 | 'topic', topic, 28 | 'private', private 29 | )::text 30 | ); 31 | END; 32 | END; 33 | $$ 34 | LANGUAGE plpgsql; 35 | """) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/realtime/api/message.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Api.Message do 2 | @moduledoc """ 3 | Defines the Message schema to be used to check RLS authorization policies 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | 8 | @primary_key {:id, Ecto.UUID, autogenerate: true} 9 | @schema_prefix "realtime" 10 | 11 | @type t :: %__MODULE__{} 12 | @timestamps_opts [type: :naive_datetime_usec] 13 | schema "messages" do 14 | field(:topic, :string) 15 | field(:extension, Ecto.Enum, values: [:broadcast, :presence]) 16 | field(:payload, :map) 17 | field(:event, :string) 18 | field(:private, :boolean) 19 | 20 | timestamps() 21 | end 22 | 23 | def changeset(message, attrs) do 24 | message 25 | |> cast(attrs, [ 26 | :topic, 27 | :extension, 28 | :payload, 29 | :event, 30 | :private, 31 | :inserted_at, 32 | :updated_at 33 | ]) 34 | |> validate_required([:topic, :extension]) 35 | |> put_timestamp(:updated_at) 36 | |> maybe_put_timestamp(:inserted_at) 37 | end 38 | 39 | defp put_timestamp(changeset, field) do 40 | put_change(changeset, field, NaiveDateTime.utc_now(:microsecond)) 41 | end 42 | 43 | defp maybe_put_timestamp(changeset, field) do 44 | case get_field(changeset, field) do 45 | nil -> put_timestamp(changeset, field) 46 | _ -> changeset 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211202204605_update_realtime_build_prepared_statement_sql_function_for_compatibility_with_all_types.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.UpdateRealtimeBuildPreparedStatementSqlFunctionForCompatibilityWithAllTypes do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute("create or replace function realtime.build_prepared_statement_sql( 8 | prepared_statement_name text, 9 | entity regclass, 10 | columns realtime.wal_column[] 11 | ) 12 | returns text 13 | language sql 14 | as $$ 15 | /* 16 | Builds a sql string that, if executed, creates a prepared statement to 17 | tests retrive a row from *entity* by its primary key columns. 18 | 19 | Example 20 | select realtime.build_prepared_statment_sql('public.notes', '{\"id\"}'::text[], '{\"bigint\"}'::text[]) 21 | */ 22 | select 23 | 'prepare ' || prepared_statement_name || ' as 24 | select 25 | exists( 26 | select 27 | 1 28 | from 29 | ' || entity || ' 30 | where 31 | ' || string_agg(quote_ident(pkc.name) || '=' || quote_nullable(pkc.value #>> '{}') , ' and ') || ' 32 | )' 33 | from 34 | unnest(columns) pkc 35 | where 36 | pkc.is_pkey 37 | group by 38 | entity 39 | $$;") 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/realtime_channel/assign.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.RealtimeChannel.Assigns do 2 | @moduledoc """ 3 | Assigns for RealtimeChannel 4 | """ 5 | 6 | defstruct [ 7 | :tenant, 8 | :log_level, 9 | :rate_counter, 10 | :limits, 11 | :tenant_topic, 12 | :pg_sub_ref, 13 | :pg_change_params, 14 | :postgres_extension, 15 | :claims, 16 | :jwt_secret, 17 | :jwt_jwks, 18 | :tenant_token, 19 | :access_token, 20 | :postgres_cdc_module, 21 | :channel_name, 22 | :headers 23 | ] 24 | 25 | @type t :: %__MODULE__{ 26 | tenant: String.t(), 27 | log_level: Logger.level(), 28 | rate_counter: Realtime.RateCounter.t(), 29 | limits: %{ 30 | max_events_per_second: integer(), 31 | max_concurrent_users: integer(), 32 | max_bytes_per_second: integer(), 33 | max_channels_per_client: integer(), 34 | max_joins_per_second: integer() 35 | }, 36 | tenant_topic: String.t(), 37 | pg_sub_ref: reference() | nil, 38 | pg_change_params: map(), 39 | postgres_extension: map(), 40 | claims: map(), 41 | jwt_secret: String.t(), 42 | jwt_jwks: map(), 43 | tenant_token: String.t(), 44 | access_token: String.t(), 45 | channel_name: String.t() 46 | } 47 | end 48 | -------------------------------------------------------------------------------- /lib/realtime/tenants/rebalancer.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Rebalancer do 2 | @moduledoc """ 3 | Responsible to tell if the executing node is in the correct region for this tenant 4 | """ 5 | 6 | alias Realtime.Api.Tenant 7 | 8 | @spec check(MapSet.t(node), MapSet.t(node), binary) :: :ok | {:error, :wrong_region} 9 | def check(previous_nodes_set, current_nodes_set, tenant_id) 10 | when is_struct(previous_nodes_set, MapSet) and is_struct(current_nodes_set, MapSet) and is_binary(tenant_id) do 11 | # Check if the current nodes set is equal to the previous nodes set 12 | # If they are equal it means that the cluster is relatively stable 13 | # We can check now if this Connect process is in the correct region 14 | if MapSet.equal?(current_nodes_set, previous_nodes_set) do 15 | with %Tenant{} = tenant <- Realtime.Tenants.Cache.get_tenant_by_external_id(tenant_id), 16 | {:ok, _node, expected_region} <- Realtime.Nodes.get_node_for_tenant(tenant), 17 | region when is_binary(region) <- Application.get_env(:realtime, :region) do 18 | if region == expected_region do 19 | :ok 20 | else 21 | {:error, :wrong_region} 22 | end 23 | else 24 | _ -> :ok 25 | end 26 | else 27 | # Nodes have changed, we can assume that the cluster is not stable enough to rebalance 28 | :ok 29 | end 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20241220035512_fix_send_function_partition_creation.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.FixSendFunctionPartitionCreation do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # We missed the schema prefix of `realtime.` in the create table partition statement 6 | def change do 7 | execute(""" 8 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true) 9 | RETURNS void 10 | AS $$ 11 | DECLARE 12 | partition_name text; 13 | partition_start timestamp; 14 | partition_end timestamp; 15 | BEGIN 16 | partition_start := date_trunc('day', NOW()); 17 | partition_end := partition_start + interval '1 day'; 18 | partition_name := 'messages_' || to_char(partition_start, 'YYYY_MM_DD'); 19 | 20 | BEGIN 21 | EXECUTE format( 22 | 'CREATE TABLE IF NOT EXISTS realtime.%I PARTITION OF realtime.messages FOR VALUES FROM (%L) TO (%L)', 23 | partition_name, 24 | partition_start, 25 | partition_end 26 | ); 27 | EXCEPTION WHEN duplicate_table THEN 28 | -- Ignore; table already exists 29 | END; 30 | 31 | INSERT INTO realtime.messages (payload, event, topic, private, extension) 32 | VALUES (payload, event, topic, private, 'broadcast'); 33 | END; 34 | $$ 35 | LANGUAGE plpgsql; 36 | """) 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /test/realtime_web/controllers/live_dasboard_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.LiveDashboardTest do 2 | use RealtimeWeb.ConnCase 3 | import Generators 4 | 5 | describe "live_dashboard" do 6 | setup do 7 | user = random_string() 8 | password = random_string() 9 | 10 | System.put_env("DASHBOARD_USER", user) 11 | System.put_env("DASHBOARD_PASSWORD", password) 12 | 13 | on_exit(fn -> 14 | System.delete_env("DASHBOARD_USER") 15 | System.delete_env("DASHBOARD_PASSWORD") 16 | end) 17 | 18 | %{user: user, password: password} 19 | end 20 | 21 | test "with credetentials renders view", %{ 22 | conn: conn, 23 | user: user, 24 | password: password 25 | } do 26 | path = 27 | conn 28 | |> using_basic_auth(user, password) 29 | |> get("/admin/dashboard") 30 | |> redirected_to(302) 31 | 32 | conn = conn |> recycle() |> using_basic_auth(user, password) |> get(path) 33 | 34 | assert html_response(conn, 200) =~ "Dashboard" 35 | end 36 | 37 | test "without credetentials returns 401", %{conn: conn} do 38 | assert conn |> get("/admin/dashboard") |> response(401) 39 | end 40 | end 41 | 42 | defp using_basic_auth(conn, username, password) do 43 | header_content = "Basic " <> Base.encode64("#{username}:#{password}") 44 | put_req_header(conn, "authorization", header_content) 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211116050929_create_realtime_quote_wal2json_function.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeQuoteWal2jsonFunction do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute("create function realtime.quote_wal2json(entity regclass) 8 | returns text 9 | language sql 10 | immutable 11 | strict 12 | as $$ 13 | select 14 | ( 15 | select string_agg('\' || ch,'') 16 | from unnest(string_to_array(nsp.nspname::text, null)) with ordinality x(ch, idx) 17 | where 18 | not (x.idx = 1 and x.ch = '\"') 19 | and not ( 20 | x.idx = array_length(string_to_array(nsp.nspname::text, null), 1) 21 | and x.ch = '\"' 22 | ) 23 | ) 24 | || '.' 25 | || ( 26 | select string_agg('\' || ch,'') 27 | from unnest(string_to_array(pc.relname::text, null)) with ordinality x(ch, idx) 28 | where 29 | not (x.idx = 1 and x.ch = '\"') 30 | and not ( 31 | x.idx = array_length(string_to_array(nsp.nspname::text, null), 1) 32 | and x.ch = '\"' 33 | ) 34 | ) 35 | from 36 | pg_class pc 37 | join pg_namespace nsp 38 | on pc.relnamespace = nsp.oid 39 | where 40 | pc.oid = entity 41 | $$;") 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # This file excludes paths from the Docker build context. 2 | # 3 | # By default, Docker's build context includes all files (and folders) in the 4 | # current directory. Even if a file isn't copied into the container it is still sent to 5 | # the Docker daemon. 6 | # 7 | # There are multiple reasons to exclude files from the build context: 8 | # 9 | # 1. Prevent nested folders from being copied into the container (ex: exclude 10 | # /assets/node_modules when copying /assets) 11 | # 2. Reduce the size of the build context and improve build time (ex. /build, /deps, /doc) 12 | # 3. Avoid sending files containing sensitive information 13 | # 14 | # More information on using .dockerignore is available here: 15 | # https://docs.docker.com/engine/reference/builder/#dockerignore-file 16 | 17 | .dockerignore 18 | 19 | # Ignore git, but keep git HEAD and refs to access current commit hash if needed: 20 | # 21 | # $ cat .git/HEAD | awk '{print ".git/"$2}' | xargs cat 22 | # d0b8727759e1e0e7aa3d41707d12376e373d5ecc 23 | .git 24 | !.git/HEAD 25 | !.git/refs 26 | 27 | # Common development/test artifacts 28 | /cover/ 29 | /doc/ 30 | /test/ 31 | /tmp/ 32 | .elixir_ls 33 | 34 | # Mix artifacts 35 | /_build/ 36 | /deps/ 37 | *.ez 38 | 39 | # Generated on crash by the VM 40 | erl_crash.dump 41 | 42 | # Static artifacts - These should be fetched and built inside the Docker image 43 | /assets/node_modules/ 44 | /priv/static/assets/ 45 | /priv/static/cache_manifest.json 46 | -------------------------------------------------------------------------------- /lib/realtime_web/controllers/broadcast_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.BroadcastController do 2 | use RealtimeWeb, :controller 3 | use OpenApiSpex.ControllerSpecs 4 | require Logger 5 | 6 | alias Realtime.Tenants.BatchBroadcast 7 | alias RealtimeWeb.OpenApiSchemas.EmptyResponse 8 | alias RealtimeWeb.OpenApiSchemas.TenantBatchParams 9 | alias RealtimeWeb.OpenApiSchemas.TooManyRequestsResponse 10 | alias RealtimeWeb.OpenApiSchemas.UnprocessableEntityResponse 11 | 12 | action_fallback(RealtimeWeb.FallbackController) 13 | 14 | operation(:broadcast, 15 | summary: "Broadcasts a batch of messages", 16 | parameters: [ 17 | token: [ 18 | in: :header, 19 | name: "Authorization", 20 | schema: %OpenApiSpex.Schema{type: :string}, 21 | required: true, 22 | example: 23 | "Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpYXQiOjE2ODAxNjIxNTR9.U9orU6YYqXAtpF8uAiw6MS553tm4XxRzxOhz2IwDhpY" 24 | ] 25 | ], 26 | request_body: TenantBatchParams.params(), 27 | responses: %{ 28 | 202 => EmptyResponse.response(), 29 | 403 => EmptyResponse.response(), 30 | 422 => UnprocessableEntityResponse.response(), 31 | 429 => TooManyRequestsResponse.response() 32 | } 33 | ) 34 | 35 | def broadcast(%{assigns: %{tenant: tenant}} = conn, attrs) do 36 | with :ok <- BatchBroadcast.broadcast(conn, tenant, attrs) do 37 | send_resp(conn, :accepted, "") 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /deploy/fly/staging.toml: -------------------------------------------------------------------------------- 1 | # fly.toml app configuration file generated for realtime-staging on 2023-06-27T07:39:20-07:00 2 | # 3 | # See https://fly.io/docs/reference/configuration/ for information about how to use this file. 4 | # 5 | 6 | app = "realtime-staging" 7 | primary_region = "lhr" 8 | kill_signal = "SIGTERM" 9 | kill_timeout = "5s" 10 | 11 | [experimental] 12 | auto_rollback = true 13 | 14 | [deploy] 15 | release_command = "/app/bin/migrate" 16 | strategy = "rolling" 17 | 18 | [env] 19 | DNS_NODES = "realtime-staging.internal" 20 | ERL_CRASH_DUMP = "/data/erl_crash.dump" 21 | ERL_CRASH_DUMP_SECONDS = "30" 22 | 23 | [[mounts]] 24 | source = "data_vol_machines" 25 | destination = "/data" 26 | processes = ["app"] 27 | 28 | [[services]] 29 | protocol = "tcp" 30 | internal_port = 4000 31 | processes = ["app"] 32 | 33 | [[services.ports]] 34 | port = 80 35 | handlers = ["http"] 36 | force_https = true 37 | 38 | [[services.ports]] 39 | port = 443 40 | handlers = ["tls", "http"] 41 | [services.concurrency] 42 | type = "connections" 43 | hard_limit = 16384 44 | soft_limit = 16384 45 | 46 | [[services.tcp_checks]] 47 | interval = "15s" 48 | timeout = "2s" 49 | grace_period = "30s" 50 | restart_limit = 6 51 | 52 | [[services.http_checks]] 53 | interval = "10s" 54 | timeout = "2s" 55 | grace_period = "5s" 56 | restart_limit = 0 57 | method = "get" 58 | path = "/" 59 | protocol = "http" 60 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/tenant_rate_limiters.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.TenantRateLimiters do 2 | @moduledoc """ 3 | Rate limiters for tenants. 4 | """ 5 | require Logger 6 | alias Realtime.UsersCounter 7 | alias Realtime.Tenants 8 | alias Realtime.RateCounter 9 | alias Realtime.Api.Tenant 10 | 11 | @spec check_tenant(Realtime.Api.Tenant.t()) :: :ok | {:error, :too_many_connections | :too_many_joins} 12 | def check_tenant(tenant) do 13 | with :ok <- max_concurrent_users_check(tenant) do 14 | max_joins_per_second_check(tenant) 15 | end 16 | end 17 | 18 | defp max_concurrent_users_check(%Tenant{max_concurrent_users: max_conn_users, external_id: external_id}) do 19 | total_conn_users = UsersCounter.tenant_users(external_id) 20 | 21 | if total_conn_users < max_conn_users, 22 | do: :ok, 23 | else: {:error, :too_many_connections} 24 | end 25 | 26 | defp max_joins_per_second_check(%Tenant{max_joins_per_second: max_joins_per_second} = tenant) do 27 | rate_args = Tenants.joins_per_second_rate(tenant.external_id, max_joins_per_second) 28 | 29 | RateCounter.new(rate_args) 30 | 31 | case RateCounter.get(rate_args) do 32 | {:ok, %{limit: %{triggered: false}}} -> 33 | :ok 34 | 35 | {:ok, %{limit: %{triggered: true}}} -> 36 | {:error, :too_many_joins} 37 | 38 | error -> 39 | Logger.error("UnknownErrorOnCounter: #{inspect(error)}") 40 | {:error, error} 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/auth/channels_authorization.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ChannelsAuthorization do 2 | @moduledoc """ 3 | Check connection is authorized to access channel 4 | """ 5 | require Logger 6 | 7 | @doc """ 8 | Authorize connection to access channel 9 | """ 10 | @spec authorize(binary(), binary(), binary() | nil) :: 11 | {:ok, map()} | {:error, any()} | {:error, :expired_token, String.t()} 12 | def authorize(token, jwt_secret, jwt_jwks) when is_binary(token) do 13 | token 14 | |> clean_token() 15 | |> RealtimeWeb.JwtVerification.verify(jwt_secret, jwt_jwks) 16 | end 17 | 18 | def authorize(_token, _jwt_secret, _jwt_jwks), do: {:error, :invalid_token} 19 | 20 | def authorize_conn(token, jwt_secret, jwt_jwks) do 21 | case authorize(token, jwt_secret, jwt_jwks) do 22 | {:ok, claims} -> 23 | required = ["role", "exp"] 24 | claims_keys = Map.keys(claims) 25 | 26 | if Enum.all?(required, &(&1 in claims_keys)), 27 | do: {:ok, claims}, 28 | else: {:error, :missing_claims} 29 | 30 | {:error, [message: validation_timer, claim: "exp", claim_val: claim_val]} when is_integer(validation_timer) -> 31 | msg = "Token has expired #{validation_timer - claim_val} seconds ago" 32 | {:error, :expired_token, msg} 33 | 34 | {:error, reason} -> 35 | {:error, reason} 36 | end 37 | end 38 | 39 | defp clean_token(token), do: Regex.replace(~r/\s|\n/, URI.decode(token), "") 40 | end 41 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20220712093339_recreate_realtime_build_prepared_statement_sql_function.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RecreateRealtimeBuildPreparedStatementSqlFunction do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(" 8 | create or replace function realtime.build_prepared_statement_sql( 9 | prepared_statement_name text, 10 | entity regclass, 11 | columns realtime.wal_column[] 12 | ) 13 | returns text 14 | language sql 15 | as $$ 16 | /* 17 | Builds a sql string that, if executed, creates a prepared statement to 18 | tests retrive a row from *entity* by its primary key columns. 19 | Example 20 | select realtime.build_prepared_statement_sql('public.notes', '{\"id\"}'::text[], '{\"bigint\"}'::text[]) 21 | */ 22 | select 23 | 'prepare ' || prepared_statement_name || ' as 24 | select 25 | exists( 26 | select 27 | 1 28 | from 29 | ' || entity || ' 30 | where 31 | ' || string_agg(quote_ident(pkc.name) || '=' || quote_nullable(pkc.value #>> '{}') , ' and ') || ' 32 | )' 33 | from 34 | unnest(columns) pkc 35 | where 36 | pkc.is_pkey 37 | group by 38 | entity 39 | $$; 40 | ") 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240401105812_create_realtime_admin_and_move_ownership.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeAdminAndMoveOwnership do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | DO 9 | $do$ 10 | BEGIN 11 | IF EXISTS ( 12 | SELECT FROM pg_catalog.pg_roles 13 | WHERE rolname = 'supabase_realtime_admin') THEN 14 | 15 | RAISE NOTICE 'Role "supabase_realtime_admin" already exists. Skipping.'; 16 | ELSE 17 | CREATE ROLE supabase_realtime_admin WITH NOINHERIT NOLOGIN NOREPLICATION; 18 | END IF; 19 | END 20 | $do$; 21 | """) 22 | 23 | execute("GRANT ALL PRIVILEGES ON SCHEMA realtime TO supabase_realtime_admin") 24 | execute("GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA realtime TO supabase_realtime_admin") 25 | execute("GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA realtime TO supabase_realtime_admin") 26 | execute("GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA realtime TO supabase_realtime_admin") 27 | 28 | execute("ALTER table realtime.channels OWNER to supabase_realtime_admin") 29 | execute("ALTER table realtime.broadcasts OWNER to supabase_realtime_admin") 30 | execute("ALTER table realtime.presences OWNER TO supabase_realtime_admin") 31 | execute("ALTER function realtime.channel_name() owner to supabase_realtime_admin") 32 | 33 | execute("GRANT supabase_realtime_admin TO postgres") 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20251103001201_broadcast_send_include_payload_id.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.BroadcastSendIncludePayloadId do 2 | @moduledoc false 3 | use Ecto.Migration 4 | 5 | # Include ID in the payload if not defined 6 | def change do 7 | execute(""" 8 | CREATE OR REPLACE FUNCTION realtime.send(payload jsonb, event text, topic text, private boolean DEFAULT true ) RETURNS void 9 | AS $$ 10 | DECLARE 11 | generated_id uuid; 12 | final_payload jsonb; 13 | BEGIN 14 | BEGIN 15 | -- Generate a new UUID for the id 16 | generated_id := gen_random_uuid(); 17 | 18 | -- Check if payload has an 'id' key, if not, add the generated UUID 19 | IF payload ? 'id' THEN 20 | final_payload := payload; 21 | ELSE 22 | final_payload := jsonb_set(payload, '{id}', to_jsonb(generated_id)); 23 | END IF; 24 | 25 | -- Set the topic configuration 26 | EXECUTE format('SET LOCAL realtime.topic TO %L', topic); 27 | 28 | -- Attempt to insert the message 29 | INSERT INTO realtime.messages (id, payload, event, topic, private, extension) 30 | VALUES (generated_id, final_payload, event, topic, private, 'broadcast'); 31 | EXCEPTION 32 | WHEN OTHERS THEN 33 | -- Capture and notify the error 34 | RAISE WARNING 'ErrorSendingBroadcastMessage: %', SQLERRM; 35 | END; 36 | END; 37 | $$ 38 | LANGUAGE plpgsql; 39 | """) 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/realtime_web/live/status_live/index.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.StatusLive.Index do 2 | use RealtimeWeb, :live_view 3 | 4 | alias Realtime.Latency.Payload 5 | alias Realtime.Nodes 6 | alias RealtimeWeb.Endpoint 7 | 8 | @impl true 9 | def mount(_params, _session, socket) do 10 | if connected?(socket), do: Endpoint.subscribe("admin:cluster") 11 | 12 | socket = 13 | socket 14 | |> assign(nodes: Enum.count(all_nodes())) 15 | |> stream(:pings, default_pings()) 16 | 17 | {:ok, socket} 18 | end 19 | 20 | @impl true 21 | def handle_params(params, _url, socket) do 22 | {:noreply, apply_action(socket, socket.assigns.live_action, params)} 23 | end 24 | 25 | @impl true 26 | def handle_info(%Phoenix.Socket.Broadcast{payload: %Payload{} = payload}, socket) do 27 | pair = pair_id(payload.from_node, payload.node) 28 | 29 | {:noreply, stream(socket, :pings, [%{id: pair, payload: payload}])} 30 | end 31 | 32 | defp apply_action(socket, :index, _params) do 33 | socket 34 | |> assign(:page_title, "Realtime Status") 35 | end 36 | 37 | defp all_nodes do 38 | [Node.self() | Node.list()] |> Enum.map(&Nodes.short_node_id_from_name/1) 39 | end 40 | 41 | defp default_pings do 42 | for n <- all_nodes(), f <- all_nodes() do 43 | pair = pair_id(f, n) 44 | 45 | %{id: pair, payload: %Payload{from_node: f, latency: "Loading...", node: n, timestamp: "Loading..."}} 46 | end 47 | end 48 | 49 | defp pair_id(from, to) do 50 | from <> "_" <> to 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/realtime_web/live/tenants_live/index.html.heex: -------------------------------------------------------------------------------- 1 | <.h1>Supabase Realtime: Multiplayer Edition 2 | <.h2>Tenants 3 |

Listing all Supabase Realtime tenants.

4 | 5 |
6 |
7 | <.form :let={f} for={@filter_changeset} phx-change="validate" phx-submit="filter_submit"> 8 |
9 | <.select form={f} field={:order_by} list={@sort_fields} selected={:inserted_at} /> 10 |
11 |
12 | <.select form={f} field={:order} list={[:desc, :asc]} selected={:desc} /> 13 |
14 |
15 | <.text_input form={f} field={:search} opts={[phx_change: "validate", placeholder: "tenant"]} /> 16 |
17 |
18 | <.text_input form={f} field={:limit} opts={[phx_change: "validate", placeholder: "limit"]} /> 19 |
20 | 21 |
22 | 23 |
24 |
    25 | <%= for t <- @tenants do %> 26 |
  • 27 | <%= t.external_id %> 28 |
    29 | <%= for {k, v} <- Map.take(t, [:max_events_per_second, :max_concurrent_users, :max_bytes_per_second]) do %> 30 |
    <%= k %>: <%= v %>
    31 | <% end %> 32 |
    33 |
    • 34 | <% end %> 35 |
36 |
37 |
38 | -------------------------------------------------------------------------------- /test/support/containers/container.ex: -------------------------------------------------------------------------------- 1 | defmodule Containers.Container do 2 | use GenServer 3 | 4 | def start_link(args \\ [], opts \\ []) do 5 | GenServer.start_link(__MODULE__, args, opts) 6 | end 7 | 8 | def port(pid), do: GenServer.call(pid, :port, 15_000) 9 | def name(pid), do: GenServer.call(pid, :name, 15_000) 10 | 11 | @impl true 12 | def handle_call(:port, _from, state) do 13 | {:reply, state[:port], state} 14 | end 15 | 16 | @impl true 17 | def handle_call(:name, _from, state) do 18 | {:reply, state[:name], state} 19 | end 20 | 21 | @impl true 22 | def init(_args) do 23 | {:ok, %{}, {:continue, :start_container}} 24 | end 25 | 26 | @impl true 27 | def handle_continue(:start_container, _state) do 28 | {:ok, name, port} = Containers.start_container() 29 | 30 | {:noreply, %{name: name, port: port}, {:continue, :check_container_ready}} 31 | end 32 | 33 | @impl true 34 | def handle_continue(:check_container_ready, state) do 35 | check_container_ready(state[:name]) 36 | {:noreply, state} 37 | end 38 | 39 | defp check_container_ready(name, attempts \\ 100) 40 | defp check_container_ready(name, 0), do: raise("Container #{name} is not ready") 41 | 42 | defp check_container_ready(name, attempts) do 43 | case System.cmd("docker", ["exec", name, "pg_isready", "-p", "5432", "-h", "localhost"]) do 44 | {_, 0} -> 45 | :ok 46 | 47 | {_, _} -> 48 | Process.sleep(250) 49 | check_container_ready(name, attempts - 1) 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/realtime_web/socket/user_broadcast.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Socket.UserBroadcast do 2 | @moduledoc """ 3 | Defines a message sent from pubsub to channels and vice-versa. 4 | 5 | The message format requires the following keys: 6 | 7 | * `:topic` - The string topic or topic:subtopic pair namespace, for example "messages", "messages:123" 8 | * `:user_event`- The string user event name, for example "my-event" 9 | * `:user_payload_encoding`- :json or :binary 10 | * `:user_payload` - The actual message payload 11 | 12 | Optionally metadata which is a map to be JSON encoded 13 | """ 14 | 15 | alias Phoenix.Socket.Broadcast 16 | 17 | @type t :: %__MODULE__{} 18 | defstruct topic: nil, user_event: nil, user_payload: nil, user_payload_encoding: nil, metadata: nil 19 | 20 | @spec convert_to_json_broadcast(t) :: {:ok, Broadcast.t()} | {:error, String.t()} 21 | def convert_to_json_broadcast(%__MODULE__{user_payload_encoding: :json} = user_broadcast) do 22 | payload = %{ 23 | "event" => user_broadcast.user_event, 24 | "payload" => Jason.Fragment.new(user_broadcast.user_payload), 25 | "type" => "broadcast" 26 | } 27 | 28 | payload = 29 | if user_broadcast.metadata do 30 | Map.put(payload, "meta", user_broadcast.metadata) 31 | else 32 | payload 33 | end 34 | 35 | {:ok, %Broadcast{event: "broadcast", payload: payload, topic: user_broadcast.topic}} 36 | end 37 | 38 | def convert_to_json_broadcast(%__MODULE__{}), do: {:error, "User payload encoding is not JSON"} 39 | end 40 | -------------------------------------------------------------------------------- /test/support/conn_case.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ConnCase do 2 | @moduledoc """ 3 | This module defines the test case to be used by 4 | tests that require setting up a connection. 5 | 6 | Such tests rely on `Phoenix.ConnTest` and also 7 | import other functionality to make it easier 8 | to build common data structures and query the data layer. 9 | 10 | Finally, if the test case interacts with the database, 11 | we enable the SQL sandbox, so changes done to the database 12 | are reverted at the end of every test. If you are using 13 | PostgreSQL, you can even run database tests asynchronously 14 | by setting `use RealtimeWeb.ConnCase, async: true`, although 15 | this option is not recommended for other databases. 16 | """ 17 | 18 | use ExUnit.CaseTemplate 19 | alias Ecto.Adapters.SQL.Sandbox 20 | 21 | using do 22 | quote do 23 | # Import conveniences for testing with connections 24 | import Generators 25 | import TenantConnection 26 | import Phoenix.ConnTest 27 | import Plug.Conn 28 | import Realtime.DataCase 29 | 30 | alias RealtimeWeb.Router.Helpers, as: Routes 31 | 32 | use RealtimeWeb, :verified_routes 33 | use Realtime.Tracing 34 | 35 | # The default endpoint for testing 36 | @endpoint RealtimeWeb.Endpoint 37 | end 38 | end 39 | 40 | setup tags do 41 | pid = Sandbox.start_owner!(Realtime.Repo, shared: not tags[:async]) 42 | on_exit(fn -> Sandbox.stop_owner(pid) end) 43 | 44 | {:ok, conn: Phoenix.ConnTest.build_conn()} 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/realtime_web/channels/auth/channels_authorization_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ChannelsAuthorizationTest do 2 | use ExUnit.Case, async: true 3 | 4 | use Mimic 5 | 6 | import Generators 7 | 8 | alias RealtimeWeb.ChannelsAuthorization 9 | alias RealtimeWeb.JwtVerification 10 | 11 | @secret "" 12 | describe "authorize_conn/3" do 13 | test "when token is authorized" do 14 | input_token = "\n token %20 1 %20 2 %20 3 " 15 | expected_token = "token123" 16 | 17 | expect(JwtVerification, :verify, 1, fn token, @secret, _jwks -> 18 | assert token == expected_token 19 | {:ok, %{}} 20 | end) 21 | 22 | assert {:ok, %{}} = ChannelsAuthorization.authorize(input_token, @secret, nil) 23 | end 24 | 25 | test "when token is unauthorized" do 26 | expect(JwtVerification, :verify, 1, fn _token, @secret, _jwks -> :error end) 27 | assert :error = ChannelsAuthorization.authorize("bad_token", @secret, nil) 28 | end 29 | 30 | test "when token is not a jwt token" do 31 | assert {:error, :token_malformed} = ChannelsAuthorization.authorize("bad_token", @secret, nil) 32 | end 33 | 34 | test "when token is not a string" do 35 | assert {:error, :invalid_token} = ChannelsAuthorization.authorize([], @secret, nil) 36 | end 37 | 38 | test "authorize_conn/3 fails when has missing headers" do 39 | jwt = generate_jwt_token(@secret, %{}) 40 | 41 | assert {:error, :missing_claims} = 42 | ChannelsAuthorization.authorize_conn(jwt, @secret, nil) 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/realtime_web/plugs/assign_tenant.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Plugs.AssignTenant do 2 | @moduledoc """ 3 | Picks out the tenant from the request and assigns it in the conn. 4 | """ 5 | import Plug.Conn 6 | import Phoenix.Controller, only: [json: 2] 7 | 8 | require Logger 9 | 10 | alias Realtime.Api 11 | alias Realtime.Api.Tenant 12 | alias Realtime.Database 13 | alias Realtime.GenCounter 14 | alias Realtime.RateCounter 15 | alias Realtime.Tenants 16 | 17 | def init(opts) do 18 | opts 19 | end 20 | 21 | def call(%Plug.Conn{host: host} = conn, _opts) do 22 | with {:ok, external_id} <- Database.get_external_id(host), 23 | %Tenant{} = tenant <- Api.get_tenant_by_external_id(external_id, use_replica?: true) do 24 | Logger.metadata(external_id: external_id, project: external_id) 25 | OpenTelemetry.Tracer.set_attributes(external_id: external_id) 26 | 27 | tenant = 28 | tenant 29 | |> tap(&initialize_counters/1) 30 | |> tap(&GenCounter.add(Tenants.requests_per_second_key(&1))) 31 | |> Api.preload_counters() 32 | 33 | assign(conn, :tenant, tenant) 34 | else 35 | nil -> error_response(conn, "Tenant not found in database") 36 | end 37 | end 38 | 39 | defp error_response(conn, message) do 40 | conn 41 | |> put_status(401) 42 | |> json(%{message: message}) 43 | |> halt() 44 | end 45 | 46 | defp initialize_counters(tenant) do 47 | RateCounter.new(Tenants.requests_per_second_rate(tenant)) 48 | RateCounter.new(Tenants.events_per_second_rate(tenant)) 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/realtime/tenants/cache.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Cache do 2 | @moduledoc """ 3 | Cache for Tenants. 4 | """ 5 | require Cachex.Spec 6 | require Logger 7 | 8 | alias Realtime.Tenants 9 | 10 | def child_spec(_) do 11 | tenant_cache_expiration = Application.get_env(:realtime, :tenant_cache_expiration) 12 | 13 | %{ 14 | id: __MODULE__, 15 | start: {Cachex, :start_link, [__MODULE__, [expiration: Cachex.Spec.expiration(default: tenant_cache_expiration)]]} 16 | } 17 | end 18 | 19 | def get_tenant_by_external_id(keyword), do: apply_repo_fun(__ENV__.function, [keyword]) 20 | 21 | @doc """ 22 | Invalidates the cache for a tenant in the local node 23 | """ 24 | def invalidate_tenant_cache(tenant_id), do: Cachex.del(__MODULE__, {{:get_tenant_by_external_id, 1}, [tenant_id]}) 25 | 26 | @doc """ 27 | Broadcasts a message to invalidate the tenant cache to all connected nodes 28 | """ 29 | @spec distributed_invalidate_tenant_cache(String.t()) :: boolean() 30 | def distributed_invalidate_tenant_cache(tenant_id) when is_binary(tenant_id) do 31 | nodes = [Node.self() | Node.list()] 32 | results = :erpc.multicall(nodes, __MODULE__, :invalidate_tenant_cache, [tenant_id], 1000) 33 | 34 | results 35 | |> Enum.map(fn 36 | {res, _} -> 37 | res 38 | 39 | exception -> 40 | Logger.error("Failed to invalidate tenant cache: #{inspect(exception)}") 41 | :error 42 | end) 43 | |> Enum.all?(&(&1 == :ok)) 44 | end 45 | 46 | defp apply_repo_fun(arg1, arg2), do: Realtime.ContextCache.apply_fun(Tenants, arg1, arg2) 47 | end 48 | -------------------------------------------------------------------------------- /lib/realtime/gen_counter/gen_counter.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.GenCounter do 2 | @moduledoc """ 3 | Process holds an ETS table where each row is a key and a counter 4 | """ 5 | 6 | use GenServer 7 | 8 | @name __MODULE__ 9 | @table :gen_counter 10 | 11 | @spec start_link(any) :: GenServer.on_start() 12 | def start_link(_), do: GenServer.start_link(__MODULE__, :ok, name: @name) 13 | 14 | @spec add(term, integer) :: integer 15 | def add(term), do: add(term, 1) 16 | 17 | def add(term, count), do: :ets.update_counter(@table, term, count, {term, 0}) 18 | 19 | @spec get(term) :: integer 20 | def get(term) do 21 | case :ets.lookup(@table, term) do 22 | [{^term, value}] -> value 23 | [] -> 0 24 | end 25 | end 26 | 27 | @doc "Reset counter to 0 and return previous value" 28 | @spec reset(term) :: integer 29 | def reset(term) do 30 | # We might lose some updates between lookup and the update 31 | case :ets.lookup(@table, term) do 32 | [{^term, 0}] -> 33 | 0 34 | 35 | [{^term, previous}] -> 36 | :ets.update_element(@table, term, {2, 0}, {term, 0}) 37 | previous 38 | 39 | [] -> 40 | 0 41 | end 42 | end 43 | 44 | @spec delete(term) :: :ok 45 | def delete(term) do 46 | :ets.delete(@table, term) 47 | :ok 48 | end 49 | 50 | @impl true 51 | def init(_) do 52 | table = 53 | :ets.new(@table, [ 54 | :set, 55 | :public, 56 | :named_table, 57 | {:decentralized_counters, true}, 58 | {:write_concurrency, :auto} 59 | ]) 60 | 61 | {:ok, table} 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /test/support/tenant_connection.ex: -------------------------------------------------------------------------------- 1 | defmodule TenantConnection do 2 | @moduledoc """ 3 | Boilerplate code to handle Realtime.Tenants.Connect during tests 4 | """ 5 | alias Realtime.Api.Message 6 | alias Realtime.Database 7 | alias Realtime.Tenants.Repo 8 | alias Realtime.Tenants.Connect 9 | alias RealtimeWeb.Endpoint 10 | 11 | def create_message(attrs, conn, opts \\ [mode: :savepoint]) do 12 | message = Message.changeset(%Message{}, attrs) 13 | 14 | {:ok, result} = 15 | Database.transaction(conn, fn transaction_conn -> 16 | with {:ok, %Message{} = message} <- Repo.insert(transaction_conn, message, Message, opts) do 17 | message 18 | end 19 | end) 20 | 21 | case result do 22 | %Ecto.Changeset{valid?: false} = error -> {:error, error} 23 | {:error, error} -> {:error, error} 24 | result -> {:ok, result} 25 | end 26 | end 27 | 28 | def ensure_connect_down(tenant_id) do 29 | # Using syn and not a normal Process.monitor because we want to ensure 30 | # that the process is down AND that the registry has been updated accordingly 31 | Endpoint.subscribe("connect:#{tenant_id}") 32 | 33 | if Connect.whereis(tenant_id) do 34 | Connect.shutdown(tenant_id) 35 | 36 | receive do 37 | %{event: "connect_down"} -> :ok 38 | after 39 | 5000 -> 40 | if Connect.whereis(tenant_id) do 41 | raise "Connect process for tenant #{tenant_id} did not shut down in time" 42 | end 43 | end 44 | end 45 | after 46 | Endpoint.unsubscribe("connect:#{tenant_id}") 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20240523004032_redefine_authorization_tables.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.RedefineAuthorizationTables do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | drop table(:broadcasts), mode: :cascade 8 | drop table(:presences), mode: :cascade 9 | drop table(:channels), mode: :cascade 10 | 11 | create_if_not_exists table(:messages) do 12 | add :topic, :text, null: false 13 | add :extension, :text, null: false 14 | timestamps() 15 | end 16 | 17 | create_if_not_exists index(:messages, [:topic]) 18 | 19 | execute("ALTER TABLE realtime.messages ENABLE row level security") 20 | execute("GRANT SELECT ON realtime.messages TO postgres, anon, authenticated, service_role") 21 | execute("GRANT UPDATE ON realtime.messages TO postgres, anon, authenticated, service_role") 22 | 23 | execute(""" 24 | GRANT INSERT ON realtime.messages TO postgres, anon, authenticated, service_role 25 | """) 26 | 27 | execute(""" 28 | GRANT USAGE ON SEQUENCE realtime.messages_id_seq TO postgres, anon, authenticated, service_role 29 | """) 30 | 31 | execute("ALTER table realtime.messages OWNER to supabase_realtime_admin") 32 | 33 | execute(""" 34 | DROP function realtime.channel_name 35 | """) 36 | 37 | execute(""" 38 | create or replace function realtime.topic() returns text as $$ 39 | select nullif(current_setting('realtime.topic', true), '')::text; 40 | $$ language sql stable; 41 | """) 42 | 43 | execute("ALTER function realtime.topic() owner to supabase_realtime_admin") 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /test/realtime_web/integration/tracing_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Integration.TracingTest do 2 | # Async due to usage of global otel_simple_processor 3 | use RealtimeWeb.ConnCase, async: false 4 | 5 | @parent_id "b7ad6b7169203331" 6 | @traceparent "00-0af7651916cd43dd8448eb211c80319c-#{@parent_id}-01" 7 | @span_parent_id Integer.parse(@parent_id, 16) |> elem(0) 8 | 9 | # This is doing a blackbox approach because tracing is not captured with normal Phoenix controller tests 10 | # We need cowboy, endpoint and router to trigger their telemetry events 11 | 12 | test "traces basic HTTP request with phoenix and cowboy information" do 13 | :otel_simple_processor.set_exporter(:otel_exporter_pid, self()) 14 | url = RealtimeWeb.Endpoint.url() <> "/healthcheck" 15 | 16 | baggage_request_id = UUID.uuid4() 17 | 18 | response = 19 | Req.get!(url, headers: [{"traceparent", @traceparent}, {"baggage", "sb-request-id=#{baggage_request_id}"}]) 20 | 21 | assert_receive {:span, span(name: "GET /healthcheck", attributes: attributes, parent_span_id: @span_parent_id)} 22 | 23 | assert attributes( 24 | map: %{ 25 | "http.request.method": :GET, 26 | "http.response.status_code": 200, 27 | "http.route": "/healthcheck", 28 | "phoenix.action": :healthcheck, 29 | "phoenix.plug": RealtimeWeb.PageController, 30 | "url.path": "/healthcheck", 31 | "url.scheme": :http 32 | } 33 | ) = attributes 34 | 35 | assert %{"x-request-id" => [^baggage_request_id]} = response.headers 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: supabase/postgres:14.1.0.105 4 | container_name: realtime-db 5 | ports: 6 | - "5432:5432" 7 | volumes: 8 | - ./dev/postgres:/docker-entrypoint-initdb.d/ 9 | command: postgres -c config_file=/etc/postgresql/postgresql.conf 10 | environment: 11 | POSTGRES_HOST: /var/run/postgresql 12 | POSTGRES_PASSWORD: postgres 13 | tenant_db: 14 | image: supabase/postgres:14.1.0.105 15 | container_name: tenant-db 16 | ports: 17 | - "5433:5432" 18 | command: postgres -c config_file=/etc/postgresql/postgresql.conf 19 | environment: 20 | POSTGRES_HOST: /var/run/postgresql 21 | POSTGRES_PASSWORD: postgres 22 | realtime: 23 | depends_on: 24 | - db 25 | build: . 26 | container_name: realtime-server 27 | ports: 28 | - "4000:4000" 29 | extra_hosts: 30 | - "host.docker.internal:host-gateway" 31 | environment: 32 | PORT: 4000 33 | DB_HOST: host.docker.internal 34 | DB_PORT: 5432 35 | DB_USER: postgres 36 | DB_PASSWORD: postgres 37 | DB_NAME: postgres 38 | DB_ENC_KEY: supabaserealtime 39 | DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' 40 | API_JWT_SECRET: dc447559-996d-4761-a306-f47a5eab1623 41 | SECRET_KEY_BASE: UpNVntn3cDxHJpq99YMc1T1AQgQpc8kfYTuRgBiYa15BLrx8etQoXz3gZv1/u2oq 42 | ERL_AFLAGS: -proto_dist inet_tcp 43 | RLIMIT_NOFILE: 1000000 44 | DNS_NODES: "''" 45 | APP_NAME: realtime 46 | RUN_JANITOR: true 47 | JANITOR_INTERVAL: 60000 48 | LOG_LEVEL: "info" 49 | SEED_SELF_HOST: true 50 | 51 | -------------------------------------------------------------------------------- /lib/realtime/monitoring/erl_sys_mon.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.ErlSysMon do 2 | @moduledoc """ 3 | Logs Erlang System Monitor events. 4 | """ 5 | 6 | use GenServer 7 | 8 | require Logger 9 | 10 | @defaults [ 11 | :busy_dist_port, 12 | :busy_port, 13 | {:long_gc, 500}, 14 | {:long_schedule, 500}, 15 | {:long_message_queue, {0, 1_000}} 16 | ] 17 | 18 | def start_link(args), do: GenServer.start_link(__MODULE__, args) 19 | 20 | def init(args) do 21 | config = Keyword.get(args, :config, @defaults) 22 | :erlang.system_monitor(self(), config) 23 | 24 | {:ok, []} 25 | end 26 | 27 | def handle_info({:monitor, pid, _type, _meta} = msg, state) when is_pid(pid) do 28 | log_process_info(msg, pid) 29 | {:noreply, state} 30 | end 31 | 32 | def handle_info(msg, state) do 33 | Logger.warning("#{__MODULE__} message: " <> inspect(msg)) 34 | {:noreply, state} 35 | end 36 | 37 | defp log_process_info(msg, pid) do 38 | pid_info = 39 | pid 40 | |> Process.info(:dictionary) 41 | |> case do 42 | {:dictionary, dict} when is_list(dict) -> 43 | {List.keyfind(dict, :"$initial_call", 0), List.keyfind(dict, :"$ancestors", 0)} 44 | 45 | other -> 46 | other 47 | end 48 | 49 | extra_info = Process.info(pid, [:registered_name, :message_queue_len, :total_heap_size]) 50 | 51 | Logger.warning( 52 | "#{__MODULE__} message: " <> 53 | inspect(msg) <> "|\n process info: #{inspect(pid_info)} #{inspect(extra_info)}" 54 | ) 55 | rescue 56 | _ -> 57 | Logger.warning("#{__MODULE__} message: " <> inspect(msg)) 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211116024918_create_realtime_subscription_table.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeSubscriptionTable do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | DO $$ 9 | BEGIN 10 | IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'equality_op') THEN 11 | CREATE TYPE realtime.equality_op AS ENUM( 12 | 'eq', 'neq', 'lt', 'lte', 'gt', 'gte' 13 | ); 14 | END IF; 15 | END$$; 16 | """) 17 | 18 | execute(""" 19 | DO $$ 20 | BEGIN 21 | IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'user_defined_filter') THEN 22 | CREATE TYPE realtime.user_defined_filter as ( 23 | column_name text, 24 | op realtime.equality_op, 25 | value text 26 | ); 27 | END IF; 28 | END$$; 29 | """) 30 | 31 | execute("create table if not exists realtime.subscription ( 32 | -- Tracks which users are subscribed to each table 33 | id bigint not null generated always as identity, 34 | user_id uuid not null, 35 | -- Populated automatically by trigger. Required to enable auth.email() 36 | email varchar(255), 37 | entity regclass not null, 38 | filters realtime.user_defined_filter[] not null default '{}', 39 | created_at timestamp not null default timezone('utc', now()), 40 | 41 | constraint pk_subscription primary key (id), 42 | unique (entity, user_id, filters) 43 | )") 44 | 45 | execute("create index if not exists ix_realtime_subscription_entity on realtime.subscription using hash (entity)") 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/realtime/metrics_cleaner.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.MetricsCleaner do 2 | @moduledoc false 3 | 4 | use GenServer 5 | require Logger 6 | 7 | defstruct [:check_ref, :interval] 8 | 9 | def start_link(args), do: GenServer.start_link(__MODULE__, args) 10 | 11 | def init(_args) do 12 | interval = Application.get_env(:realtime, :metrics_cleaner_schedule_timer_in_ms) 13 | 14 | Logger.info("Starting MetricsCleaner") 15 | {:ok, %{check_ref: check(interval), interval: interval}} 16 | end 17 | 18 | def handle_info(:check, %{interval: interval} = state) do 19 | Process.cancel_timer(state.check_ref) 20 | 21 | {exec_time, _} = :timer.tc(fn -> loop_and_cleanup_metrics_table() end, :millisecond) 22 | 23 | if exec_time > :timer.seconds(5), 24 | do: Logger.warning("Metrics check took: #{exec_time} ms") 25 | 26 | {:noreply, %{state | check_ref: check(interval)}} 27 | end 28 | 29 | def handle_info(msg, state) do 30 | Logger.error("Unexpected message: #{inspect(msg)}") 31 | {:noreply, state} 32 | end 33 | 34 | defp check(interval), do: Process.send_after(self(), :check, interval) 35 | 36 | @peep_filter_spec [{{{:_, %{tenant: :"$1"}}, :_}, [{:is_binary, :"$1"}], [:"$1"]}] 37 | 38 | defp loop_and_cleanup_metrics_table do 39 | tenant_ids = Realtime.Tenants.Connect.list_tenants() |> MapSet.new() 40 | 41 | {_, {tid, _}} = Peep.Persistent.storage(Realtime.PromEx.Metrics) 42 | 43 | tid 44 | |> :ets.select(@peep_filter_spec) 45 | |> Enum.uniq() 46 | |> Stream.reject(fn tenant_id -> MapSet.member?(tenant_ids, tenant_id) end) 47 | |> Enum.map(fn tenant_id -> %{tenant: tenant_id} end) 48 | |> then(&Peep.prune_tags(Realtime.PromEx.Metrics, &1)) 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20220908172859_null_passes_filters_recreate_is_visible_through_filters.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.NullPassesFiltersRecreateIsVisibleThroughFilters do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(" 8 | create or replace function realtime.is_visible_through_filters(columns realtime.wal_column[], filters realtime.user_defined_filter[]) 9 | returns bool 10 | language sql 11 | immutable 12 | as $$ 13 | /* 14 | Should the record be visible (true) or filtered out (false) after *filters* are applied 15 | */ 16 | select 17 | -- Default to allowed when no filters present 18 | $2 is null -- no filters. this should not happen because subscriptions has a default 19 | or array_length($2, 1) is null -- array length of an empty array is null 20 | or bool_and( 21 | coalesce( 22 | realtime.check_equality_op( 23 | op:=f.op, 24 | type_:=coalesce( 25 | col.type_oid::regtype, -- null when wal2json version <= 2.4 26 | col.type_name::regtype 27 | ), 28 | -- cast jsonb to text 29 | val_1:=col.value #>> '{}', 30 | val_2:=f.value 31 | ), 32 | false -- if null, filter does not match 33 | ) 34 | ) 35 | from 36 | unnest(filters) f 37 | join unnest(columns) col 38 | on f.column_name = col.name; 39 | $$; 40 | ") 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/realtime/tenants/repo/migrations/20211116212300_create_realtime_build_prepared_statement_sql_function.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Migrations.CreateRealtimeBuildPreparedStatementSqlFunction do 2 | @moduledoc false 3 | 4 | use Ecto.Migration 5 | 6 | def change do 7 | execute(""" 8 | DO $$ 9 | BEGIN 10 | IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'wal_column') THEN 11 | CREATE TYPE realtime.wal_column AS ( 12 | name text, 13 | type text, 14 | value jsonb, 15 | is_pkey boolean, 16 | is_selectable boolean 17 | ); 18 | END IF; 19 | END$$; 20 | """) 21 | 22 | execute("create function realtime.build_prepared_statement_sql( 23 | prepared_statement_name text, 24 | entity regclass, 25 | columns realtime.wal_column[] 26 | ) 27 | returns text 28 | language sql 29 | as $$ 30 | /* 31 | Builds a sql string that, if executed, creates a prepared statement to 32 | tests retrive a row from *entity* by its primary key columns. 33 | 34 | Example 35 | select realtime.build_prepared_statment_sql('public.notes', '{\"id\"}'::text[], '{\"bigint\"}'::text[]) 36 | */ 37 | select 38 | 'prepare ' || prepared_statement_name || ' as 39 | select 40 | exists( 41 | select 42 | 1 43 | from 44 | ' || entity || ' 45 | where 46 | ' || string_agg(quote_ident(pkc.name) || '=' || quote_nullable(pkc.value) , ' and ') || ' 47 | )' 48 | from 49 | unnest(columns) pkc 50 | where 51 | pkc.is_pkey 52 | group by 53 | entity 54 | $$;") 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/realtime_web/views/error_helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.ErrorHelpers do 2 | @moduledoc """ 3 | Conveniences for translating and building error messages. 4 | """ 5 | 6 | use Phoenix.HTML 7 | 8 | @doc """ 9 | Generates tag for inlined form input errors. 10 | """ 11 | def error_tag(form, field) do 12 | Enum.map(Keyword.get_values(form.errors, field), fn error -> 13 | content_tag(:span, translate_error(error), 14 | class: "invalid-feedback", 15 | phx_feedback_for: input_id(form, field) 16 | ) 17 | end) 18 | end 19 | 20 | @doc """ 21 | Translates an error message using gettext. 22 | """ 23 | def translate_error({msg, opts}) do 24 | # When using gettext, we typically pass the strings we want 25 | # to translate as a static argument: 26 | # 27 | # # Translate "is invalid" in the "errors" domain 28 | # dgettext("errors", "is invalid") 29 | # 30 | # # Translate the number of files with plural rules 31 | # dngettext("errors", "1 file", "%{count} files", count) 32 | # 33 | # Because the error messages we show in our forms and APIs 34 | # are defined inside Ecto, we need to translate them dynamically. 35 | # This requires us to call the Gettext module passing our gettext 36 | # backend as first argument. 37 | # 38 | # Note we use the "errors" domain, which means translations 39 | # should be written to the errors.po file. The :count option is 40 | # set by Ecto and indicates we should also apply plural rules. 41 | if count = opts[:count] do 42 | Gettext.dngettext(RealtimeWeb.Gettext, "errors", msg, msg, count, opts) 43 | else 44 | Gettext.dgettext(RealtimeWeb.Gettext, "errors", msg, opts) 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /test/realtime_web/plugs/rate_limiter_test.exs: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Plugs.RateLimiterTest do 2 | use RealtimeWeb.ConnCase 3 | 4 | alias Realtime.Api 5 | 6 | @tenant %{ 7 | "external_id" => "localhost", 8 | "name" => "localhost", 9 | "max_events_per_second" => 0, 10 | "extensions" => [ 11 | %{ 12 | "type" => "postgres_cdc_rls", 13 | "settings" => %{ 14 | "db_host" => "127.0.0.1", 15 | "db_name" => "postgres", 16 | "db_user" => "supabase_admin", 17 | "db_password" => "postgres", 18 | "db_port" => "6432", 19 | "poll_interval" => 100, 20 | "poll_max_changes" => 100, 21 | "poll_max_record_bytes" => 1_048_576, 22 | "region" => "us-east-1" 23 | } 24 | } 25 | ], 26 | "postgres_cdc_default" => "postgres_cdc_rls", 27 | "jwt_secret" => "new secret" 28 | } 29 | 30 | setup %{conn: conn} do 31 | conn = 32 | conn 33 | |> put_req_header("accept", "application/json") 34 | 35 | {:ok, _tenant} = Api.create_tenant(@tenant) 36 | 37 | {:ok, conn: conn} 38 | end 39 | 40 | test "serve a 429 when rate limit is set to 0", %{conn: conn} do 41 | conn = 42 | conn 43 | |> Map.put(:host, "localhost.localhost.com") 44 | |> get(Routes.ping_path(conn, :ping)) 45 | 46 | assert conn.status == 429 47 | end 48 | 49 | test "serve a 200 when rate limit is set to 100", %{conn: conn} do 50 | {:ok, _tenant} = Api.update_tenant_by_external_id(@tenant["external_id"], %{"max_events_per_second" => 100}) 51 | 52 | conn = 53 | conn 54 | |> Map.put(:host, "localhost.localhost.com") 55 | |> get(Routes.ping_path(conn, :ping)) 56 | 57 | assert conn.status == 200 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/extensions/postgres_cdc_rls/worker_supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule Extensions.PostgresCdcRls.WorkerSupervisor do 2 | @moduledoc false 3 | use Supervisor 4 | 5 | alias Extensions.PostgresCdcRls 6 | alias PostgresCdcRls.ReplicationPoller 7 | alias PostgresCdcRls.SubscriptionManager 8 | alias PostgresCdcRls.SubscriptionsChecker 9 | alias Realtime.Tenants.Cache 10 | alias Realtime.PostgresCdc.Exception 11 | 12 | def start_link(args) do 13 | name = PostgresCdcRls.supervisor_id(args["id"], args["region"]) 14 | Supervisor.start_link(__MODULE__, args, name: {:via, :syn, name}) 15 | end 16 | 17 | @impl true 18 | def init(%{"id" => tenant} = args) when is_binary(tenant) do 19 | Logger.metadata(external_id: tenant, project: tenant) 20 | unless Cache.get_tenant_by_external_id(tenant), do: raise(Exception) 21 | 22 | subscribers_pids_table = :ets.new(__MODULE__, [:public, :bag]) 23 | subscribers_nodes_table = :ets.new(__MODULE__, [:public, :set]) 24 | 25 | tid_args = 26 | Map.merge(args, %{ 27 | "subscribers_pids_table" => subscribers_pids_table, 28 | "subscribers_nodes_table" => subscribers_nodes_table 29 | }) 30 | 31 | children = [ 32 | %{ 33 | id: ReplicationPoller, 34 | start: {ReplicationPoller, :start_link, [tid_args]}, 35 | restart: :transient 36 | }, 37 | %{ 38 | id: SubscriptionManager, 39 | start: {SubscriptionManager, :start_link, [tid_args]}, 40 | restart: :transient 41 | }, 42 | %{ 43 | id: SubscriptionsChecker, 44 | start: {SubscriptionsChecker, :start_link, [tid_args]}, 45 | restart: :transient 46 | } 47 | ] 48 | 49 | Supervisor.init(children, strategy: :rest_for_one, max_restarts: 10, max_seconds: 60) 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/support/data_case.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.DataCase do 2 | @moduledoc """ 3 | This module defines the setup for tests requiring 4 | access to the application's data layer. 5 | You may define functions here to be used as helpers in 6 | your tests. 7 | Finally, if the test case interacts with the database, 8 | we enable the SQL sandbox, so changes done to the database 9 | are reverted at the end of every test. If you are using 10 | PostgreSQL, you can even run database tests asynchronously 11 | by setting `use Realtime.DataCase, async: true`, although 12 | this option is not recommended for other databases. 13 | """ 14 | 15 | use ExUnit.CaseTemplate 16 | alias Ecto.Adapters.SQL.Sandbox 17 | 18 | using do 19 | quote do 20 | alias Realtime.Repo 21 | 22 | import Ecto 23 | import Ecto.Changeset 24 | import Ecto.Query 25 | import Realtime.DataCase 26 | import Generators 27 | import TenantConnection 28 | end 29 | end 30 | 31 | setup tags do 32 | pid = Sandbox.start_owner!(Realtime.Repo, shared: not tags[:async]) 33 | on_exit(fn -> Sandbox.stop_owner(pid) end) 34 | 35 | {:ok, conn: Phoenix.ConnTest.build_conn()} 36 | end 37 | 38 | @doc """ 39 | A helper that transforms changeset errors into a map of messages. 40 | assert {:error, changeset} = Accounts.create_user(%{password: "short"}) 41 | assert "password is too short" in errors_on(changeset).password 42 | assert %{password: ["password is too short"]} = errors_on(changeset) 43 | """ 44 | def errors_on(changeset) do 45 | Ecto.Changeset.traverse_errors(changeset, fn {message, opts} -> 46 | Regex.replace(~r"%{(\w+)}", message, fn _, key -> 47 | opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() 48 | end) 49 | end) 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /.github/workflows/prod_linter.yml: -------------------------------------------------------------------------------- 1 | name: Production Formatting Checks 2 | on: 3 | pull_request: 4 | branches: 5 | - release 6 | 7 | jobs: 8 | format: 9 | name: Formatting Checks 10 | runs-on: blacksmith-4vcpu-ubuntu-2404 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Setup elixir 15 | id: beam 16 | uses: erlef/setup-beam@v1 17 | with: 18 | otp-version: 27.x # Define the OTP version [required] 19 | elixir-version: 1.18.x # Define the elixir version [required] 20 | - name: Cache Mix 21 | uses: actions/cache@v4 22 | with: 23 | path: deps 24 | key: ${{ runner.os }}-mix-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} 25 | restore-keys: | 26 | ${{ runner.os }}-mix- 27 | 28 | - name: Install dependencies 29 | run: mix deps.get 30 | - name: Set up Postgres 31 | run: docker compose -f docker-compose.dbs.yml up -d 32 | - name: Run database migrations 33 | run: mix ecto.migrate 34 | - name: Run format check 35 | run: mix format --check-formatted 36 | - name: Credo checks 37 | run: mix credo --strict --mute-exit-status 38 | - name: Retrieve PLT Cache 39 | uses: actions/cache@v4 40 | id: plt-cache 41 | with: 42 | path: priv/plts 43 | key: ${{ runner.os }}-${{ steps.beam.outputs.otp-version }}-${{ steps.beam.outputs.elixir-version }}-plts-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }} 44 | - name: Create PLTs 45 | if: steps.plt-cache.outputs.cache-hit != 'true' 46 | run: | 47 | mkdir -p priv/plts 48 | mix dialyzer.build 49 | - name: Run dialyzer 50 | run: mix dialyzer 51 | - name: Run tests 52 | run: mix test 53 | -------------------------------------------------------------------------------- /test/realtime/tenants/rebalancer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.RebalancerTest do 2 | use Realtime.DataCase, async: true 3 | 4 | alias Realtime.Tenants.Rebalancer 5 | alias Realtime.Nodes 6 | 7 | use Mimic 8 | 9 | setup do 10 | tenant = Containers.checkout_tenant(run_migrations: true) 11 | # Warm cache to avoid Cachex and Ecto.Sandbox ownership issues 12 | Cachex.put!(Realtime.Tenants.Cache, {{:get_tenant_by_external_id, 1}, [tenant.external_id]}, {:cached, tenant}) 13 | %{tenant: tenant} 14 | end 15 | 16 | describe "check/3" do 17 | test "different node set returns :ok", %{tenant: tenant} do 18 | external_id = tenant.external_id 19 | 20 | # Don't even try to look for the region 21 | reject(&Nodes.get_node_for_tenant/1) 22 | 23 | assert Rebalancer.check(MapSet.new([node()]), MapSet.new([node(), :other_node]), external_id) == :ok 24 | end 25 | 26 | test "same node set correct region set returns :ok", %{tenant: tenant} do 27 | external_id = tenant.external_id 28 | current_region = Application.fetch_env!(:realtime, :region) 29 | 30 | expect(Nodes, :get_node_for_tenant, fn ^tenant -> {:ok, :some_node, current_region} end) 31 | reject(&Nodes.get_node_for_tenant/1) 32 | 33 | assert Rebalancer.check(MapSet.new([node(), :some_node]), MapSet.new([node(), :some_node]), external_id) == :ok 34 | end 35 | 36 | test "same node set different region set returns :ok", %{tenant: tenant} do 37 | external_id = tenant.external_id 38 | 39 | expect(Nodes, :get_node_for_tenant, fn ^tenant -> {:ok, :some_node, "ap-southeast-1"} end) 40 | reject(&Nodes.get_node_for_tenant/1) 41 | 42 | assert Rebalancer.check(MapSet.new([node(), :some_node]), MapSet.new([node(), :some_node]), external_id) == 43 | {:error, :wrong_region} 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: 3 | pull_request: 4 | paths: 5 | - "lib/**" 6 | - "test/**" 7 | - "config/**" 8 | - "priv/**" 9 | - "assets/**" 10 | - "rel/**" 11 | - "mix.exs" 12 | - "Dockerfile" 13 | - "run.sh" 14 | 15 | push: 16 | branches: 17 | - main 18 | 19 | concurrency: 20 | group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 21 | cancel-in-progress: true 22 | 23 | env: 24 | MIX_ENV: test 25 | 26 | jobs: 27 | tests: 28 | name: Tests 29 | runs-on: blacksmith-8vcpu-ubuntu-2404 30 | 31 | steps: 32 | - uses: actions/checkout@v2 33 | - name: Setup elixir 34 | id: beam 35 | uses: erlef/setup-beam@v1 36 | with: 37 | otp-version: 27.x # Define the OTP version [required] 38 | elixir-version: 1.18.x # Define the elixir version [required] 39 | - name: Cache Mix 40 | uses: actions/cache@v4 41 | with: 42 | path: | 43 | deps 44 | _build 45 | key: ${{ github.workflow }}-${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}-${{ hashFiles('**/mix.lock') }} 46 | restore-keys: | 47 | ${{ github.workflow }}-${{ runner.os }}-mix-${{ env.elixir }}-${{ env.otp }}- 48 | 49 | - name: Pull postgres image quietly in background (used by test/support/containers.ex) 50 | run: docker pull supabase/postgres:15.8.1.040 > /dev/null 2>&1 & 51 | - name: Install dependencies 52 | run: mix deps.get 53 | - name: Set up Postgres 54 | run: docker compose -f docker-compose.dbs.yml up -d 55 | - name: Start epmd 56 | run: epmd -daemon 57 | - name: Run tests 58 | run: MIX_ENV=test MAX_CASES=3 mix coveralls.github 59 | env: 60 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 61 | -------------------------------------------------------------------------------- /lib/realtime/adapters/changes.ex: -------------------------------------------------------------------------------- 1 | # This file draws heavily from https://github.com/cainophile/cainophile 2 | # License: https://github.com/cainophile/cainophile/blob/master/LICENSE 3 | 4 | require Protocol 5 | 6 | defmodule Realtime.Adapters.Changes do 7 | @moduledoc """ 8 | This module provides structures of CDC changes. 9 | """ 10 | defmodule Transaction do 11 | @moduledoc false 12 | defstruct [:changes, :commit_timestamp] 13 | end 14 | 15 | defmodule NewRecord do 16 | @moduledoc false 17 | @derive {Jason.Encoder, except: [:subscription_ids]} 18 | defstruct [ 19 | :columns, 20 | :commit_timestamp, 21 | :errors, 22 | :schema, 23 | :table, 24 | :record, 25 | :subscription_ids, 26 | :type 27 | ] 28 | end 29 | 30 | defmodule UpdatedRecord do 31 | @moduledoc false 32 | @derive {Jason.Encoder, except: [:subscription_ids]} 33 | defstruct [ 34 | :columns, 35 | :commit_timestamp, 36 | :errors, 37 | :schema, 38 | :table, 39 | :old_record, 40 | :record, 41 | :subscription_ids, 42 | :type 43 | ] 44 | end 45 | 46 | defmodule DeletedRecord do 47 | @moduledoc false 48 | @derive {Jason.Encoder, except: [:subscription_ids]} 49 | defstruct [ 50 | :columns, 51 | :commit_timestamp, 52 | :errors, 53 | :schema, 54 | :table, 55 | :old_record, 56 | :subscription_ids, 57 | :type 58 | ] 59 | end 60 | 61 | defmodule TruncatedRelation do 62 | @moduledoc false 63 | defstruct [:type, :schema, :table, :commit_timestamp] 64 | end 65 | end 66 | 67 | Protocol.derive(Jason.Encoder, Realtime.Adapters.Changes.Transaction) 68 | Protocol.derive(Jason.Encoder, Realtime.Adapters.Changes.TruncatedRelation) 69 | Protocol.derive(Jason.Encoder, Realtime.Adapters.Postgres.Decoder.Messages.Relation.Column) 70 | -------------------------------------------------------------------------------- /test/support/metrics_helper.ex: -------------------------------------------------------------------------------- 1 | defmodule MetricsHelper do 2 | @spec search(String.t(), String.t(), map() | keyword() | nil) :: 3 | {:ok, String.t(), map(), String.t()} | {:error, String.t()} 4 | def search(prometheus_metrics, metric_name, expected_tags \\ nil) do 5 | # Escape the metric_name to handle any special regex characters 6 | escaped_name = Regex.escape(metric_name) 7 | regex = ~r/^(?#{escaped_name})\{(?[^}]+)\}\s+(?\d+(?:\.\d+)?)$/ 8 | 9 | prometheus_metrics 10 | |> IO.iodata_to_binary() 11 | |> String.split("\n", trim: true) 12 | |> Enum.find_value( 13 | nil, 14 | fn item -> 15 | case parse(item, regex, expected_tags) do 16 | {:ok, value} -> value 17 | {:error, _reason} -> false 18 | end 19 | end 20 | ) 21 | |> case do 22 | nil -> nil 23 | number -> String.to_integer(number) 24 | end 25 | end 26 | 27 | defp parse(metric_string, regex, expected_tags) do 28 | case Regex.named_captures(regex, metric_string) do 29 | %{"name" => _name, "tags" => tags_string, "value" => value} -> 30 | tags = parse_tags(tags_string) 31 | 32 | if expected_tags && !matching_tags(tags, expected_tags) do 33 | {:error, "Tags do not match expected tags"} 34 | else 35 | {:ok, value} 36 | end 37 | 38 | nil -> 39 | {:error, "Invalid metric format or metric name mismatch"} 40 | end 41 | end 42 | 43 | defp parse_tags(tags_string) do 44 | ~r/(?[a-zA-Z_][a-zA-Z0-9_]*)="(?[^"]*)"/ 45 | |> Regex.scan(tags_string, capture: :all_names) 46 | |> Enum.map(fn [key, value] -> {key, value} end) 47 | |> Map.new() 48 | end 49 | 50 | defp matching_tags(tags, expected_tags) do 51 | Enum.all?(expected_tags, fn {k, v} -> Map.get(tags, to_string(k)) == to_string(v) end) 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/realtime/monitoring/prom_ex/plugins/tenants.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.PromEx.Plugins.Tenants do 2 | @moduledoc false 3 | 4 | use PromEx.Plugin 5 | 6 | alias PromEx.MetricTypes.Event 7 | alias Realtime.Tenants.Connect 8 | 9 | require Logger 10 | 11 | defmodule Buckets do 12 | @moduledoc false 13 | use Peep.Buckets.Custom, buckets: [10, 250, 5000, 15_000] 14 | end 15 | 16 | @event_connected [:prom_ex, :plugin, :realtime, :tenants, :connected] 17 | 18 | @impl true 19 | def event_metrics(_) do 20 | Event.build(:realtime, [ 21 | distribution( 22 | [:realtime, :global, :rpc], 23 | event_name: [:realtime, :rpc], 24 | description: "Global Latency of rpc calls", 25 | measurement: :latency, 26 | unit: {:microsecond, :millisecond}, 27 | tags: [:success, :mechanism], 28 | reporter_options: [peep_bucket_calculator: Buckets] 29 | ) 30 | ]) 31 | end 32 | 33 | @impl true 34 | def polling_metrics(opts) do 35 | poll_rate = Keyword.get(opts, :poll_rate) 36 | 37 | [ 38 | Polling.build( 39 | :realtime_tenants_events, 40 | poll_rate, 41 | {__MODULE__, :execute_metrics, []}, 42 | [ 43 | last_value( 44 | [:realtime, :tenants, :connected], 45 | event_name: @event_connected, 46 | description: "The total count of connected tenants.", 47 | measurement: :connected 48 | ) 49 | ], 50 | detach_on_error: false 51 | ) 52 | ] 53 | end 54 | 55 | def execute_metrics do 56 | connected = 57 | if Enum.member?(:syn.node_scopes(), Connect), 58 | do: :syn.local_registry_count(Connect), 59 | else: -1 60 | 61 | execute_metrics(@event_connected, %{connected: connected}) 62 | end 63 | 64 | defp execute_metrics(event, metrics) do 65 | :telemetry.execute(event, metrics, %{}) 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test/realtime/metrics_cleaner_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.MetricsCleanerTest do 2 | # async: false due to potentially polluting metrics with other tenant metrics from other tests 3 | use Realtime.DataCase, async: false 4 | 5 | alias Realtime.MetricsCleaner 6 | alias Realtime.Tenants.Connect 7 | 8 | setup do 9 | interval = Application.get_env(:realtime, :metrics_cleaner_schedule_timer_in_ms) 10 | Application.put_env(:realtime, :metrics_cleaner_schedule_timer_in_ms, 100) 11 | on_exit(fn -> Application.put_env(:realtime, :metrics_cleaner_schedule_timer_in_ms, interval) end) 12 | 13 | tenant = Containers.checkout_tenant(run_migrations: true) 14 | 15 | %{tenant: tenant} 16 | end 17 | 18 | describe "metrics cleanup" do 19 | test "cleans up metrics for users that have been disconnected", %{tenant: %{external_id: external_id}} do 20 | start_supervised!(MetricsCleaner) 21 | {:ok, _} = Connect.lookup_or_start_connection(external_id) 22 | # Wait for promex to collect the metrics 23 | Process.sleep(6000) 24 | 25 | :telemetry.execute( 26 | [:realtime, :connections], 27 | %{connected: 10, connected_cluster: 10, limit: 100}, 28 | %{tenant: external_id} 29 | ) 30 | 31 | :telemetry.execute( 32 | [:realtime, :connections], 33 | %{connected: 20, connected_cluster: 20, limit: 100}, 34 | %{tenant: "disconnected-tenant"} 35 | ) 36 | 37 | metrics = Realtime.PromEx.get_metrics() |> IO.iodata_to_binary() 38 | 39 | assert String.contains?(metrics, external_id) 40 | assert String.contains?(metrics, "disconnected-tenant") 41 | 42 | # Wait for clenaup to run 43 | Process.sleep(200) 44 | 45 | metrics = Realtime.PromEx.get_metrics() |> IO.iodata_to_binary() 46 | 47 | assert String.contains?(metrics, external_id) 48 | refute String.contains?(metrics, "disconnected-tenant") 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /priv/repo/seeds.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | import Ecto.Adapters.SQL, only: [query: 3] 4 | 5 | alias Realtime.Api.Tenant 6 | alias Realtime.Repo 7 | alias Realtime.Tenants 8 | 9 | tenant_name = System.get_env("SELF_HOST_TENANT_NAME", "realtime-dev") 10 | default_db_host = "host.docker.internal" 11 | 12 | {:ok, tenant} = 13 | Repo.transaction(fn -> 14 | case Repo.get_by(Tenant, external_id: tenant_name) do 15 | %Tenant{} = tenant -> Repo.delete!(tenant) 16 | nil -> {:ok, nil} 17 | end 18 | 19 | %Tenant{} 20 | |> Tenant.changeset(%{ 21 | "name" => tenant_name, 22 | "external_id" => tenant_name, 23 | "jwt_secret" => System.get_env("API_JWT_SECRET", "super-secret-jwt-token-with-at-least-32-characters-long"), 24 | "jwt_jwks" => System.get_env("API_JWT_JWKS") |> then(fn v -> if v, do: Jason.decode!(v) end), 25 | "extensions" => [ 26 | %{ 27 | "type" => "postgres_cdc_rls", 28 | "settings" => %{ 29 | "db_name" => System.get_env("DB_NAME", "postgres"), 30 | "db_host" => System.get_env("DB_HOST", default_db_host), 31 | "db_user" => System.get_env("DB_USER", "supabase_admin"), 32 | "db_password" => System.get_env("DB_PASSWORD", "postgres"), 33 | "db_port" => System.get_env("DB_PORT", "5433"), 34 | "region" => "us-east-1", 35 | "poll_interval_ms" => 100, 36 | "poll_max_record_bytes" => 1_048_576, 37 | "ssl_enforced" => false 38 | } 39 | } 40 | ] 41 | }) 42 | |> Repo.insert!() 43 | end) 44 | 45 | tenant = Tenants.get_tenant_by_external_id(tenant_name) 46 | 47 | with res when res in [:noop, :ok] <- Tenants.Migrations.run_migrations(tenant), 48 | :ok <- Tenants.Janitor.MaintenanceTask.run(tenant.external_id) do 49 | Logger.info("Tenant set-up successfully") 50 | else 51 | error -> 52 | Logger.info("Failed to set-up tenant: #{inspect(error)}") 53 | end 54 | -------------------------------------------------------------------------------- /lib/realtime/logs.ex: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Logs do 2 | @moduledoc """ 3 | Logging operations for Realtime 4 | """ 5 | require Logger 6 | 7 | defmacro __using__(_opts) do 8 | quote do 9 | require Logger 10 | 11 | import Realtime.Logs 12 | end 13 | end 14 | 15 | @doc """ 16 | Prepares a value to be logged 17 | """ 18 | def to_log(value) when is_binary(value), do: value 19 | def to_log(value), do: inspect(value, pretty: true) 20 | 21 | defmacro log_error(code, error, metadata \\ []) do 22 | quote bind_quoted: [code: code, error: error, metadata: metadata], location: :keep do 23 | Logger.error("#{code}: #{Realtime.Logs.to_log(error)}", [error_code: code] ++ metadata) 24 | end 25 | end 26 | 27 | defmacro log_warning(code, warning, metadata \\ []) do 28 | quote bind_quoted: [code: code, warning: warning, metadata: metadata], location: :keep do 29 | Logger.warning("#{code}: #{Realtime.Logs.to_log(warning)}", [{:error_code, code} | metadata]) 30 | end 31 | end 32 | end 33 | 34 | defimpl Jason.Encoder, for: DBConnection.ConnectionError do 35 | def encode( 36 | %DBConnection.ConnectionError{message: message, reason: reason, severity: severity}, 37 | _opts 38 | ) do 39 | inspect(%{message: message, reason: reason, severity: severity}, pretty: true) 40 | end 41 | end 42 | 43 | defimpl Jason.Encoder, for: Postgrex.Error do 44 | def encode( 45 | %Postgrex.Error{ 46 | message: message, 47 | postgres: %{code: code, schema: schema, table: table} 48 | }, 49 | _opts 50 | ) do 51 | inspect(%{message: message, schema: schema, table: table, code: code}, pretty: true) 52 | end 53 | end 54 | 55 | defimpl Jason.Encoder, for: Tuple do 56 | require Logger 57 | 58 | def encode(tuple, _opts) do 59 | Logger.error("UnableToEncodeJson: Tuple encoding not supported: #{inspect(tuple)}") 60 | inspect(%{error: "unable to parse response"}, pretty: true) 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/realtime_web/plugs/auth_tenant.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.AuthTenant do 2 | @moduledoc """ 3 | Authorization plug to ensure that only authorized clients can connect to the their tenant's endpoints. 4 | """ 5 | require Logger 6 | 7 | import Plug.Conn 8 | import Phoenix.Controller, only: [json: 2] 9 | 10 | alias Realtime.Api.Tenant 11 | alias Realtime.Crypto 12 | 13 | alias RealtimeWeb.ChannelsAuthorization 14 | 15 | def init(opts), do: opts 16 | 17 | def call(%{assigns: %{tenant: tenant}} = conn, _opts) do 18 | Logger.metadata(external_id: tenant.external_id, project: tenant.external_id) 19 | 20 | with %Tenant{jwt_secret: jwt_secret, jwt_jwks: jwt_jwks} <- tenant, 21 | token when is_binary(token) <- access_token(conn), 22 | jwt_secret_dec <- Crypto.decrypt!(jwt_secret), 23 | {:ok, claims} <- ChannelsAuthorization.authorize_conn(token, jwt_secret_dec, jwt_jwks) do 24 | conn 25 | |> assign(:claims, claims) 26 | |> assign(:jwt, token) 27 | |> assign(:role, claims["role"]) 28 | |> assign(:sub, claims["sub"]) 29 | else 30 | _error -> unauthorized(conn) 31 | end 32 | end 33 | 34 | def call(conn, _opts), do: unauthorized(conn) 35 | 36 | defp access_token(conn) do 37 | authorization = get_req_header(conn, "authorization") 38 | apikey = get_req_header(conn, "apikey") 39 | 40 | authorization = 41 | case authorization do 42 | [] -> 43 | nil 44 | 45 | [""] -> 46 | nil 47 | 48 | [value | _] -> 49 | [bearer, token] = value |> String.split(" ") 50 | bearer = String.downcase(bearer) 51 | if bearer == "bearer", do: token 52 | end 53 | 54 | apikey = 55 | case apikey do 56 | [] -> nil 57 | [value | _] -> value 58 | end 59 | 60 | cond do 61 | authorization -> authorization 62 | apikey -> apikey 63 | true -> nil 64 | end 65 | end 66 | 67 | defp unauthorized(conn), 68 | do: conn |> put_status(401) |> json(%{message: "Unauthorized"}) |> halt() 69 | end 70 | -------------------------------------------------------------------------------- /lib/realtime_web/telemetry.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Telemetry do 2 | @moduledoc false 3 | 4 | use Supervisor 5 | import Telemetry.Metrics 6 | 7 | def start_link(arg) do 8 | Supervisor.start_link(__MODULE__, arg, name: __MODULE__) 9 | end 10 | 11 | @impl true 12 | def init(_arg) do 13 | children = [ 14 | # Telemetry poller will execute the given period measurements 15 | # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics 16 | {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} 17 | # Add reporters as children of your supervision tree. 18 | # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} 19 | ] 20 | 21 | Supervisor.init(children, strategy: :one_for_one) 22 | end 23 | 24 | def metrics do 25 | [ 26 | # Phoenix Metrics 27 | summary("phoenix.endpoint.stop.duration", 28 | unit: {:native, :millisecond} 29 | ), 30 | summary("phoenix.router_dispatch.stop.duration", 31 | tags: [:route], 32 | unit: {:native, :millisecond} 33 | ), 34 | 35 | # Database Metrics 36 | summary("realtime.repo.query.total_time", unit: {:native, :millisecond}), 37 | summary("realtime.repo.query.decode_time", unit: {:native, :millisecond}), 38 | summary("realtime.repo.query.query_time", unit: {:native, :millisecond}), 39 | summary("realtime.repo.query.queue_time", unit: {:native, :millisecond}), 40 | summary("realtime.repo.query.idle_time", unit: {:native, :millisecond}), 41 | 42 | # VM Metrics 43 | summary("vm.memory.total", unit: {:byte, :kilobyte}), 44 | summary("vm.total_run_queue_lengths.total"), 45 | summary("vm.total_run_queue_lengths.cpu"), 46 | summary("vm.total_run_queue_lengths.io") 47 | ] 48 | end 49 | 50 | defp periodic_measurements do 51 | [ 52 | # A module, function and arguments to be invoked periodically. 53 | # This function must call :telemetry.execute/3 and a metric must be added above. 54 | # {RealtimeWeb, :count_users, []} 55 | ] 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/extensions/postgres_cdc_rls/message_dispatcher.ex: -------------------------------------------------------------------------------- 1 | # This file draws from https://github.com/phoenixframework/phoenix/blob/9941711736c8464b27b40914a4d954ed2b4f5958/lib/phoenix/channel/server.ex 2 | # License: https://github.com/phoenixframework/phoenix/blob/518a4640a70aa4d1370a64c2280d598e5b928168/LICENSE.md 3 | 4 | defmodule Extensions.PostgresCdcRls.MessageDispatcher do 5 | @moduledoc """ 6 | Hook invoked by Phoenix.PubSub dispatch. 7 | """ 8 | 9 | alias Phoenix.Socket.Broadcast 10 | 11 | def dispatch([_ | _] = topic_subscriptions, _from, {type, payload, sub_ids}) do 12 | _ = 13 | Enum.reduce(topic_subscriptions, %{}, fn 14 | {_pid, {:subscriber_fastlane, fastlane_pid, serializer, ids, join_topic, is_new_api}}, cache -> 15 | for {bin_id, id} <- ids, reduce: [] do 16 | acc -> 17 | if MapSet.member?(sub_ids, bin_id) do 18 | [id | acc] 19 | else 20 | acc 21 | end 22 | end 23 | |> case do 24 | [_ | _] = valid_ids -> 25 | new_payload = 26 | if is_new_api do 27 | %Broadcast{ 28 | topic: join_topic, 29 | event: "postgres_changes", 30 | payload: %{ids: valid_ids, data: Jason.Fragment.new(payload)} 31 | } 32 | else 33 | %Broadcast{topic: join_topic, event: type, payload: Jason.Fragment.new(payload)} 34 | end 35 | 36 | broadcast_message(cache, fastlane_pid, new_payload, serializer) 37 | 38 | _ -> 39 | cache 40 | end 41 | end) 42 | 43 | :ok 44 | end 45 | 46 | defp broadcast_message(cache, fastlane_pid, msg, serializer) do 47 | case cache do 48 | %{^msg => encoded_msg} -> 49 | send(fastlane_pid, encoded_msg) 50 | cache 51 | 52 | %{} -> 53 | encoded_msg = serializer.fastlane!(msg) 54 | send(fastlane_pid, encoded_msg) 55 | Map.put(cache, msg, encoded_msg) 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # Configure your database 4 | # 5 | # The MIX_TEST_PARTITION environment variable can be used 6 | # to provide built-in test partitioning in CI environment. 7 | # Run `mix help test` for more information. 8 | for repo <- [ 9 | Realtime.Repo, 10 | Realtime.Repo.Replica.FRA, 11 | Realtime.Repo.Replica.IAD, 12 | Realtime.Repo.Replica.SIN, 13 | Realtime.Repo.Replica.SJC, 14 | Realtime.Repo.Replica.Singapore, 15 | Realtime.Repo.Replica.London, 16 | Realtime.Repo.Replica.NorthVirginia, 17 | Realtime.Repo.Replica.Oregon, 18 | Realtime.Repo.Replica.SanJose 19 | ] do 20 | config :realtime, repo, 21 | username: "postgres", 22 | password: "postgres", 23 | database: "realtime_test", 24 | hostname: "127.0.0.1", 25 | pool: Ecto.Adapters.SQL.Sandbox 26 | end 27 | 28 | # Running server during tests to run integration tests 29 | config :realtime, RealtimeWeb.Endpoint, 30 | http: [port: 4002], 31 | server: true 32 | 33 | # that's what config/runtime.exs expects to see as region 34 | System.put_env("REGION", "us-east-1") 35 | 36 | config :realtime, 37 | regional_broadcasting: true, 38 | region: "us-east-1", 39 | db_enc_key: "1234567890123456", 40 | jwt_claim_validators: System.get_env("JWT_CLAIM_VALIDATORS", "{}"), 41 | api_jwt_secret: System.get_env("API_JWT_SECRET", "secret"), 42 | metrics_jwt_secret: "test", 43 | prom_poll_rate: 5_000, 44 | request_id_baggage_key: "sb-request-id" 45 | 46 | # Print nothing during tests unless captured or a test failure happens 47 | config :logger, 48 | backends: [], 49 | level: :info 50 | 51 | # Configures Elixir's Logger 52 | config :logger, :console, 53 | format: "$time $metadata[$level] $message\n", 54 | metadata: [:error_code, :request_id, :project, :external_id, :application_name, :sub, :iss, :exp] 55 | 56 | config :opentelemetry, 57 | span_processor: :simple, 58 | traces_exporter: :none, 59 | processors: [{:otel_simple_processor, %{}}] 60 | 61 | # Using different ports so that a remote node during test can connect using the same local network 62 | # See Clustered module 63 | config :gen_rpc, 64 | tcp_server_port: 5969, 65 | tcp_client_port: 5970 66 | -------------------------------------------------------------------------------- /test/realtime/tenants/connect/register_process_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Realtime.Tenants.Connect.RegisterProcessTest do 2 | use Realtime.DataCase, async: true 3 | alias Realtime.Tenants.Connect.RegisterProcess 4 | alias Realtime.Database 5 | 6 | describe "run/1" do 7 | setup do 8 | tenant = Containers.checkout_tenant(run_migrations: true) 9 | # Warm cache to avoid Cachex and Ecto.Sandbox ownership issues 10 | Cachex.put!(Realtime.Tenants.Cache, {{:get_tenant_by_external_id, 1}, [tenant.external_id]}, {:cached, tenant}) 11 | {:ok, conn} = Database.connect(tenant, "realtime_test") 12 | %{tenant_id: tenant.external_id, db_conn_pid: conn} 13 | end 14 | 15 | test "registers the process in syn and Registry and updates metadata", %{tenant_id: tenant_id, db_conn_pid: conn} do 16 | # Fake the process registration in :syn 17 | :syn.register(Realtime.Tenants.Connect, tenant_id, self(), %{conn: nil}) 18 | assert {:ok, _} = RegisterProcess.run(%{tenant_id: tenant_id, db_conn_pid: conn}) 19 | assert {pid, %{conn: ^conn}} = :syn.lookup(Realtime.Tenants.Connect, tenant_id) 20 | assert [{^pid, %{}}] = Registry.lookup(Realtime.Tenants.Connect.Registry, tenant_id) 21 | end 22 | 23 | test "fails to register the process in syn and Registry and updates metadata", %{ 24 | tenant_id: tenant_id, 25 | db_conn_pid: conn 26 | } do 27 | # Fake the process registration in :syn 28 | :syn.register(Realtime.Tenants.Connect, tenant_id, self(), %{conn: nil}) 29 | 30 | # Register normally 31 | assert {:ok, _} = RegisterProcess.run(%{tenant_id: tenant_id, db_conn_pid: conn}) 32 | assert {pid, %{conn: ^conn}} = :syn.lookup(Realtime.Tenants.Connect, tenant_id) 33 | assert [{^pid, %{}}] = Registry.lookup(Realtime.Tenants.Connect.Registry, tenant_id) 34 | 35 | # Check failure 36 | assert {:error, :already_registered} = RegisterProcess.run(%{tenant_id: tenant_id, db_conn_pid: conn}) 37 | end 38 | 39 | test "handles undefined process error" do 40 | assert {:error, :process_not_found} = 41 | RegisterProcess.run(%{tenant_id: Generators.random_string(), db_conn_pid: nil}) 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/realtime_web/channels/payloads/join.ex: -------------------------------------------------------------------------------- 1 | defmodule RealtimeWeb.Channels.Payloads.Join do 2 | @moduledoc """ 3 | Payload validation for the phx_join event. 4 | """ 5 | use Ecto.Schema 6 | import Ecto.Changeset 7 | alias RealtimeWeb.Channels.Payloads.Config 8 | alias RealtimeWeb.Channels.Payloads.Broadcast 9 | alias RealtimeWeb.Channels.Payloads.Presence 10 | 11 | embedded_schema do 12 | embeds_one :config, Config 13 | field :access_token, :string 14 | field :user_token, :string 15 | end 16 | 17 | def changeset(join, attrs) do 18 | join 19 | |> cast(attrs, [:access_token, :user_token], message: &error_message/2) 20 | |> cast_embed(:config, invalid_message: "unable to parse, expected a map") 21 | end 22 | 23 | @spec validate(map()) :: {:ok, %__MODULE__{}} | {:error, :invalid_join_payload, map()} 24 | def validate(params) do 25 | case changeset(%__MODULE__{}, params) do 26 | %Ecto.Changeset{valid?: true} = changeset -> 27 | {:ok, Ecto.Changeset.apply_changes(changeset)} 28 | 29 | %Ecto.Changeset{valid?: false} = changeset -> 30 | errors = Ecto.Changeset.traverse_errors(changeset, &elem(&1, 0)) 31 | {:error, :invalid_join_payload, errors} 32 | end 33 | end 34 | 35 | def presence_enabled?(%__MODULE__{config: %Config{presence: %Presence{enabled: enabled}}}), do: enabled 36 | def presence_enabled?(_), do: true 37 | 38 | def presence_key(%__MODULE__{config: %Config{presence: %Presence{key: ""}}}), do: UUID.uuid1() 39 | def presence_key(%__MODULE__{config: %Config{presence: %Presence{key: key}}}), do: key 40 | def presence_key(_), do: UUID.uuid1() 41 | 42 | def ack_broadcast?(%__MODULE__{config: %Config{broadcast: %Broadcast{ack: ack}}}), do: ack 43 | def ack_broadcast?(_), do: false 44 | 45 | def self_broadcast?(%__MODULE__{config: %Config{broadcast: %Broadcast{self: self}}}), do: self 46 | def self_broadcast?(_), do: false 47 | 48 | def private?(%__MODULE__{config: %Config{private: private}}), do: private 49 | def private?(_), do: false 50 | 51 | def error_message(_field, meta) do 52 | type = Keyword.get(meta, :type) 53 | 54 | if type, 55 | do: "unable to parse, expected #{type}", 56 | else: "unable to parse" 57 | end 58 | end 59 | --------------------------------------------------------------------------------