├── test ├── test_helper.exs ├── database │ └── 00-init.sql └── postgres_replication_test.exs ├── example ├── inspect_calls │ ├── test │ │ ├── test_helper.exs │ │ └── inspect_calls_test.exs │ ├── db │ │ └── 00-init.sql │ ├── .formatter.exs │ ├── docker-compose.yml │ ├── README.md │ ├── mix.exs │ ├── .gitignore │ ├── lib │ │ ├── handler.ex │ │ └── inspect_calls.ex │ └── mix.lock └── wal_replication │ ├── test │ ├── test_helper.exs │ └── wal_replication_test.exs │ ├── db │ └── 00-init.sql │ ├── .formatter.exs │ ├── docker-compose.yml │ ├── README.md │ ├── mix.exs │ ├── .gitignore │ ├── mix.lock │ └── lib │ ├── wal_replication.ex │ └── wal_replication │ └── handler.ex ├── .formatter.exs ├── docker-compose.yml ├── .gitignore ├── mix.exs ├── lib ├── postgres_replication │ ├── handler.ex │ ├── protocol │ │ ├── keep_alive.ex │ │ └── write.ex │ ├── protocol.ex │ └── plugin │ │ └── pgoutput │ │ ├── oid_database.ex │ │ └── decoder.ex └── postgres_replication.ex ├── README.md └── mix.lock /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /example/inspect_calls/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /example/wal_replication/test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /test/database/00-init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE random_values (id SERIAL PRIMARY KEY, value text); -------------------------------------------------------------------------------- /example/inspect_calls/db/00-init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE public.random_values (id SERIAL PRIMARY KEY, value text); -------------------------------------------------------------------------------- /example/wal_replication/db/00-init.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE public.random_values (id SERIAL PRIMARY KEY, value text); -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /example/inspect_calls/.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /example/wal_replication/.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /example/inspect_calls/test/inspect_calls_test.exs: -------------------------------------------------------------------------------- 1 | defmodule InspectCallsTest do 2 | use ExUnit.Case 3 | doctest InspectCalls 4 | 5 | test "greets the world" do 6 | assert InspectCalls.hello() == :world 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /example/wal_replication/test/wal_replication_test.exs: -------------------------------------------------------------------------------- 1 | defmodule WalReplicationTest do 2 | use ExUnit.Case 3 | doctest WalReplication 4 | 5 | test "greets the world" do 6 | assert WalReplication.hello() == :world 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: postgres:14 4 | volumes: 5 | - ./test/database:/docker-entrypoint-initdb.d/ 6 | ports: 7 | - "5432:5432" 8 | command: ["postgres", "-c", "wal_level=logical"] 9 | environment: 10 | POSTGRES_PASSWORD: postgres 11 | -------------------------------------------------------------------------------- /example/inspect_calls/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: postgres:14 4 | volumes: 5 | - ./db:/docker-entrypoint-initdb.d/ 6 | ports: 7 | - "5432:5432" 8 | command: ["postgres", "-c", "wal_level=logical"] 9 | environment: 10 | POSTGRES_PASSWORD: postgres 11 | -------------------------------------------------------------------------------- /example/wal_replication/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db-1: 3 | image: postgres:14 4 | volumes: 5 | - ./db:/docker-entrypoint-initdb.d/ 6 | ports: 7 | - "5432:5432" 8 | command: ["postgres", "-c", "wal_level=logical"] 9 | environment: 10 | POSTGRES_PASSWORD: postgres 11 | db-2: 12 | image: postgres:14 13 | volumes: 14 | - ./db:/docker-entrypoint-initdb.d/ 15 | ports: 16 | - "5433:5432" 17 | environment: 18 | POSTGRES_PASSWORD: postgres -------------------------------------------------------------------------------- /example/inspect_calls/README.md: -------------------------------------------------------------------------------- 1 | # InspectCalls 2 | 3 | **TODO: Add description** 4 | 5 | ## Installation 6 | 7 | If [available in Hex](https://hex.pm/docs/publish), the package can be installed 8 | by adding `inspect_calls` to your list of dependencies in `mix.exs`: 9 | 10 | ```elixir 11 | def deps do 12 | [ 13 | {:inspect_calls, "~> 0.1.0"} 14 | ] 15 | end 16 | ``` 17 | 18 | Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) 19 | and published on [HexDocs](https://hexdocs.pm). Once published, the docs can 20 | be found at . 21 | 22 | -------------------------------------------------------------------------------- /example/wal_replication/README.md: -------------------------------------------------------------------------------- 1 | # WalReplication 2 | 3 | **TODO: Add description** 4 | 5 | ## Installation 6 | 7 | If [available in Hex](https://hex.pm/docs/publish), the package can be installed 8 | by adding `wal_replication` to your list of dependencies in `mix.exs`: 9 | 10 | ```elixir 11 | def deps do 12 | [ 13 | {:wal_replication, "~> 0.1.0"} 14 | ] 15 | end 16 | ``` 17 | 18 | Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) 19 | and published on [HexDocs](https://hexdocs.pm). Once published, the docs can 20 | be found at . 21 | 22 | -------------------------------------------------------------------------------- /example/inspect_calls/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule InspectCalls.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :inspect_calls, 7 | version: "0.1.0", 8 | elixir: "~> 1.17", 9 | start_permanent: Mix.env() == :prod, 10 | deps: deps() 11 | ] 12 | end 13 | 14 | # Run "mix help compile.app" to learn about applications. 15 | def application do 16 | [ 17 | extra_applications: [:logger], 18 | mod: {InspectCalls, []} 19 | ] 20 | end 21 | 22 | # Run "mix help deps" to learn about dependencies. 23 | defp deps do 24 | [ 25 | {:postgres_replication, path: "../../"} 26 | ] 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /example/wal_replication/mix.exs: -------------------------------------------------------------------------------- 1 | defmodule WalReplication.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :wal_replication, 7 | version: "0.1.0", 8 | elixir: "~> 1.17", 9 | start_permanent: Mix.env() == :prod, 10 | deps: deps() 11 | ] 12 | end 13 | 14 | # Run "mix help compile.app" to learn about applications. 15 | def application do 16 | [ 17 | extra_applications: [:logger], 18 | mod: {WalReplication, []} 19 | ] 20 | end 21 | 22 | # Run "mix help deps" to learn about dependencies. 23 | defp deps do 24 | [ 25 | {:postgres_replication, path: "../../"} 26 | ] 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | postgres_replication-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :postgres_replication, 7 | version: "0.1.0", 8 | elixir: "~> 1.17", 9 | start_permanent: Mix.env() == :prod, 10 | deps: deps() 11 | ] 12 | end 13 | 14 | # Run "mix help compile.app" to learn about applications. 15 | def application do 16 | [ 17 | extra_applications: [:logger] 18 | ] 19 | end 20 | 21 | # Run "mix help deps" to learn about dependencies. 22 | defp deps do 23 | [ 24 | {:postgrex, "~> 0.20"}, 25 | {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, 26 | {:sobelow, "~> 0.13.0", only: [:dev, :test], runtime: false} 27 | ] 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /example/inspect_calls/.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | inspect_calls-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /example/wal_replication/.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | wal_replication-*.tar 24 | 25 | # Temporary files, for example, from tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /example/inspect_calls/lib/handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Handler do 2 | @behaviour PostgresReplication.Handler 3 | import PostgresReplication.Protocol 4 | alias PostgresReplication.Protocol.KeepAlive 5 | 6 | @impl true 7 | def call(message, _state) when is_write(message) do 8 | message 9 | |> PostgresReplication.Protocol.parse() 10 | |> PostgresReplication.Decoder.decode_message() 11 | 12 | :noreply 13 | end 14 | 15 | def call(message, _state) when is_keep_alive(message) do 16 | reply = 17 | case parse(message) do 18 | %KeepAlive{reply: :now, wal_end: wal_end} -> 19 | standby(wal_end + 1, wal_end + 1, wal_end + 1, :now) 20 | 21 | _ -> 22 | hold() 23 | end 24 | 25 | {:reply, reply} 26 | end 27 | 28 | def call(_, _), do: :noreply 29 | end 30 | -------------------------------------------------------------------------------- /lib/postgres_replication/handler.ex: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication.Handler do 2 | @moduledoc """ 3 | A behaviour module for handling logical replication messages. 4 | """ 5 | @type t :: module() 6 | @doc """ 7 | The `call/2` callback is called by the `PostgresReplication` module to send messages to the parent process. It also sends back to the server connection a message in return if the user wants to. 8 | 9 | ## Parameters 10 | * `message` - The message to be sent to the parent process. 11 | * `target_pid` - The PID of the process to send the message to. 12 | 13 | ## Returns 14 | * `{:reply, [term]}` - The message to be sent to server connection. Read more in PostgresReplication.Protocol 15 | * `:noreply` - No message is sent back to the server. 16 | """ 17 | @callback call(any, PostgresReplication.t()) :: {:reply, [term]} | :noreply 18 | end 19 | -------------------------------------------------------------------------------- /lib/postgres_replication/protocol/keep_alive.ex: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication.Protocol.KeepAlive do 2 | @moduledoc """ 3 | Primary keepalive message (B) 4 | Byte1('k') 5 | Identifies the message as a sender keepalive. 6 | 7 | Int64 8 | The current end of WAL on the server. 9 | 10 | Int64 11 | The server's system clock at the time of transmission, as microseconds since midnight on 2000-01-01. 12 | 13 | Byte1 14 | 1 means that the client should reply to this message as soon as possible, to avoid a timeout disconnect. 0 otherwise. 15 | 16 | The receiving process can send replies back to the sender at any time, using one of the following message formats (also in the payload of a CopyData message): 17 | """ 18 | @type t :: %__MODULE__{ 19 | wal_end: integer(), 20 | clock: integer(), 21 | reply: :now | :await 22 | } 23 | defstruct [:wal_end, :clock, :reply] 24 | end 25 | -------------------------------------------------------------------------------- /lib/postgres_replication/protocol/write.ex: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication.Protocol.Write do 2 | @moduledoc """ 3 | XLogData (B) 4 | Byte1('w') 5 | Identifies the message as WAL data. 6 | 7 | Int64 8 | The starting point of the WAL data in this message. 9 | 10 | Int64 11 | The current end of WAL on the server. 12 | 13 | Int64 14 | The server's system clock at the time of transmission, as microseconds since midnight on 2000-01-01. 15 | 16 | Byten 17 | A section of the WAL data stream. 18 | 19 | A single WAL record is never split across two XLogData messages. When a WAL record crosses a WAL page boundary, and is therefore already split using continuation records, it can be split at the page boundary. In other words, the first main WAL record and its continuation records can be sent in different XLogData messages. 20 | """ 21 | defstruct [:server_wal_start, :server_wal_end, :server_system_clock, :message] 22 | end 23 | -------------------------------------------------------------------------------- /example/inspect_calls/mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, 3 | "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, 4 | "postgrex": {:hex, :postgrex, "0.18.0", "f34664101eaca11ff24481ed4c378492fed2ff416cd9b06c399e90f321867d7e", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a042989ba1bc1cca7383ebb9e461398e3f89f868c92ce6671feb7ef132a252d1"}, 5 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 6 | } 7 | -------------------------------------------------------------------------------- /example/wal_replication/mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, 3 | "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, 4 | "postgrex": {:hex, :postgrex, "0.18.0", "f34664101eaca11ff24481ed4c378492fed2ff416cd9b06c399e90f321867d7e", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a042989ba1bc1cca7383ebb9e461398e3f89f868c92ce6671feb7ef132a252d1"}, 5 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 6 | } 7 | -------------------------------------------------------------------------------- /example/inspect_calls/lib/inspect_calls.ex: -------------------------------------------------------------------------------- 1 | defmodule InspectCalls do 2 | use Application 3 | 4 | @db1 [ 5 | hostname: "localhost", 6 | username: "postgres", 7 | password: "postgres", 8 | database: "postgres", 9 | port: 5432, 10 | parameters: [ 11 | application_name: "PostgresReplication" 12 | ] 13 | ] 14 | 15 | def start(_type, _args) do 16 | children = [ 17 | {PostgresReplication, 18 | %PostgresReplication{ 19 | connection_opts: @db1, 20 | table: :all, 21 | opts: [name: __MODULE__, auto_reconnect: true], 22 | handler_module: Handler 23 | }} 24 | ] 25 | 26 | opts = [strategy: :one_for_one, name: WalReplication.Supervisor] 27 | Supervisor.start_link(children, opts) 28 | end 29 | 30 | def db1(), do: Postgrex.start_link([name: WalReplication.DB1] ++ @db1) 31 | 32 | def run_insert() do 33 | db1() 34 | query = "INSERT INTO random_values (value) VALUES ('Random Text 1') RETURNING id" 35 | 36 | %{rows: [[id]]} = 37 | Postgrex.query!(Process.whereis(WalReplication.DB1), query, []) 38 | id 39 | end 40 | 41 | def run_delete() do 42 | id = run_insert() 43 | query = "DELETE FROM random_values WHERE id = #{id}" 44 | Postgrex.query!(Process.whereis(WalReplication.DB1), query, []) 45 | end 46 | 47 | def run_update() do 48 | id = run_insert() 49 | query = "UPDATE random_values SET value = 'Random Text 2' WHERE id = #{id}" 50 | Postgrex.query!(Process.whereis(WalReplication.DB1), query, []) 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /example/wal_replication/lib/wal_replication.ex: -------------------------------------------------------------------------------- 1 | defmodule WalReplication do 2 | use Application 3 | 4 | @db1 [ 5 | hostname: "localhost", 6 | username: "postgres", 7 | password: "postgres", 8 | database: "postgres", 9 | port: 5432, 10 | parameters: [ 11 | application_name: "PostgresReplication" 12 | ] 13 | ] 14 | 15 | @db2 [ 16 | hostname: "localhost", 17 | username: "postgres", 18 | password: "postgres", 19 | database: "postgres", 20 | port: 5433, 21 | parameters: [ 22 | application_name: "PostgresReplication" 23 | ] 24 | ] 25 | 26 | def start(_type, _args) do 27 | configuration = %PostgresReplication{ 28 | connection_opts: [ 29 | hostname: "localhost", 30 | username: "postgres", 31 | password: "postgres", 32 | database: "postgres", 33 | port: 5432 34 | ], 35 | table: :all, 36 | opts: [name: __MODULE__, auto_reconnect: true], 37 | handler_module: WalReplication.Handler 38 | } 39 | 40 | replication_slot = PostgresReplication.replication_slot_name(configuration) 41 | handler_name = {:via, Registry, {Registry.Handler, replication_slot}} 42 | 43 | children = [ 44 | {Registry, keys: :unique, name: Registry.Handler}, 45 | {WalReplication.Handler, @db2 ++ [name: handler_name]}, 46 | {PostgresReplication, configuration} 47 | ] 48 | 49 | opts = [strategy: :one_for_one, name: WalReplication.Supervisor] 50 | Supervisor.start_link(children, opts) 51 | end 52 | 53 | def db1(), do: Postgrex.start_link([name: WalReplication.DB1] ++ @db1) 54 | def db2(), do: Postgrex.start_link([name: WalReplication.DB2] ++ @db2) 55 | 56 | def run_insert() do 57 | db1() 58 | query = "INSERT INTO random_values (value) VALUES ('Random Text 1') RETURNING id" 59 | %{rows: [[id]]} = Postgrex.query!(Process.whereis(WalReplication.DB1), query, []) 60 | id 61 | end 62 | 63 | def run_delete() do 64 | id = run_insert() 65 | query = "DELETE FROM random_values WHERE id = #{id}" 66 | Postgrex.query!(Process.whereis(WalReplication.DB1), query, []) 67 | end 68 | 69 | def run_update() do 70 | id = run_insert() 71 | query = "UPDATE random_values SET value = 'Random Text 2' WHERE id = #{id}" 72 | Postgrex.query!(Process.whereis(WalReplication.DB1), query, []) 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PostgresReplication 2 | 3 | Simple wrapper to consume WAL entries from your Postgres Database. It offers an abstraction on top of Postgrex to simplify WAL consumption. 4 | 5 | It also offers a decoder in PostgresReplication.Decoder based on [https://github.com/cainophile/cainophile](https://github.com/cainophile/cainophile) 6 | 7 | ## Usage 8 | Provide the options to connect to the database and the message handler module. As an example present in the [examples](./example/) folder here's how you can track events on all tables: 9 | 10 | ```elixir 11 | Mix.install([{:postgres_replication, git: "https://github.com/filipecabaco/postgres_replication.git"}]) 12 | 13 | defmodule Handler do 14 | @behaviour PostgresReplication.Handler 15 | import PostgresReplication.Protocol 16 | alias PostgresReplication.Protocol.KeepAlive 17 | alias PostgresReplication.Plugin.Pgoutput.Decoder 18 | 19 | @impl true 20 | def call(message, _parent_pid) when is_write(message) do 21 | message 22 | |> PostgresReplication.Protocol.parse() 23 | |> Decoder.decode_message() 24 | |> IO.inspect() 25 | 26 | :noreply 27 | end 28 | 29 | def call(message, _parent_pid) when is_keep_alive(message) do 30 | reply = 31 | case parse(message) do 32 | %KeepAlive{reply: :now, wal_end: wal_end} -> 33 | wal_end = wal_end + 1 34 | standby(wal_end, wal_end, wal_end, :now) 35 | 36 | _ -> 37 | hold() 38 | end 39 | 40 | {:reply, reply} 41 | end 42 | 43 | def call(_, _), do: :noreply 44 | end 45 | 46 | 47 | defmodule Replication do 48 | def start do 49 | options = %PostgresReplication{ 50 | connection_opts: [ 51 | hostname: "localhost", 52 | username: "postgres", 53 | password: "postgres", 54 | database: "postgres", 55 | ], 56 | table: :all, 57 | opts: [name: __MODULE__, auto_reconnect: true], 58 | handler_module: Handler, 59 | } 60 | 61 | {:ok, _pid} = PostgresReplication.start_link(options) 62 | Process.sleep(:infinity) 63 | end 64 | end 65 | 66 | Replication.start() 67 | ``` 68 | 69 | ## Installation 70 | 71 | 72 | ```elixir 73 | def deps do 74 | [ 75 | {:postgres_replication, "~> 0.1.0"} 76 | ] 77 | end 78 | ``` 79 | 80 | You need your database to set your `wal_level` to the value of `logical` on start. 81 | 82 | ```sql 83 | ALTER SYSTEM SET wal_level='logical' 84 | ``` 85 | 86 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, 3 | "credo": {:hex, :credo, "1.7.11", "d3e805f7ddf6c9c854fd36f089649d7cf6ba74c42bc3795d587814e3c9847102", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "56826b4306843253a66e47ae45e98e7d284ee1f95d53d1612bb483f88a8cf219"}, 4 | "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, 5 | "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, 6 | "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, 7 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 8 | "postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"}, 9 | "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"}, 10 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 11 | } 12 | -------------------------------------------------------------------------------- /lib/postgres_replication/protocol.ex: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication.Protocol do 2 | @moduledoc """ 3 | Functions for parsing different types of WAL messages. 4 | """ 5 | alias PostgresReplication.Protocol.KeepAlive 6 | alias PostgresReplication.Protocol.Write 7 | 8 | defguard is_write(value) when binary_part(value, 0, 1) == <> 9 | defguard is_keep_alive(value) when binary_part(value, 0, 1) == <> 10 | 11 | def parse( 12 | <> 13 | ) do 14 | %Write{ 15 | server_wal_start: server_wal_start, 16 | server_wal_end: server_wal_end, 17 | server_system_clock: server_system_clock, 18 | message: message 19 | } 20 | end 21 | 22 | def parse(<>) do 23 | reply = 24 | case reply do 25 | 0 -> :later 26 | 1 -> :now 27 | end 28 | 29 | %KeepAlive{wal_end: wal_end, clock: clock, reply: reply} 30 | end 31 | 32 | @doc """ 33 | Message to send to the server to request a standby status update. 34 | 35 | Check https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-STANDBY-STATUS-UPDATE for more information 36 | """ 37 | @spec standby(integer(), integer(), integer(), :now | :later, integer() | nil) :: [binary()] 38 | def standby(last_wal_received, last_wal_flushed, last_wal_applied, reply, clock \\ nil) 39 | 40 | def standby(last_wal_received, last_wal_flushed, last_wal_applied, reply, nil) do 41 | standby(last_wal_received, last_wal_flushed, last_wal_applied, reply, current_time()) 42 | end 43 | 44 | def standby(last_wal_received, last_wal_flushed, last_wal_applied, reply, clock) do 45 | reply = 46 | case reply do 47 | :now -> 1 48 | :later -> 0 49 | end 50 | 51 | [ 52 | <> 54 | ] 55 | end 56 | 57 | @doc """ 58 | Message to send to ths server to request a hot standby status update. 59 | 60 | https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-HOT-STANDBY-FEEDBACK-MESSAGE 61 | """ 62 | @spec hot_standby(integer(), integer(), integer(), integer(), integer() | nil) :: [binary()] 63 | def hot_standby( 64 | standby_global_xmin, 65 | standby_global_xmin_epoch, 66 | standby_catalog_xmin, 67 | standby_catalog_xmin_epoch, 68 | clock \\ nil 69 | ) 70 | 71 | def hot_standby( 72 | standby_global_xmin, 73 | standby_global_xmin_epoch, 74 | standby_catalog_xmin, 75 | standby_catalog_xmin_epoch, 76 | nil 77 | ) do 78 | hot_standby( 79 | standby_global_xmin, 80 | standby_global_xmin_epoch, 81 | standby_catalog_xmin, 82 | standby_catalog_xmin_epoch, 83 | current_time() 84 | ) 85 | end 86 | 87 | def hot_standby( 88 | standby_global_xmin, 89 | standby_global_xmin_epoch, 90 | standby_catalog_xmin, 91 | standby_catalog_xmin_epoch, 92 | clock 93 | ) do 94 | [ 95 | <> 97 | ] 98 | end 99 | 100 | @doc """ 101 | Message to send the server to not do any operation since the server can wait 102 | """ 103 | @spec hold :: [] 104 | def hold, do: [] 105 | 106 | @epoch DateTime.to_unix(~U[2000-01-01 00:00:00Z], :microsecond) 107 | def current_time, do: System.os_time(:microsecond) - @epoch 108 | end 109 | -------------------------------------------------------------------------------- /test/postgres_replication_test.exs: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplicationTest do 2 | use ExUnit.Case 3 | alias PostgresReplication.Plugin.Pgoutput.Decoder 4 | 5 | test "handles database connection and receives WAL changes" do 6 | opts = %PostgresReplication{ 7 | connection_opts: [ 8 | hostname: "localhost", 9 | username: "postgres", 10 | password: "postgres", 11 | database: "postgres", 12 | parameters: [ 13 | application_name: "PostgresReplication" 14 | ] 15 | ], 16 | table: :all, 17 | opts: [name: __MODULE__], 18 | handler_module: PostgresReplicationTest.PgoutputHandler, 19 | metadata: %{pid: self()} 20 | } 21 | 22 | {:ok, conn} = Postgrex.start_link(opts.connection_opts) 23 | PostgresReplication.start_link(opts) 24 | 25 | Postgrex.query!( 26 | conn, 27 | "INSERT INTO random_values (value) VALUES ('Random Text 1')", 28 | [] 29 | ) 30 | 31 | assert_receive %Decoder.Messages.Begin{} 32 | assert_receive %Decoder.Messages.Relation{} 33 | 34 | assert_receive %Decoder.Messages.Insert{ 35 | tuple_data: {_, "Random Text 1"} 36 | } 37 | 38 | assert_receive %Decoder.Messages.Commit{} 39 | end 40 | 41 | test "handles database connection, receives WAL changes and you can set plugin options" do 42 | opts = %PostgresReplication{ 43 | connection_opts: [ 44 | hostname: "localhost", 45 | username: "postgres", 46 | password: "postgres", 47 | database: "postgres", 48 | parameters: [ 49 | application_name: "PostgresReplication" 50 | ] 51 | ], 52 | table: :all, 53 | opts: [name: __MODULE__], 54 | publication_name: "test_publication", 55 | output_plugin: "pgoutput", 56 | output_plugin_options: [ 57 | proto_version: "2", 58 | publication_names: :publication_name 59 | ], 60 | handler_module: PostgresReplicationTest.PgoutputHandler, 61 | metadata: %{pid: self()} 62 | } 63 | 64 | {:ok, conn} = Postgrex.start_link(opts.connection_opts) 65 | PostgresReplication.start_link(opts) 66 | 67 | Postgrex.query!( 68 | conn, 69 | "INSERT INTO random_values (value) VALUES ('Random Text 1')", 70 | [] 71 | ) 72 | 73 | assert_receive %Decoder.Messages.Begin{} 74 | assert_receive %Decoder.Messages.Relation{} 75 | 76 | assert_receive %Decoder.Messages.Insert{ 77 | tuple_data: {_, "Random Text 1"} 78 | } 79 | 80 | assert_receive %Decoder.Messages.Commit{} 81 | end 82 | 83 | test "handles database connection, receives WAL changes and can use plugins without options" do 84 | opts = %PostgresReplication{ 85 | connection_opts: [ 86 | hostname: "localhost", 87 | username: "postgres", 88 | password: "postgres", 89 | database: "postgres", 90 | parameters: [ 91 | application_name: "PostgresReplication" 92 | ] 93 | ], 94 | table: :all, 95 | opts: [name: __MODULE__], 96 | publication_name: "test_publication", 97 | output_plugin: "test_decoding", 98 | output_plugin_options: [], 99 | handler_module: PostgresReplicationTest.TestDecodingHandler, 100 | metadata: %{pid: self()} 101 | } 102 | 103 | {:ok, conn} = Postgrex.start_link(opts.connection_opts) 104 | PostgresReplication.start_link(opts) 105 | 106 | Postgrex.query!( 107 | conn, 108 | "INSERT INTO random_values (value) VALUES ('Random Text 1')", 109 | [] 110 | ) 111 | 112 | assert_receive :ok 113 | assert_receive :ok 114 | assert_receive :ok 115 | end 116 | 117 | defmodule TestDecodingHandler do 118 | @behaviour PostgresReplication.Handler 119 | 120 | @impl true 121 | def call(<>, %{metadata: %{pid: pid}}) do 122 | send(pid, :ok) 123 | :noreply 124 | end 125 | 126 | # Handles keep alive messages 127 | def call(<>, _) do 128 | messages = 129 | case reply do 130 | 1 -> [<>] 131 | 0 -> [] 132 | end 133 | 134 | {:reply, messages} 135 | end 136 | 137 | @epoch DateTime.to_unix(~U[2000-01-01 00:00:00Z], :microsecond) 138 | defp current_time, do: System.os_time(:microsecond) - @epoch 139 | end 140 | 141 | defmodule PgoutputHandler do 142 | @behaviour PostgresReplication.Handler 143 | 144 | @impl true 145 | def call(<>, %{metadata: %{pid: pid}}) do 146 | message |> Decoder.decode_message() |> then(&send(pid, &1)) 147 | :noreply 148 | end 149 | 150 | # Handles keep alive messages 151 | def call(<>, _) do 152 | messages = 153 | case reply do 154 | 1 -> [<>] 155 | 0 -> [] 156 | end 157 | 158 | {:reply, messages} 159 | end 160 | 161 | @epoch DateTime.to_unix(~U[2000-01-01 00:00:00Z], :microsecond) 162 | defp current_time, do: System.os_time(:microsecond) - @epoch 163 | end 164 | end 165 | -------------------------------------------------------------------------------- /lib/postgres_replication/plugin/pgoutput/oid_database.ex: -------------------------------------------------------------------------------- 1 | # CREDITS 2 | # This file draws heavily from https://github.com/cainophile/pgoutput_decoder 3 | # License: https://github.com/cainophile/pgoutput_decoder/blob/master/LICENSE 4 | 5 | # Lifted from epgsql (src/epgsql_binary.erl), this module licensed under 6 | # 3-clause BSD found here: https://raw.githubusercontent.com/epgsql/epgsql/devel/LICENSE 7 | 8 | # https://github.com/brianc/node-pg-types/blob/master/lib/builtins.js 9 | # MIT License (MIT) 10 | 11 | # Following query was used to generate this file: 12 | # SELECT json_object_agg(UPPER(PT.typname), PT.oid::int4 ORDER BY pt.oid) 13 | # FROM pg_type PT 14 | # WHERE typnamespace = (SELECT pgn.oid FROM pg_namespace pgn WHERE nspname = 'pg_catalog') -- Take only builting Postgres types with stable OID (extension types are not guaranteed to be stable) 15 | # AND typtype = 'b' -- Only basic types 16 | # AND typisdefined -- Ignore undefined types 17 | # credo:disable-for-this-file 18 | defmodule PostgresReplication.Plugin.Pgoutput.OidDatabase do 19 | @moduledoc "This module maps a numeric PostgreSQL type ID to a descriptive string." 20 | 21 | @doc """ 22 | Maps a numeric PostgreSQL type ID to a descriptive string. 23 | 24 | ## Examples 25 | 26 | iex> name_for_type_id(1700) 27 | "numeric" 28 | 29 | iex> name_for_type_id(25) 30 | "text" 31 | 32 | iex> name_for_type_id(3802) 33 | "jsonb" 34 | 35 | """ 36 | def name_for_type_id(type_id) do 37 | case type_id do 38 | 16 -> "bool" 39 | 17 -> "bytea" 40 | 18 -> "char" 41 | 19 -> "name" 42 | 20 -> "int8" 43 | 21 -> "int2" 44 | 22 -> "int2vector" 45 | 23 -> "int4" 46 | 24 -> "regproc" 47 | 25 -> "text" 48 | 26 -> "oid" 49 | 27 -> "tid" 50 | 28 -> "xid" 51 | 29 -> "cid" 52 | 30 -> "oidvector" 53 | 114 -> "json" 54 | 142 -> "xml" 55 | 143 -> "_xml" 56 | 194 -> "pg_node_tree" 57 | 199 -> "_json" 58 | 210 -> "smgr" 59 | 600 -> "point" 60 | 601 -> "lseg" 61 | 602 -> "path" 62 | 603 -> "box" 63 | 604 -> "polygon" 64 | 628 -> "line" 65 | 629 -> "_line" 66 | 650 -> "cidr" 67 | 651 -> "_cidr" 68 | 700 -> "float4" 69 | 701 -> "float8" 70 | 702 -> "abstime" 71 | 703 -> "reltime" 72 | 704 -> "tinterval" 73 | 718 -> "circle" 74 | 719 -> "_circle" 75 | 774 -> "macaddr8" 76 | 775 -> "_macaddr8" 77 | 790 -> "money" 78 | 791 -> "_money" 79 | 829 -> "macaddr" 80 | 869 -> "inet" 81 | 1000 -> "_bool" 82 | 1001 -> "_bytea" 83 | 1002 -> "_char" 84 | 1003 -> "_name" 85 | 1005 -> "_int2" 86 | 1006 -> "_int2vector" 87 | 1007 -> "_int4" 88 | 1008 -> "_regproc" 89 | 1009 -> "_text" 90 | 1010 -> "_tid" 91 | 1011 -> "_xid" 92 | 1012 -> "_cid" 93 | 1013 -> "_oidvector" 94 | 1014 -> "_bpchar" 95 | 1015 -> "_varchar" 96 | 1016 -> "_int8" 97 | 1017 -> "_point" 98 | 1018 -> "_lseg" 99 | 1019 -> "_path" 100 | 1020 -> "_box" 101 | 1021 -> "_float4" 102 | 1022 -> "_float8" 103 | 1023 -> "_abstime" 104 | 1024 -> "_reltime" 105 | 1025 -> "_tinterval" 106 | 1027 -> "_polygon" 107 | 1028 -> "_oid" 108 | 1033 -> "aclitem" 109 | 1034 -> "_aclitem" 110 | 1040 -> "_macaddr" 111 | 1041 -> "_inet" 112 | 1042 -> "bpchar" 113 | 1043 -> "varchar" 114 | 1082 -> "date" 115 | 1083 -> "time" 116 | 1114 -> "timestamp" 117 | 1115 -> "_timestamp" 118 | 1182 -> "_date" 119 | 1183 -> "_time" 120 | 1184 -> "timestamptz" 121 | 1185 -> "_timestamptz" 122 | 1186 -> "interval" 123 | 1187 -> "_interval" 124 | 1231 -> "_numeric" 125 | 1263 -> "_cstring" 126 | 1266 -> "timetz" 127 | 1270 -> "_timetz" 128 | 1560 -> "bit" 129 | 1561 -> "_bit" 130 | 1562 -> "varbit" 131 | 1563 -> "_varbit" 132 | 1700 -> "numeric" 133 | 1790 -> "refcursor" 134 | 2201 -> "_refcursor" 135 | 2202 -> "regprocedure" 136 | 2203 -> "regoper" 137 | 2204 -> "regoperator" 138 | 2205 -> "regclass" 139 | 2206 -> "regtype" 140 | 2207 -> "_regprocedure" 141 | 2208 -> "_regoper" 142 | 2209 -> "_regoperator" 143 | 2210 -> "_regclass" 144 | 2211 -> "_regtype" 145 | 2949 -> "_txid_snapshot" 146 | 2950 -> "uuid" 147 | 2951 -> "_uuid" 148 | 2970 -> "txid_snapshot" 149 | 3220 -> "pg_lsn" 150 | 3221 -> "_pg_lsn" 151 | 3361 -> "pg_ndistinct" 152 | 3402 -> "pg_dependencies" 153 | 3614 -> "tsvector" 154 | 3615 -> "tsquery" 155 | 3642 -> "gtsvector" 156 | 3643 -> "_tsvector" 157 | 3644 -> "_gtsvector" 158 | 3645 -> "_tsquery" 159 | 3734 -> "regconfig" 160 | 3735 -> "_regconfig" 161 | 3769 -> "regdictionary" 162 | 3770 -> "_regdictionary" 163 | 3802 -> "jsonb" 164 | 3807 -> "_jsonb" 165 | 3905 -> "_int4range" 166 | 3907 -> "_numrange" 167 | 3909 -> "_tsrange" 168 | 3911 -> "_tstzrange" 169 | 3913 -> "_daterange" 170 | 3927 -> "_int8range" 171 | 4089 -> "regnamespace" 172 | 4090 -> "_regnamespace" 173 | 4096 -> "regrole" 174 | 4097 -> "_regrole" 175 | _ -> type_id 176 | end 177 | end 178 | end 179 | -------------------------------------------------------------------------------- /example/wal_replication/lib/wal_replication/handler.ex: -------------------------------------------------------------------------------- 1 | defmodule WalReplication.Handler do 2 | use GenServer 3 | 4 | import PostgresReplication.Protocol 5 | import PostgresReplication.Decoder 6 | 7 | alias PostgresReplication.Protocol.{Write, KeepAlive} 8 | 9 | @behaviour PostgresReplication.Handler 10 | def start_link(opts) do 11 | GenServer.start_link(__MODULE__, opts, opts) 12 | end 13 | 14 | @impl true 15 | def init(opts) do 16 | opts = Keyword.delete(opts, :name) 17 | {:ok, conn} = Postgrex.start_link(opts) 18 | {:ok, %{pid: conn, operation: nil, relations: %{}}} 19 | end 20 | 21 | @impl true 22 | def call(message, %PostgresReplication{replication_slot_name: replication_slot_name}) 23 | when is_write(message) do 24 | %Write{message: message} = parse(message) 25 | [{pid, _}] = Registry.lookup(Registry.Handler, replication_slot_name) 26 | 27 | message 28 | |> decode_message() 29 | |> then(&send(pid, &1)) 30 | 31 | :noreply 32 | end 33 | 34 | def call(message, _state) when is_keep_alive(message) do 35 | %KeepAlive{reply: reply, wal_end: wal_end} = parse(message) 36 | 37 | message = 38 | case reply do 39 | :now -> standby(wal_end + 1, wal_end + 1, wal_end + 1, reply) 40 | :later -> hold() 41 | end 42 | 43 | {:reply, message} 44 | end 45 | 46 | # Fallback 47 | def call(_, _) do 48 | :no_reply 49 | end 50 | 51 | @impl true 52 | def handle_info(%PostgresReplication.Decoder.Messages.Begin{}, state) do 53 | {:noreply, state} 54 | end 55 | 56 | def handle_info(%PostgresReplication.Decoder.Messages.Relation{} = msg, state) do 57 | %PostgresReplication.Decoder.Messages.Relation{ 58 | id: id, 59 | namespace: namespace, 60 | name: name, 61 | columns: columns 62 | } = 63 | msg 64 | 65 | relation = %{namespace: namespace, name: name, columns: columns} 66 | relations = Map.put(state.relations, id, relation) 67 | state = Map.put(state, :relations, relations) 68 | {:noreply, state} 69 | end 70 | 71 | def handle_info(%PostgresReplication.Decoder.Messages.Insert{} = msg, state) do 72 | %PostgresReplication.Decoder.Messages.Insert{ 73 | tuple_data: tuple_data, 74 | relation_id: relation_id 75 | } = msg 76 | 77 | values = 78 | for {value, index} <- tuple_data |> Tuple.to_list() |> Enum.with_index() do 79 | case state.relations 80 | |> Map.get(relation_id) 81 | |> then(& &1.columns) 82 | |> Enum.at(index) do 83 | %{type: "text", name: name} -> {name, value} 84 | %{type: "int" <> _, name: name} -> {name, String.to_integer(value)} 85 | end 86 | end 87 | 88 | state = 89 | state 90 | |> Map.put(:values, values) 91 | |> Map.put(:relation_id, relation_id) 92 | |> Map.put(:operation, :insert) 93 | 94 | {:noreply, state} 95 | end 96 | 97 | def handle_info(%PostgresReplication.Decoder.Messages.Delete{} = msg, state) do 98 | %PostgresReplication.Decoder.Messages.Delete{ 99 | relation_id: relation_id, 100 | changed_key_tuple_data: changed_key_tuple_data 101 | } = msg 102 | 103 | values = 104 | for {value, index} <- changed_key_tuple_data |> Tuple.to_list() |> Enum.with_index(), 105 | value != nil do 106 | case state.relations 107 | |> Map.get(relation_id) 108 | |> then(& &1.columns) 109 | |> Enum.at(index) do 110 | %{type: "text", name: name} -> {name, value} 111 | %{type: "int" <> _, name: name} -> {name, String.to_integer(value)} 112 | end 113 | end 114 | 115 | state = 116 | state 117 | |> Map.put(:values, values) 118 | |> Map.put(:relation_id, relation_id) 119 | |> Map.put(:operation, :delete) 120 | 121 | {:noreply, state} 122 | end 123 | 124 | @impl true 125 | def handle_info(%PostgresReplication.Decoder.Messages.Update{} = message, state) do 126 | %PostgresReplication.Decoder.Messages.Update{ 127 | relation_id: relation_id, 128 | tuple_data: tuple_data 129 | } = message 130 | 131 | values = 132 | for {value, index} <- tuple_data |> Tuple.to_list() |> Enum.with_index() do 133 | case state.relations 134 | |> Map.get(relation_id) 135 | |> then(& &1.columns) 136 | |> Enum.at(index) do 137 | %{type: "text", name: name} -> {name, value} 138 | %{type: "int" <> _, name: name} -> {name, String.to_integer(value)} 139 | end 140 | end 141 | 142 | state = 143 | state 144 | |> Map.put(:values, values) 145 | |> Map.put(:relation_id, relation_id) 146 | |> Map.put(:operation, :update) 147 | 148 | {:noreply, state} 149 | end 150 | 151 | def handle_info( 152 | %PostgresReplication.Decoder.Messages.Commit{}, 153 | %{pid: pid, relation_id: relation_id} = state 154 | ) do 155 | case state.operation do 156 | :insert -> 157 | column_names = Enum.map(state.values, &elem(&1, 0)) 158 | values = Enum.map(state.values, &elem(&1, 1)) 159 | 160 | arg_index = 161 | column_names 162 | |> Enum.with_index(1) 163 | |> Enum.map_join(",", fn {_, index} -> "$#{index}" end) 164 | 165 | schema = state.relations |> Map.get(relation_id) |> then(& &1.namespace) 166 | table = state.relations |> Map.get(relation_id) |> then(& &1.name) 167 | 168 | query = 169 | "INSERT INTO #{schema}.#{table} (#{Enum.join(column_names, ", ")}) VALUES (#{arg_index})" 170 | 171 | Postgrex.query!(pid, query, values) 172 | 173 | :delete -> 174 | column_names = Enum.map(state.values, &elem(&1, 0)) 175 | values = Enum.map(state.values, &elem(&1, 1)) 176 | 177 | conditions = 178 | column_names 179 | |> Enum.with_index(1) 180 | |> Enum.map_join(" AND ", fn {name, index} -> "#{name} = $#{index}" end) 181 | 182 | schema = state.relations |> Map.get(relation_id) |> then(& &1.namespace) 183 | table = state.relations |> Map.get(relation_id) |> then(& &1.name) 184 | 185 | query = 186 | "DELETE FROM #{schema}.#{table} WHERE #{conditions}" 187 | 188 | Postgrex.query!(pid, query, values) 189 | 190 | :update -> 191 | column_names = Enum.map(state.values, &elem(&1, 0)) 192 | values = Enum.map(state.values, &elem(&1, 1)) 193 | 194 | set_clause = 195 | column_names 196 | |> Enum.with_index(1) 197 | |> Enum.map_join(", ", fn {name, index} -> "#{name} = $#{index}" end) 198 | 199 | schema = state.relations |> Map.get(relation_id) |> then(& &1.namespace) 200 | table = state.relations |> Map.get(relation_id) |> then(& &1.name) 201 | 202 | query = 203 | "UPDATE #{schema}.#{table} SET #{set_clause} WHERE id = $1" 204 | 205 | Postgrex.query!(pid, query, values) 206 | 207 | _ -> 208 | :ok 209 | end 210 | 211 | {:noreply, %{pid: pid, relations: state.relations}} 212 | end 213 | end 214 | -------------------------------------------------------------------------------- /lib/postgres_replication.ex: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication do 2 | @moduledoc """ 3 | PostgresReplication is a module that provides a way to stream data from a PostgreSQL database using logical replication. 4 | 5 | ## Struct parameters 6 | * `connection_opts` - The connection options to connect to the database. 7 | * `table` - The table to replicate. If `:all` is passed, it will replicate all tables. 8 | * `schema` - The schema of the table to replicate. If not provided, it will use the `public` schema. If `:all` is passed, this option is ignored. 9 | * `opts` - The options to pass to this module 10 | * `step` - The current step of the replication process 11 | * `publication_name` - The name of the publication to create. If not provided, it will use the schema and table name. 12 | * `replication_slot_name` - The name of the replication slot to create. If not provided, it will use the schema and table name. 13 | * `output_plugin` - The output plugin to use. Default is `pgoutput`. 14 | * `output_plugin_options` - The options to pass to the output plugin If you use :publication_name as the value of the key, the lib will automatically set the generated or set value of `publication_name`. Default is [proto_version: "1", publication_names: :publication_name]. 15 | * `handler_module` - The module that will handle the data received from the replication stream. 16 | * `target_pid` - The PID of the parent process that will receive the data. 17 | 18 | """ 19 | use Postgrex.ReplicationConnection 20 | require Logger 21 | alias PostgresReplication.Handler 22 | 23 | @default_opts [ 24 | auto_reconnect: true, 25 | sync_connect: true 26 | ] 27 | @type t :: %__MODULE__{ 28 | connection_opts: Keyword.t(), 29 | table: String.t(), 30 | schema: String.t(), 31 | opts: Keyword.t(), 32 | step: 33 | :disconnected 34 | | :check_replication_slot 35 | | :create_publication 36 | | :check_publication 37 | | :create_slot 38 | | :start_replication_slot 39 | | :streaming, 40 | publication_name: String.t(), 41 | replication_slot_name: String.t(), 42 | output_plugin: String.t(), 43 | output_plugin_options: Keyword.t(), 44 | handler_module: Handler.t(), 45 | metadata: map() 46 | } 47 | defstruct connection_opts: nil, 48 | table: nil, 49 | schema: "public", 50 | opts: [], 51 | step: :disconnected, 52 | publication_name: nil, 53 | replication_slot_name: nil, 54 | output_plugin: "pgoutput", 55 | output_plugin_options: [proto_version: "1", publication_names: :publication_name], 56 | handler_module: nil, 57 | metadata: %{} 58 | 59 | def start_link(%__MODULE__{opts: opts, connection_opts: connection_opts} = attrs) do 60 | Postgrex.ReplicationConnection.start_link( 61 | __MODULE__, 62 | attrs, 63 | @default_opts |> Keyword.merge(opts) |> Keyword.merge(connection_opts) 64 | ) 65 | end 66 | 67 | @impl true 68 | def init(attrs) do 69 | Logger.info( 70 | "Initializing connection with the status: #{inspect(attrs |> Map.from_struct() |> Map.drop([:connection_opts]))}" 71 | ) 72 | 73 | publication_name = publication_name(attrs) 74 | replication_slot_name = replication_slot_name(attrs) 75 | 76 | {:ok, 77 | %{ 78 | attrs 79 | | step: :disconnected, 80 | publication_name: publication_name, 81 | replication_slot_name: replication_slot_name 82 | }} 83 | end 84 | 85 | @impl true 86 | def handle_connect(state) do 87 | replication_slot_name = replication_slot_name(state) 88 | Logger.info("Checking if replication slot #{replication_slot_name} exists") 89 | 90 | query = 91 | "SELECT * FROM pg_replication_slots WHERE slot_name = '#{replication_slot_name}'" 92 | 93 | {:query, query, %{state | step: :check_replication_slot}} 94 | end 95 | 96 | @impl true 97 | def handle_result( 98 | [%Postgrex.Result{num_rows: 1}], 99 | %__MODULE__{step: :check_replication_slot} = state 100 | ) do 101 | {:query, "SELECT 1", %{state | step: :check_publication}} 102 | end 103 | 104 | def handle_result( 105 | [%Postgrex.Result{num_rows: 0}], 106 | %__MODULE__{step: :check_replication_slot} = state 107 | ) do 108 | %__MODULE__{ 109 | output_plugin: output_plugin, 110 | replication_slot_name: replication_slot_name, 111 | step: :check_replication_slot 112 | } = state 113 | 114 | Logger.info("Create replication slot #{replication_slot_name} using plugin #{output_plugin}") 115 | 116 | query = 117 | "CREATE_REPLICATION_SLOT #{replication_slot_name} TEMPORARY LOGICAL #{output_plugin} NOEXPORT_SNAPSHOT" 118 | 119 | {:query, query, %{state | step: :check_publication}} 120 | end 121 | 122 | def handle_result( 123 | [%Postgrex.Result{}], 124 | %__MODULE__{step: :check_publication} = state 125 | ) do 126 | %__MODULE__{table: table, schema: schema, publication_name: publication_name} = state 127 | 128 | Logger.info("Check publication #{publication_name} for table #{schema}.#{table} exists") 129 | query = "SELECT * FROM pg_publication WHERE pubname = '#{publication_name}'" 130 | 131 | {:query, query, %{state | step: :create_publication}} 132 | end 133 | 134 | def handle_result( 135 | [%Postgrex.Result{num_rows: 0}], 136 | %__MODULE__{step: :create_publication, table: :all} = state 137 | ) do 138 | %{publication_name: publication_name} = state 139 | Logger.info("Create publication #{publication_name} for all tables") 140 | 141 | query = 142 | "CREATE PUBLICATION #{publication_name} FOR ALL TABLES" 143 | 144 | {:query, query, %{state | step: :start_replication_slot}} 145 | end 146 | 147 | def handle_result( 148 | [%Postgrex.Result{num_rows: 0}], 149 | %__MODULE__{step: :create_publication} = state 150 | ) do 151 | %__MODULE__{ 152 | table: table, 153 | schema: schema, 154 | publication_name: publication_name 155 | } = state 156 | 157 | Logger.info("Create publication #{publication_name} for table #{schema}.#{table}") 158 | 159 | query = 160 | "CREATE PUBLICATION #{publication_name} FOR TABLE #{schema}.#{table}" 161 | 162 | {:query, query, %{state | step: :start_replication_slot}} 163 | end 164 | 165 | def handle_result( 166 | [%Postgrex.Result{num_rows: 1}], 167 | %__MODULE__{step: :create_publication} = state 168 | ) do 169 | {:query, "SELECT 1", %{state | step: :start_replication_slot}} 170 | end 171 | 172 | @impl true 173 | def handle_result( 174 | [%Postgrex.Result{}], 175 | %__MODULE__{step: :start_replication_slot} = state 176 | ) do 177 | %__MODULE__{ 178 | replication_slot_name: replication_slot_name, 179 | publication_name: publication_name, 180 | output_plugin: output_plugin, 181 | output_plugin_options: output_plugin_options 182 | } = state 183 | 184 | Logger.info( 185 | "Starting stream replication for slot #{replication_slot_name} using the #{output_plugin} plugin with the options: #{inspect(output_plugin_options)}" 186 | ) 187 | 188 | output_plugin_options = 189 | output_plugin_options 190 | |> Enum.map_join(", ", fn 191 | {k, :publication_name} -> "#{k} '#{publication_name}'" 192 | {k, v} -> "#{k} '#{v}'" 193 | end) 194 | |> String.trim() 195 | |> then(fn 196 | "" -> "" 197 | config -> " (#{config})" 198 | end) 199 | 200 | query = 201 | "START_REPLICATION SLOT #{replication_slot_name} LOGICAL 0/0#{output_plugin_options}" 202 | 203 | {:stream, query, [], %{state | step: :streaming}} 204 | end 205 | 206 | @impl true 207 | def handle_disconnect(state) do 208 | Logger.error( 209 | "Disconnected from the server: #{inspect(state |> Map.from_struct() |> Map.drop([:connection_opts]))}" 210 | ) 211 | 212 | {:noreply, %{state | step: :disconnected}} 213 | end 214 | 215 | @impl true 216 | def handle_data(data, state) do 217 | %__MODULE__{handler_module: handler_module} = state 218 | 219 | case handler_module.call(data, state) do 220 | {:reply, messages} -> {:noreply, messages, state} 221 | :noreply -> {:noreply, state} 222 | end 223 | end 224 | 225 | def publication_name(%__MODULE__{publication_name: nil, table: :all}) do 226 | "all_table_publication" 227 | end 228 | 229 | def publication_name(%__MODULE__{publication_name: nil, table: table, schema: schema}) do 230 | "#{schema}_#{table}_publication" 231 | end 232 | 233 | def publication_name(%__MODULE__{publication_name: publication_name}) do 234 | publication_name 235 | end 236 | 237 | def replication_slot_name(%__MODULE__{replication_slot_name: nil, table: :all}) do 238 | "all_table_slot" 239 | end 240 | 241 | def replication_slot_name(%__MODULE__{replication_slot_name: nil, table: table, schema: schema}) do 242 | "#{schema}_#{table}_replication_slot" 243 | end 244 | 245 | def replication_slot_name(%__MODULE__{replication_slot_name: replication_slot_name}) do 246 | replication_slot_name 247 | end 248 | end 249 | -------------------------------------------------------------------------------- /lib/postgres_replication/plugin/pgoutput/decoder.ex: -------------------------------------------------------------------------------- 1 | defmodule PostgresReplication.Plugin.Pgoutput.Decoder do 2 | @moduledoc """ 3 | Functions for decoding different types of logical replication messages. 4 | """ 5 | defmodule Messages do 6 | @moduledoc """ 7 | Different types of logical replication messages from Postgres 8 | """ 9 | defmodule Begin do 10 | @moduledoc """ 11 | Struct representing the BEGIN message in PostgreSQL's logical decoding output. 12 | 13 | * `final_lsn` - The LSN of the commit that this transaction ended at. 14 | * `commit_timestamp` - The timestamp of the commit that this transaction ended at. 15 | * `xid` - The transaction ID of this transaction. 16 | """ 17 | defstruct [:final_lsn, :commit_timestamp, :xid] 18 | end 19 | 20 | defmodule Commit do 21 | @moduledoc """ 22 | Struct representing the COMMIT message in PostgreSQL's logical decoding output. 23 | 24 | * `flags` - Bitmask of flags associated with this commit. 25 | * `lsn` - The LSN of the commit. 26 | * `end_lsn` - The LSN of the next record in the WAL stream. 27 | * `commit_timestamp` - The timestamp of the commit. 28 | """ 29 | defstruct [:flags, :lsn, :end_lsn, :commit_timestamp] 30 | end 31 | 32 | defmodule Origin do 33 | @moduledoc """ 34 | Struct representing the ORIGIN message in PostgreSQL's logical decoding output. 35 | 36 | * `origin_commit_lsn` - The LSN of the commit in the database that the change originated from. 37 | * `name` - The name of the origin. 38 | """ 39 | defstruct [:origin_commit_lsn, :name] 40 | end 41 | 42 | defmodule Relation do 43 | @moduledoc """ 44 | Struct representing the RELATION message in PostgreSQL's logical decoding output. 45 | 46 | * `id` - The OID of the relation. 47 | * `namespace` - The OID of the namespace that the relation belongs to. 48 | * `name` - The name of the relation. 49 | * `replica_identity` - The replica identity setting of the relation. 50 | * `columns` - A list of columns in the relation. 51 | """ 52 | defstruct [:id, :namespace, :name, :replica_identity, :columns] 53 | 54 | defmodule Column do 55 | @moduledoc """ 56 | Struct representing a column in a relation. 57 | 58 | * `flags` - Bitmask of flags associated with this column. 59 | * `name` - The name of the column. 60 | * `type` - The OID of the data type of the column. 61 | * `type_modifier` - The type modifier of the column. 62 | """ 63 | defstruct [:flags, :name, :type, :type_modifier] 64 | end 65 | end 66 | 67 | defmodule Insert do 68 | @moduledoc """ 69 | Struct representing the INSERT message in PostgreSQL's logical decoding output. 70 | 71 | * `relation_id` - The OID of the relation that the tuple was inserted into. 72 | * `tuple_data` - The data of the inserted tuple. 73 | """ 74 | defstruct [:relation_id, :tuple_data] 75 | end 76 | 77 | defmodule Update do 78 | @moduledoc """ 79 | Struct representing the UPDATE message in PostgreSQL's logical decoding output. 80 | 81 | * `relation_id` - The OID of the relation that the tuple was updated in. 82 | * `changed_key_tuple_data` - The data of the tuple with the old key values. 83 | * `old_tuple_data` - The data of the tuple before the update. 84 | * `tuple_data` - The data of the tuple after the update. 85 | """ 86 | defstruct [:relation_id, :changed_key_tuple_data, :old_tuple_data, :tuple_data] 87 | end 88 | 89 | defmodule Delete do 90 | @moduledoc """ 91 | Struct representing the DELETE message in PostgreSQL's logical decoding output. 92 | 93 | * `relation_id` - The OID of the relation that the tuple was deleted from. 94 | * `changed_key_tuple_data` - The data of the tuple with the old key values. 95 | * `old_tuple_data` - The data of the tuple before the delete. 96 | """ 97 | defstruct [:relation_id, :changed_key_tuple_data, :old_tuple_data] 98 | end 99 | 100 | defmodule Truncate do 101 | @moduledoc """ 102 | Struct representing the TRUNCATE message in PostgreSQL's logical decoding output. 103 | 104 | * `number_of_relations` - The number of truncated relations. 105 | * `options` - Additional options provided when truncating the relations. 106 | * `truncated_relations` - List of relations that have been truncated. 107 | """ 108 | defstruct [:number_of_relations, :options, :truncated_relations] 109 | end 110 | 111 | defmodule Type do 112 | @moduledoc """ 113 | Struct representing the TYPE message in PostgreSQL's logical decoding output. 114 | 115 | * `id` - The OID of the type. 116 | * `namespace` - The namespace of the type. 117 | * `name` - The name of the type. 118 | """ 119 | defstruct [:id, :namespace, :name] 120 | end 121 | 122 | defmodule Unsupported do 123 | @moduledoc """ 124 | Struct representing an unsupported message in PostgreSQL's logical decoding output. 125 | 126 | * `data` - The raw data of the unsupported message. 127 | """ 128 | defstruct [:data] 129 | end 130 | end 131 | 132 | require Logger 133 | 134 | @pg_epoch DateTime.from_iso8601("2000-01-01T00:00:00Z") 135 | 136 | alias Messages.{ 137 | Begin, 138 | Commit, 139 | Delete, 140 | Insert, 141 | Origin, 142 | Relation, 143 | Relation.Column, 144 | Truncate, 145 | Type, 146 | Unsupported, 147 | Update 148 | } 149 | 150 | alias PostgresReplication.Protocol.Write 151 | 152 | alias PostgresReplication.Plugin.Pgoutput.OidDatabase 153 | 154 | @doc """ 155 | Parses logical replication messages from Postgres 156 | 157 | ## Examples 158 | 159 | iex> decode_message(<<73, 0, 0, 96, 0, 78, 0, 2, 116, 0, 0, 0, 3, 98, 97, 122, 116, 0, 0, 0, 3, 53, 54, 48>>) 160 | %Realtime.Adapters.Postgres.Decoder.Messages.Insert{relation_id: 24576, tuple_data: {"baz", "560"}} 161 | 162 | """ 163 | def decode_message(%Write{message: message}) when is_binary(message) do 164 | decode_message(message) 165 | end 166 | 167 | def decode_message(message) when is_binary(message) do 168 | decode_message_impl(message) 169 | end 170 | 171 | defp decode_message_impl(<<"B", lsn::binary-8, timestamp::integer-64, xid::integer-32>>) do 172 | %Begin{ 173 | final_lsn: decode_lsn(lsn), 174 | commit_timestamp: pgtimestamp_to_timestamp(timestamp), 175 | xid: xid 176 | } 177 | end 178 | 179 | defp decode_message_impl( 180 | <<"C", _flags::binary-1, lsn::binary-8, end_lsn::binary-8, timestamp::integer-64>> 181 | ) do 182 | %Commit{ 183 | flags: [], 184 | lsn: decode_lsn(lsn), 185 | end_lsn: decode_lsn(end_lsn), 186 | commit_timestamp: pgtimestamp_to_timestamp(timestamp) 187 | } 188 | end 189 | 190 | defp decode_message_impl(<<"O", lsn::binary-8, name::binary>>) do 191 | %Origin{ 192 | origin_commit_lsn: decode_lsn(lsn), 193 | name: name 194 | } 195 | end 196 | 197 | defp decode_message_impl(<<"R", id::integer-32, rest::binary>>) do 198 | [ 199 | namespace 200 | | [name | [<>]] 201 | ] = String.split(rest, <<0>>, parts: 3) 202 | 203 | friendly_replica_identity = 204 | case replica_identity do 205 | "d" -> :default 206 | "n" -> :nothing 207 | "f" -> :all_columns 208 | "i" -> :index 209 | end 210 | 211 | %Relation{ 212 | id: id, 213 | namespace: namespace, 214 | name: name, 215 | replica_identity: friendly_replica_identity, 216 | columns: decode_columns(columns) 217 | } 218 | end 219 | 220 | defp decode_message_impl( 221 | <<"I", relation_id::integer-32, "N", number_of_columns::integer-16, tuple_data::binary>> 222 | ) do 223 | {<<>>, decoded_tuple_data} = decode_tuple_data(tuple_data, number_of_columns) 224 | 225 | %Insert{ 226 | relation_id: relation_id, 227 | tuple_data: decoded_tuple_data 228 | } 229 | end 230 | 231 | defp decode_message_impl( 232 | <<"U", relation_id::integer-32, "N", number_of_columns::integer-16, tuple_data::binary>> 233 | ) do 234 | {<<>>, decoded_tuple_data} = decode_tuple_data(tuple_data, number_of_columns) 235 | 236 | %Update{ 237 | relation_id: relation_id, 238 | tuple_data: decoded_tuple_data 239 | } 240 | end 241 | 242 | defp decode_message_impl( 243 | <<"U", relation_id::integer-32, key_or_old::binary-1, number_of_columns::integer-16, 244 | tuple_data::binary>> 245 | ) 246 | when key_or_old == "O" or key_or_old == "K" do 247 | {<<"N", new_number_of_columns::integer-16, new_tuple_binary::binary>>, old_decoded_tuple_data} = 248 | decode_tuple_data(tuple_data, number_of_columns) 249 | 250 | {<<>>, decoded_tuple_data} = decode_tuple_data(new_tuple_binary, new_number_of_columns) 251 | 252 | base_update_msg = %Update{ 253 | relation_id: relation_id, 254 | tuple_data: decoded_tuple_data 255 | } 256 | 257 | case key_or_old do 258 | "K" -> Map.put(base_update_msg, :changed_key_tuple_data, old_decoded_tuple_data) 259 | "O" -> Map.put(base_update_msg, :old_tuple_data, old_decoded_tuple_data) 260 | end 261 | end 262 | 263 | defp decode_message_impl( 264 | <<"D", relation_id::integer-32, key_or_old::binary-1, number_of_columns::integer-16, 265 | tuple_data::binary>> 266 | ) 267 | when key_or_old == "K" or key_or_old == "O" do 268 | {<<>>, decoded_tuple_data} = decode_tuple_data(tuple_data, number_of_columns) 269 | 270 | base_delete_msg = %Delete{ 271 | relation_id: relation_id 272 | } 273 | 274 | case key_or_old do 275 | "K" -> Map.put(base_delete_msg, :changed_key_tuple_data, decoded_tuple_data) 276 | "O" -> Map.put(base_delete_msg, :old_tuple_data, decoded_tuple_data) 277 | end 278 | end 279 | 280 | defp decode_message_impl( 281 | <<"T", number_of_relations::integer-32, options::integer-8, column_ids::binary>> 282 | ) do 283 | truncated_relations = 284 | for relation_id_bin <- column_ids |> :binary.bin_to_list() |> Enum.chunk_every(4), 285 | do: relation_id_bin |> :binary.list_to_bin() |> :binary.decode_unsigned() 286 | 287 | decoded_options = 288 | case options do 289 | 0 -> [] 290 | 1 -> [:cascade] 291 | 2 -> [:restart_identity] 292 | 3 -> [:cascade, :restart_identity] 293 | end 294 | 295 | %Truncate{ 296 | number_of_relations: number_of_relations, 297 | options: decoded_options, 298 | truncated_relations: truncated_relations 299 | } 300 | end 301 | 302 | defp decode_message_impl(<<"Y", data_type_id::integer-32, namespace_and_name::binary>>) do 303 | [namespace, name_with_null] = :binary.split(namespace_and_name, <<0>>) 304 | name = String.slice(name_with_null, 0..-2//1) 305 | 306 | %Type{ 307 | id: data_type_id, 308 | namespace: namespace, 309 | name: name 310 | } 311 | end 312 | 313 | defp decode_message_impl(binary), do: %Unsupported{data: binary} 314 | 315 | defp decode_tuple_data(binary, columns_remaining, accumulator \\ []) 316 | 317 | defp decode_tuple_data(remaining_binary, 0, accumulator) when is_binary(remaining_binary), 318 | do: {remaining_binary, accumulator |> Enum.reverse() |> List.to_tuple()} 319 | 320 | defp decode_tuple_data(<<"n", rest::binary>>, columns_remaining, accumulator), 321 | do: decode_tuple_data(rest, columns_remaining - 1, [nil | accumulator]) 322 | 323 | defp decode_tuple_data(<<"u", rest::binary>>, columns_remaining, accumulator), 324 | do: decode_tuple_data(rest, columns_remaining - 1, [:unchanged_toast | accumulator]) 325 | 326 | defp decode_tuple_data( 327 | <<"t", column_length::integer-32, rest::binary>>, 328 | columns_remaining, 329 | accumulator 330 | ), 331 | do: 332 | decode_tuple_data( 333 | :erlang.binary_part(rest, {byte_size(rest), -(byte_size(rest) - column_length)}), 334 | columns_remaining - 1, 335 | [ 336 | :erlang.binary_part(rest, {0, column_length}) | accumulator 337 | ] 338 | ) 339 | 340 | defp decode_columns(binary, accumulator \\ []) 341 | defp decode_columns(<<>>, accumulator), do: Enum.reverse(accumulator) 342 | 343 | defp decode_columns(<>, accumulator) do 344 | [name | [<>]] = 345 | String.split(rest, <<0>>, parts: 2) 346 | 347 | decoded_flags = 348 | case flags do 349 | 1 -> [:key] 350 | _ -> [] 351 | end 352 | 353 | decode_columns(columns, [ 354 | %Column{ 355 | name: name, 356 | flags: decoded_flags, 357 | type: OidDatabase.name_for_type_id(data_type_id), 358 | # type: data_type_id, 359 | type_modifier: type_modifier 360 | } 361 | | accumulator 362 | ]) 363 | end 364 | 365 | defp pgtimestamp_to_timestamp(microsecond_offset) when is_integer(microsecond_offset) do 366 | {:ok, epoch, 0} = @pg_epoch 367 | 368 | DateTime.add(epoch, microsecond_offset, :microsecond) 369 | end 370 | 371 | defp decode_lsn(<>), 372 | do: {xlog_file, xlog_offset} 373 | end 374 | --------------------------------------------------------------------------------