├── .formatter.exs ├── .github └── workflows │ └── pull.yaml ├── .gitignore ├── .logo_for_docs.png ├── .tool-versions ├── CHANGELOG.md ├── LICENSE ├── README.md ├── _plts └── .gitkeep ├── config └── config.exs ├── lib ├── carbonite.ex ├── carbonite │ ├── change.ex │ ├── migrations.ex │ ├── migrations │ │ ├── helper.ex │ │ ├── v1.ex │ │ ├── v10.ex │ │ ├── v11.ex │ │ ├── v12.ex │ │ ├── v2.ex │ │ ├── v3.ex │ │ ├── v4.ex │ │ ├── v5.ex │ │ ├── v6.ex │ │ ├── v7.ex │ │ ├── v8.ex │ │ ├── v9.ex │ │ └── version.ex │ ├── multi.ex │ ├── outbox.ex │ ├── prefix.ex │ ├── query.ex │ ├── schema.ex │ ├── transaction.ex │ └── trigger.ex └── mix │ └── tasks │ └── carbonite.gen.initial_migration.ex ├── mix.exs ├── mix.lock └── test ├── capture_test.exs ├── carbonite ├── change_test.exs ├── migrations_test.exs ├── multi_test.exs ├── outbox_test.exs ├── query_test.exs └── transaction_test.exs ├── carbonite_test.exs ├── support ├── api_case.ex ├── rabbit.ex ├── test_repo.ex └── test_repo │ └── migrations │ ├── 20210704201537_create_rabbits.exs │ ├── 20210704201627_install_carbonite.exs │ ├── 20221011201645_install_carbonite_alternate_test_schema.exs │ ├── 20221102120000_create_deferred_rabbits.exs │ └── 20250101140000_create_weird_character_rabbits.exs └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.github/workflows/pull.yaml: -------------------------------------------------------------------------------- 1 | name: QA 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | 8 | permissions: read-all 9 | 10 | env: 11 | MIX_ENV: test 12 | 13 | jobs: 14 | test: 15 | runs-on: ubuntu-latest 16 | name: test 17 | strategy: 18 | matrix: 19 | # Earliest combination possible: 20 | # - ubuntu-latest (Ubuntu 22) mandates OTP >= 24 21 | # - dialyxir uses `Kernel.then` and hence mandates Elixir >= 1.12 22 | lang: [{otp: '24.3.4.13', elixir: '1.12.3'}, {otp: '26.0.2', elixir: '1.15.5'}, {otp: '26.2.3', elixir: '1.17.1'}] 23 | postgres: ['13.12', '14.5', '16.0'] 24 | services: 25 | postgres: 26 | image: postgres:${{matrix.postgres}} 27 | env: 28 | POSTGRES_PASSWORD: postgres 29 | POSTGRES_USER: postgres 30 | POSTGRES_DB: carbonite_test 31 | options: >- 32 | --health-cmd pg_isready 33 | --health-interval 10s 34 | --health-timeout 5s 35 | --health-retries 5 36 | ports: 37 | - 5432:5432 38 | steps: 39 | - uses: actions/checkout@v2 40 | - uses: erlef/setup-beam@v1 41 | with: 42 | otp-version: ${{matrix.lang.otp}} 43 | elixir-version: ${{matrix.lang.elixir}} 44 | - uses: actions/cache@v4 45 | with: 46 | path: deps 47 | key: v2-${{ runner.os }}-deps-${{ matrix.lang.otp }}-${{ matrix.lang.elixir }}-${{ env.MIX_ENV }}-${{ hashFiles('**/mix.lock') }} 48 | restore-keys: | 49 | v2-${{ runner.os }}-deps-${{ matrix.lang.otp }}-${{ matrix.lang.elixir }}-${{ env.MIX_ENV }} 50 | v2-${{ runner.os }}-deps-${{ matrix.lang.otp }}-${{ matrix.lang.elixir }} 51 | v2-${{ runner.os }}-deps 52 | - uses: actions/cache@v4 53 | with: 54 | path: _build 55 | key: v2-${{ runner.os }}-build-${{ matrix.lang.otp }}-${{ matrix.lang.elixir }}-${{ env.MIX_ENV }}-${{ hashFiles('**/mix.lock') }} 56 | restore-keys: | 57 | v2-${{ runner.os }}-build-${{ matrix.lang.otp }}-${{ matrix.lang.elixir }}-${{ env.MIX_ENV }} 58 | v2-${{ runner.os }}-build-${{ matrix.lang.otp }}-${{ matrix.lang.elixir}} 59 | v2-${{ runner.os }}-build 60 | - run: mix deps.get 61 | - run: mix do ecto.create, ecto.migrate 62 | - run: mix compile 63 | - run: mix test 64 | 65 | lint: 66 | runs-on: ubuntu-latest 67 | name: lint 68 | env: 69 | otp_version: 26.0.2 70 | elixir_version: 1.15.5 71 | steps: 72 | - uses: actions/checkout@v2 73 | - uses: erlef/setup-beam@v1 74 | with: 75 | otp-version: ${{ env.otp_version }} 76 | elixir-version: ${{ env.elixir_version }} 77 | - uses: actions/cache@v4 78 | with: 79 | path: deps 80 | key: v2-${{ runner.os }}-deps-${{ env.otp_version }}-${{ env.elixir_version }}-${{ env.MIX_ENV }}-${{ hashFiles('**/mix.lock') }} 81 | restore-keys: | 82 | v2-${{ runner.os }}-deps-${{ env.otp_version }}-${{ env.elixir_version }}-${{ env.MIX_ENV }} 83 | v2-${{ runner.os }}-deps-${{ env.otp_version }}-${{ env.elixir_version }} 84 | v2-${{ runner.os }}-deps 85 | - uses: actions/cache@v4 86 | with: 87 | path: _build 88 | key: v2-${{ runner.os }}-build-${{ env.otp_version }}-${{ env.elixir_version }}-${{ env.MIX_ENV }}-${{ hashFiles('**/mix.lock') }} 89 | restore-keys: | 90 | v2-${{ runner.os }}-build-${{ env.otp_version }}-${{ env.elixir_version }}-${{ env.MIX_ENV }} 91 | v2-${{ runner.os }}-build-${{ env.otp_version }}-${{ env.elixir_version }} 92 | v2-${{ runner.os }}-build 93 | - uses: actions/cache@v4 94 | with: 95 | path: _plts 96 | key: v2-${{ runner.os }}-plts-${{ env.otp_version }}-${{ env.elixir_version }}-${{ env.MIX_ENV }}-${{ hashFiles('**/mix.lock') }} 97 | restore-keys: | 98 | v2-${{ runner.os }}-plts-${{ env.otp_version }}-${{ env.elixir_version }}-${{ env.MIX_ENV }} 99 | v2-${{ runner.os }}-plts-${{ env.otp_version }}-${{ env.elixir_version }} 100 | v2-${{ runner.os }}-plts 101 | - run: mix deps.get 102 | - run: mix compile --warnings-as-errors 103 | - run: mix lint 104 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _build/ 2 | _plts/ 3 | cover/ 4 | deps/ 5 | doc/ 6 | .fetch 7 | erl_crash.dump 8 | *.ez 9 | carbonite-*.tar 10 | tmp/ 11 | -------------------------------------------------------------------------------- /.logo_for_docs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bitcrowd/carbonite/0df3c1061d768f1826ab0f22c77d5144648315a6/.logo_for_docs.png -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | elixir 1.16.3 2 | erlang 26.2.3 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## Unreleased 2 | 3 | ### Added 4 | 5 | - Better error handling when the trigger record is non-existent. 6 | 7 | ## [0.15.0] - 2025-01-01 8 | 9 | ### Fixed 10 | 11 | - Quote table prefix and name correctly in `Carbonite.Migrations` functions. (@ravensiris) 12 | 13 | ## [0.14.2] - 2024-08-21 14 | 15 | **New migration patches:** 11 16 | 17 | ### Fixed 18 | 19 | - Reverted refactoring from v0.12 causing an exception in Postgres 14.0 to 14.5 when inserting into or deleting from multiple tables within a single transaction. 20 | 21 | ## [0.14.1] - 2024-08-20 22 | 23 | ### Added 24 | 25 | - Add `:order_by` option to `Carbonite.Query.changes/2` to either disable default ordering or specify the order. 26 | 27 | ## [0.14.0] - 2024-07-25 28 | 29 | **New migration patches:** 10 30 | 31 | ### Fixed 32 | 33 | - Improve performance of test setups by using a transaction-local variable in `override_mode/2` . The previous implementation could cause parallel tests to wait for row locks on the `triggers` table. 34 | - Replaced calls to `Ecto.Migration.timestamps/1` in migrations with explicit `add(:inserted_at, ...)` calls. This ensures that we don't be affected by users' `@migration_timestamps` options. 35 | 36 | ## [0.13.0] - 2024-06-27 37 | 38 | **New migration patches:** 9 39 | 40 | ### Fixed 41 | 42 | - Fix `set_transaction_id` trigger not using the PK index on `transactions` due to being dependent on `CURRVAL` for every row. Replaced with a CTE the `EXIST` query is now an index-only scan as it was intended to be. As a consequence, scalability of inserts into the `transactions` table has been greatly improved. 43 | 44 | ## [0.12.1] - 2024-05-21 45 | 46 | ### Fixed 47 | 48 | - Fix incomplete rollback logic in migration 8, leading to leftover procedures in the carbonite prefix when rolling back all the way (e.g. `Carbonite.Migrations.down(8..1)`). (#100, @xadhoom) 49 | 50 | ## [0.12.0] - 2024-05-17 51 | 52 | **New migration patches:** 8 53 | 54 | ### Fixed 55 | 56 | - Make the trigger function correctly mask the `filtered_columns` in the `changed_from` JSON object if `store_changed_from` is set. (#96) 57 | 58 | ⚠️ Unfortunately, this bug caused Carbonite to persist potentially sensitive data in the `changed_from` object even though they were listed in the `filtered_columns`. All users who combine `store_changed_from: true` with a `filtered_columns` setting should upgrade and assess the impact of this potential data leak. 59 | 60 | ## [0.11.1] - 2024-04-25 61 | 62 | ### Fixed 63 | 64 | - Migration: Creating table `changes` failed when Ecto's `@migration_foreign_key` is set to `[column: ]`. 65 | 66 | ## [0.11.0] - 2023-11-14 67 | 68 | **New migration patches:** 7 69 | 70 | ### Added 71 | 72 | - Add a new index on `transaction_id` in the `changes` table, to speed up queries for a transaction's changes and the `purge` operation. The existing `changes_transaction_id_index` (on `transaction_xact_id` column) has been renamed to `changes_transaction_xact_id_index`. 73 | - Allow ranges of migration patches in `Carbonite.Migrations.up/2` / `down/2`. 74 | 75 | ## [0.10.0] - 2023-09-20 76 | 77 | ### Fixed 78 | 79 | * Fixed a minor incompatibility with recently released Postgres 16.0. 80 | 81 | ## [0.9.0] - 2023-04-27 82 | 83 | **New migration patches:** 6 84 | 85 | ### Fixed 86 | 87 | * Correctly detect changes to array fields. 88 | 89 | Previously, detection of changes was done via the `@>` operator to test "containment" of a `{col: old_value}` JSON object in the JSON object of the new record. Unfortunately, Postgres' "jsonb containment" (see [docs](https://www.postgresql.org/docs/15/datatype-json.html#JSON-CONTAINMENT)) views array subsets as contained within their subsets, as well as arrays in different orders to be contained within each other. Both of these cases we want to track as a changed value. 90 | 91 | ⚠️ This bug caused Carbonite to not identify a changed array field correctly, meaning it may not have been listed in the `changed` and `changed_from` columns of the `Carbonite.Change` record. Unfortunately, this also means that **versions may have been discarded entirely** if no other fields of a record were updated at the same time. 92 | 93 | ## [0.8.0] - 2023-03-27 94 | 95 | ### Added 96 | 97 | * Add `:to` option to `Carbonite.override_mode/2` to specify an explicit target mode. Useful for toggling the mode around test fixtures. 98 | 99 | ## [0.7.2] - 2023-03-07 100 | 101 | ### Fixed 102 | 103 | * Fix `Carbonite.process/4` success typing by explicitly picking options. 104 | 105 | ## [0.7.1] - 2023-02-15 106 | 107 | ### Fixed 108 | 109 | * Fix `carbonite_prefix` on `Query.outbox_done/2`. Apply the prefix on the nested outbox query as well. 110 | 111 | ## [0.7.0] - 2023-02-15 112 | 113 | ### Fixed 114 | 115 | * `Carbonite.override_mode/2` and most functions in `Carbonite.Query` were broken when using a non-default `carbonite_prefix` option. Fixed by moving the `prefix` option onto the `from` expression. 116 | 117 | ### Added 118 | 119 | * Added `:initially` option to `create_trigger/2` to create triggers with `IMMEDIATE DEFERRED` constraint option. This allows to conditionally insert the `Carbonite.Transaction` record at the end of the transaction. In order to use this for already existing triggers, you need to drop them (`drop_trigger/2`) and re-create them. 120 | 121 | ### Changed 122 | 123 | * Made the changes trigger `DEFERRABLE`. As part of the `:initially` option of `create_trigger/2`, we chose to make triggers `DEFERRABLE` by default. Again, for any existing triggers, this won't take effect, but newly created triggers will use this constraint option. 124 | * Dropped support for `preload: atom | [atom]`-style of specifying preloads on a query in `Carbonite.Query`. 125 | 126 | ## [0.6.0] - 2022-10-12 127 | 128 | **New migration patches:** 5 129 | 130 | ### Added 131 | 132 | * Optional tracking of previous data in changes. Set the `store_changed_from` trigger option. 133 | 134 | ### Changed 135 | 136 | * Added `@schema_prefix "carbonite_default"` on all schemas. This will enable the manual usage of `Repo.insert/2` & friends on transactions without specifying a prefix, when using the default carbonite prefix. 137 | 138 | ## [0.5.0] - 2022-02-25 139 | 140 | **New migration patches:** 4 141 | 142 | ### Switch to normal identity column on `transactions` 143 | 144 | * The `id` column on `transactions` has been replaced with an ordinary autoincrementing integer PK, filled from a sequence. Next to it a new `xact_id` column continues to store the transaction id (from `pg_current_xact_id`). Both values used together ensure that, first the `id` is monotonically increasing and survives a backup restore (see issue #45), and second the `changes` records can still only be inserted within the same transaction. 145 | 146 | ### Added 147 | 148 | * `Carbonite.Migrations.insert_migration_transaction/1` and its macro friend, `insert_migration_transaction_after_begin`, help with data migrations. 149 | * `Carbonite.fetch_changes` returns the changes of the current (ongoing) transaction. 150 | 151 | ## [0.4.0] - 2021-11-07 152 | 153 | **New migration patches:** 2, 3 154 | 155 | ### Switch to top-level API with `repo` param 156 | 157 | * `Carbonite.override_mode/2` (kept a wrapper in `Carbonite.Multi`) 158 | * `Carbonite.insert_transaction/3` (kept a wrapper in `Carbonite.Multi`) 159 | * `Carbonite.process/4` (previously `Carbonite.Outbox.process/3` with major changes) 160 | * `Carbonite.purge/2` (previously `Carbonite.Outbox.purge/1` with major changes) 161 | 162 | ### Big outbox overhaul 163 | 164 | * Split into query / processing 165 | * Simplify processing 166 | * No more transaction 167 | * New capabilities: memo, halting, chunking 168 | 169 | ### Migration versioning 170 | 171 | * Explicit for now with `Carbonite.Migrations.up(non_neg_integer())` 172 | * `Carbonite.Migrations.install_schema/1` is now `Carbonite.Migrations.up/2` 173 | * `Carbonite.Migrations.put_trigger_option/4` to ensure old migrations continue to work 174 | * At the same time removed long configuration statement from `Carbonite.Migrations.install_trigger/1`, so this does not need to be versioned and continues to work 175 | * Mix task for generating the "initial" migration 176 | 177 | ### Other Changes 178 | 179 | * Optionally derive Jason.Encoder for `Carbonite.Transaction` and `Carbonite.Change` 180 | * Made all prefix options binary-only (no atom) as `Ecto.Query.put_query_prefix/2` only accepts strings 181 | 182 | ## [0.3.1] - 2021-10-23 183 | 184 | ### Added 185 | 186 | * `table_prefix` option to `Query.changes/2` allows to override schema prefix of given record 187 | * `Query.transactions/1` query selects all transactions 188 | 189 | ## [0.3.0] - 2021-10-22 190 | 191 | ### Added 192 | 193 | * `Carbonite.Migrations.drop_tables/1` allows to drop the carbonite audit trail without removing the schema 194 | 195 | ### Changed 196 | 197 | * Renamed the option that can be passed to `Carbonite.Migrations.drop_schema/1` from `prefix` to `carbonite_prefix` 198 | * Changed `Carbonite.Migrations.drop_schema/1` to also drop the tables 199 | * Made `Carbonite.Multi.insert_transaction/3` ignore conflicting `INSERT`s within the same transaction 200 | * Also, changed `Carbonite.Multi.insert_transaction/3` to always reloads all fields from the database after insertion, immediately returning the JSONinified `meta` payload 201 | 202 | ### Fixed 203 | 204 | * Fixed ignore mode when `override_transaction_id` is NULL 205 | 206 | ## [0.2.1] - 2021-10-10 207 | 208 | ### Fixed 209 | 210 | * Fixed broken documentation 211 | 212 | ## [0.2.0] - 2021-10-10 213 | 214 | ### Added 215 | 216 | * Store primary key(s) on changes table and index them 217 | * Add `Carbonite.Query` module 218 | - `current_transaction/2` allows to fetch the ongoing transaction (for sandbox tests) 219 | - `changes/2` allows to fetch the changes of an individual source record 220 | * Update Postgrex to 0.15.11 and drop local `Xid8` type 221 | * Add `mode` field to trigger (capture or ignore) 222 | * Add "override mode" reversing the `mode` option for the current transaction to enable/disable capturing on demand (e.g. in tests) 223 | * Add filtered columns 224 | 225 | ### Changed 226 | 227 | * Moved top-level functions to nested modules `Transaction` and `Multi` 228 | * Made `table_pk` be `NULL` when `primary_key_columns` is an empty array 229 | * Default `primary_key_columns` to `["id"]` 230 | * Renamed `prefix` option to `carbonite_prefix` on `install_schema/2` for consistency 231 | 232 | ## [0.1.0] - 2021-09-01 233 | 234 | * Initial release. 235 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021-2022 Bitcrowd GmbH 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /_plts/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bitcrowd/carbonite/0df3c1061d768f1826ab0f22c77d5144648315a6/_plts/.gitkeep -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | import Config 4 | 5 | if config_env() in [:dev, :test] do 6 | config :carbonite, Carbonite.TestRepo, 7 | database: "carbonite_#{config_env()}", 8 | username: "postgres", 9 | password: "postgres", 10 | hostname: "localhost", 11 | priv: "test/support/test_repo" 12 | 13 | config :carbonite, ecto_repos: [Carbonite.TestRepo] 14 | end 15 | 16 | if config_env() == :test do 17 | # Set to :debug to see SQL logs. 18 | config :logger, level: :info 19 | 20 | config :carbonite, Carbonite.TestRepo, pool: Ecto.Adapters.SQL.Sandbox 21 | end 22 | -------------------------------------------------------------------------------- /lib/carbonite.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite do 4 | @readme Path.join([__DIR__, "../README.md"]) 5 | @external_resource @readme 6 | 7 | @moduledoc @readme 8 | |> File.read!() 9 | |> String.split("") 10 | |> Enum.drop(1) 11 | |> Enum.take_every(2) 12 | |> Enum.join("\n") 13 | 14 | @moduledoc since: "0.1.0" 15 | 16 | alias Carbonite.{Outbox, Prefix, Query, Schema, Transaction, Trigger} 17 | require Prefix 18 | require Schema 19 | 20 | @type prefix :: binary() 21 | @type repo :: Ecto.Repo.t() 22 | 23 | @type prefix_option :: {:carbonite_prefix, prefix()} 24 | 25 | @doc "Returns the default audit trail prefix." 26 | @doc since: "0.1.0" 27 | @spec default_prefix() :: prefix() 28 | def default_prefix, do: Prefix.default_prefix() 29 | 30 | @doc """ 31 | Inserts a `t:Carbonite.Transaction.t/0` into the database. 32 | 33 | Make sure to run this within a transaction. 34 | 35 | ## Parameters 36 | 37 | * `repo` - the Ecto repository 38 | * `params` - map of params for the `Carbonite.Transaction` (e.g., `:meta`) 39 | * `opts` - optional keyword list 40 | 41 | ## Options 42 | 43 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 44 | 45 | ## Multiple inserts in the same transaction 46 | 47 | Normally, you should have exactly one `insert_transaction/3` call per database transaction. In 48 | practise, there are two scenarios in this function may be called multiple times: 49 | 50 | 1. If an operation A, which calls `insert_transaction/3`, sometimes is nested within an outer 51 | operation B, which also calls `insert_transaction/3`. 52 | 2. In tests using Ecto's SQL sandbox, subsequent calls to transactional operations (even to the 53 | same operation twice) are wrapped inside the overarching test transaction, and hence also 54 | effectively call `insert_transaction/3` within the same transaction. 55 | 56 | While the first scenario can be resolved using appropriate control flow (e.g. by conditionally 57 | disabling the inner `insert_transaction/3` call), the second scenario is quite common and often 58 | unavoidable. 59 | 60 | Therefore, `insert_transaction/3` **ignores** subsequent calls within the same database 61 | transaction (equivalent to `ON CONFLICT DO NOTHING`), **discarding metadata** passed to all 62 | calls but the first. 63 | """ 64 | @doc since: "0.4.0" 65 | @spec insert_transaction(repo()) :: {:ok, Transaction.t()} | {:error, Ecto.Changeset.t()} 66 | @spec insert_transaction(repo(), params :: map()) :: 67 | {:ok, Transaction.t()} | {:error, Ecto.Changeset.t()} 68 | @spec insert_transaction(repo(), params :: map(), [prefix_option()]) :: 69 | {:ok, Transaction.t()} | {:error, Ecto.Changeset.t()} 70 | def insert_transaction(repo, params \\ %{}, opts \\ []) do 71 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 72 | 73 | # NOTE: ON CONFLICT DO NOTHING does not combine with RETURNING, so we're forcing an UPDATE. 74 | 75 | params 76 | |> Transaction.changeset() 77 | |> repo.insert( 78 | prefix: carbonite_prefix, 79 | on_conflict: {:replace, [:id]}, 80 | conflict_target: [:id], 81 | returning: true 82 | ) 83 | end 84 | 85 | @doc """ 86 | Fetches all changes of the current transaction from the database. 87 | 88 | Make sure to run this within a transaction. 89 | 90 | ## Parameters 91 | 92 | * `repo` - the Ecto repository 93 | * `opts` - optional keyword list 94 | 95 | ## Options 96 | 97 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 98 | """ 99 | @doc since: "0.5.0" 100 | @spec fetch_changes(repo()) :: {:ok, [Carbonite.Change.t()]} 101 | @spec fetch_changes(repo(), [prefix_option()]) :: {:ok, [Carbonite.Change.t()]} 102 | def fetch_changes(repo, opts \\ []) do 103 | %Carbonite.Transaction{changes: changes} = 104 | [preload: true] 105 | |> Keyword.merge(opts) 106 | |> Query.current_transaction() 107 | |> repo.one() 108 | 109 | {:ok, changes} 110 | end 111 | 112 | @doc """ 113 | Sets the current transaction to "override mode" for all tables in the audit log. 114 | 115 | ## Parameters 116 | 117 | * `repo` - the Ecto repository 118 | * `opts` - optional keyword list 119 | 120 | ## Options 121 | 122 | * `to` - allows to specify the target mode 123 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 124 | """ 125 | @doc since: "0.4.0" 126 | @spec override_mode(repo()) :: :ok 127 | @spec override_mode(repo(), [{:to, Trigger.mode()} | prefix_option()]) :: :ok 128 | def override_mode(repo, opts \\ []) do 129 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 130 | mode = Keyword.get(opts, :to, :override) 131 | 132 | repo.query!("SET LOCAL #{carbonite_prefix}.override_mode = '#{mode}';") 133 | 134 | :ok 135 | end 136 | 137 | @type process_option :: 138 | Carbonite.Query.outbox_queue_option() 139 | | {:filter, (Ecto.Query.t() -> Ecto.Query.t())} 140 | | {:chunk, pos_integer()} 141 | 142 | @type process_func_option :: {:memo, Outbox.memo()} | {:discard_last, boolean()} 143 | 144 | @typedoc """ 145 | This type defines the callback function signature for `Carbonite.process/3`. 146 | 147 | The processor function receives the current chunk of transactions and the memo of the last 148 | function application, and must return one of 149 | 150 | * `:cont` - continue processing 151 | * `:halt` - stop processing after this chunk 152 | * `{:cont | :halt, opts}` - cont/halt and set some options 153 | 154 | After the process function invocation the Outbox is updated with new attributes. 155 | 156 | ## Options 157 | 158 | Returned options can be: 159 | 160 | * `memo` - memo to store on Outbox, defaults to previous memo 161 | * `last_transaction_id` - last transaction id to remember as processed, defaults to previous 162 | `last_transaction_id` on `:halt`, defaults to last id in current 163 | chunk when `:cont` is returned 164 | """ 165 | @type process_func :: 166 | ([Transaction.t()], Outbox.memo() -> 167 | :cont | :halt | {:cont | :halt, [process_func_option()]}) 168 | 169 | @doc """ 170 | Processes an outbox queue. 171 | 172 | This function sends chunks of persisted transactions to a user-supplied processing function. It 173 | looks up the current "reading position" from a given `Carbonite.Outbox` and yields transactions 174 | matching the given filter criteria (`min_age`, etc.) until either the input source is exhausted 175 | or a processing function application returns `:halt`. 176 | 177 | Returns either `{:ok, outbox}` or `{:halt, outbox}` depending on whether processing was halted 178 | explicitly or due the exhausted input source. 179 | 180 | See `Carbonite.Query.outbox_queue/2` for query options. 181 | 182 | ## Examples 183 | 184 | Carbonite.process(MyApp.Repo, "rabbit_holes", fn [transaction], _memo -> 185 | # The transaction has its changes preloaded. 186 | transaction 187 | |> MyApp.Foo.serialize() 188 | |> MyApp.Foo.send_to_external_database() 189 | 190 | :cont 191 | end) 192 | 193 | ### Memo passing 194 | 195 | The `memo` is useful to carry data between each processor application. Let's say you wanted to 196 | generate a hashsum chain on your processed data: 197 | 198 | Carbonite.process(MyApp.Repo, "rabbit_holes", fn [transaction], %{"checksum" => checksum} -> 199 | {payload, checksum} = MyApp.Foo.serialize_and_hash(transaction, checksum) 200 | 201 | MyApp.Foo.send_to_external_database(payload) 202 | 203 | {:cont, memo: %{"checksum" => checksum}} 204 | end) 205 | 206 | ### Chunking / Limiting 207 | 208 | The examples above received a single-element list as their first parameter: This is because the 209 | transactions are actually processed in "chunks" and the default chunk size is 1. If you would 210 | like to process more transactions in one chunk, set the `chunk` option: 211 | 212 | Carbonite.process(MyApp.Repo, "rabbit_holes", [chunk: 50], fn transactions, _memo -> 213 | for transaction <- transactions do 214 | transaction 215 | |> MyApp.Foo.serialize() 216 | |> MyApp.Foo.send_to_external_database() 217 | end 218 | 219 | :cont 220 | end) 221 | 222 | The query that is executed to fetch the data from the database is controlled with the `limit` 223 | option and is independent of the chunk size. 224 | 225 | ### Error handling 226 | 227 | In case you run into an error midway into processing a batch, you may choose to halt processing 228 | while remembering about the last processed transaction. This is equivalent to raising an 229 | exception from the processing function. 230 | 231 | Carbonite.process(MyApp.Repo, "rabbit_holes", fn [transaction], _memo -> 232 | case send_to_external_database(transaction) do 233 | :ok -> 234 | :cont 235 | 236 | {:error, _term} -> 237 | :halt 238 | end 239 | end 240 | 241 | You can, however, if you know the first half of a batch has been processed, still update the 242 | `memo` and `last_transaction_id`. 243 | 244 | Carbonite.process(MyApp.Repo, "rabbit_holes", fn transactions, _memo -> 245 | case process_transactions(transactions) do 246 | {:error, last_successful_transaction} -> 247 | {:halt, last_transaction_id: last_successful_transaction.id} 248 | 249 | :ok -> 250 | :cont 251 | end 252 | end 253 | 254 | ## Parameters 255 | 256 | * `repo` - the Ecto repository 257 | * `outbox_name` - name of the outbox to process 258 | * `opts` - optional keyword list 259 | * `process_func` - see `t:process_func/0` for details 260 | 261 | ## Options 262 | 263 | * `min_age` - the minimum age of a record, defaults to 300 seconds (set nil to disable) 264 | * `limit` - limits the query in size, defaults to 100 (set nil to disable) 265 | * `filter` - function for refining the batch query, defaults to nil 266 | * `chunk` - defines the size of the chunk passed to the process function, defaults to 1 267 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 268 | """ 269 | @doc since: "0.4.0" 270 | @spec process(repo(), Outbox.name(), process_func()) :: {:ok | :halt, Outbox.t()} 271 | @spec process(repo(), Outbox.name(), [process_option()], process_func()) :: 272 | {:ok | :halt, Outbox.t()} 273 | def process(repo, outbox_name, opts \\ [], process_func) do 274 | outbox = load_outbox(repo, outbox_name, opts) 275 | 276 | functions = %{ 277 | query: query_func(repo, opts), 278 | process: process_func(process_func), 279 | update: update_func(repo) 280 | } 281 | 282 | process_outbox(outbox, functions) 283 | end 284 | 285 | defp load_outbox(repo, outbox_name, opts) do 286 | opts = Keyword.take(opts, [:carbonite_prefix]) 287 | 288 | outbox_name 289 | |> Carbonite.Query.outbox(opts) 290 | |> repo.one!() 291 | end 292 | 293 | defp query_func(repo, opts) do 294 | filter = Keyword.get(opts, :filter) || (& &1) 295 | chunk = Keyword.get(opts, :chunk, 1) 296 | 297 | fn outbox -> 298 | outbox 299 | |> Carbonite.Query.outbox_queue(opts) 300 | |> filter.() 301 | |> repo.all() 302 | |> Enum.chunk_every(chunk) 303 | end 304 | end 305 | 306 | defp process_func(process_func) do 307 | fn chunk, outbox -> 308 | case process_func.(chunk, outbox.memo) do 309 | cont_or_halt when is_atom(cont_or_halt) -> {cont_or_halt, []} 310 | {cont_or_halt, opts} -> {cont_or_halt, opts} 311 | end 312 | end 313 | end 314 | 315 | defp update_func(repo) do 316 | fn outbox, attrs -> 317 | outbox 318 | |> Outbox.changeset(attrs) 319 | |> repo.update!() 320 | end 321 | end 322 | 323 | defp process_outbox(outbox, functions) do 324 | case functions.query.(outbox) do 325 | [] -> {:ok, outbox} 326 | chunks -> process_chunks(chunks, outbox, functions) 327 | end 328 | end 329 | 330 | defp process_chunks([], outbox, functions) do 331 | process_outbox(outbox, functions) 332 | end 333 | 334 | defp process_chunks([chunk | rest], outbox, functions) do 335 | case process_chunk(chunk, outbox, functions) do 336 | {:cont, outbox} -> process_chunks(rest, outbox, functions) 337 | halt -> halt 338 | end 339 | end 340 | 341 | defp process_chunk(chunk, outbox, functions) do 342 | {cont_or_halt, result_opts} = functions.process.(chunk, outbox) 343 | 344 | defaults = 345 | if cont_or_halt == :cont do 346 | %{last_transaction_id: List.last(chunk).id} 347 | else 348 | %{} 349 | end 350 | 351 | results = 352 | result_opts 353 | |> Keyword.take([:memo, :last_transaction_id]) 354 | |> Map.new() 355 | 356 | outbox = functions.update.(outbox, Map.merge(defaults, results)) 357 | 358 | {cont_or_halt, outbox} 359 | end 360 | 361 | @type purge_option :: Carbonite.Query.outbox_done_option() 362 | 363 | @doc """ 364 | Deletes transactions that have been fully processed. 365 | 366 | See `Carbonite.Query.outbox_done/1` for query options. 367 | 368 | Returns the number of deleted transactions. 369 | 370 | ## Parameters 371 | 372 | * `repo` - the Ecto repository 373 | * `opts` - optional keyword list 374 | 375 | ## Options 376 | 377 | * `min_age` - the minimum age of a record, defaults to 300 seconds (set nil to disable) 378 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 379 | """ 380 | @doc since: "0.4.0" 381 | @spec purge(repo()) :: {:ok, non_neg_integer()} 382 | @spec purge(repo(), [purge_option()]) :: {:ok, non_neg_integer()} 383 | def purge(repo, opts \\ []) do 384 | {deleted, nil} = 385 | opts 386 | |> Carbonite.Query.outbox_done() 387 | |> repo.delete_all() 388 | 389 | {:ok, deleted} 390 | end 391 | end 392 | -------------------------------------------------------------------------------- /lib/carbonite/change.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Change do 4 | @moduledoc """ 5 | A `Carbonite.Change` records a mutation on a database table. 6 | 7 | `INSERT` statements lead to a `Change` where the `data` field contains the inserted row as a 8 | JSON object while the `changed` field is an empty list. 9 | 10 | `UPDATE` statements contain the updated record in `data` while the `changed` field is a list 11 | of attributes that have changed. The `changed_from` field may additionally contain a partial 12 | representation of the replaced record, showing only the diff to the updated record. This 13 | behaviour is optional and can be enabled with the `store_changed_from` trigger option. 14 | 15 | `DELETE` statements have the deleted data in `data` while `changed` is again an empty list. 16 | """ 17 | 18 | @moduledoc since: "0.1.0" 19 | 20 | use Carbonite.Schema 21 | 22 | if Code.ensure_loaded?(Jason.Encoder) do 23 | @derive {Jason.Encoder, 24 | only: [ 25 | :id, 26 | :op, 27 | :table_prefix, 28 | :table_name, 29 | :table_pk, 30 | :data, 31 | :changed, 32 | :changed_from 33 | ]} 34 | end 35 | 36 | @primary_key false 37 | 38 | @type id :: non_neg_integer() 39 | 40 | @type t :: %__MODULE__{ 41 | id: id(), 42 | op: :insert | :update | :delete, 43 | table_prefix: String.t(), 44 | table_name: String.t(), 45 | table_pk: nil | [String.t()], 46 | data: map(), 47 | changed: [String.t()], 48 | changed_from: nil | map(), 49 | transaction: Ecto.Association.NotLoaded.t() | Carbonite.Transaction.t(), 50 | transaction_xact_id: non_neg_integer() 51 | } 52 | 53 | schema "changes" do 54 | field(:id, :integer, primary_key: true) 55 | field(:op, Ecto.Enum, values: [:insert, :update, :delete]) 56 | field(:table_prefix, :string) 57 | field(:table_name, :string) 58 | field(:table_pk, {:array, :string}) 59 | field(:data, :map) 60 | field(:changed, {:array, :string}) 61 | field(:changed_from, :map) 62 | 63 | belongs_to(:transaction, Carbonite.Transaction) 64 | 65 | field(:transaction_xact_id, :integer) 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/carbonite/migrations.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations do 4 | @moduledoc """ 5 | Functions to setup Carbonite audit trails in your migrations. 6 | """ 7 | 8 | @moduledoc since: "0.1.0" 9 | 10 | use Ecto.Migration 11 | import Carbonite.Migrations.Helper 12 | import Carbonite.Prefix 13 | 14 | @type patch :: non_neg_integer() 15 | @type prefix :: binary() 16 | @type table_name :: binary() | atom() 17 | 18 | # --------------------------------- patch levels --------------------------------- 19 | 20 | @initial_patch 1 21 | @current_patch 12 22 | 23 | @doc false 24 | @spec initial_patch :: non_neg_integer() 25 | def initial_patch, do: @initial_patch 26 | 27 | @doc false 28 | @spec current_patch :: non_neg_integer() 29 | def current_patch, do: @current_patch 30 | 31 | @doc false 32 | @spec patch_range :: Range.t() 33 | def patch_range, do: @initial_patch..@current_patch 34 | 35 | # --------------------------------- main schema ---------------------------------- 36 | 37 | @type up_option :: {:carbonite_prefix, prefix()} 38 | 39 | @doc """ 40 | Runs one of Carbonite's migrations. 41 | 42 | ## Migration patchlevels 43 | 44 | Make sure that you run all migrations in your host application. 45 | 46 | * Initial patch: #{@initial_patch} 47 | * Current patch: #{@current_patch} 48 | 49 | ## Options 50 | 51 | * `carbonite_prefix` defines the audit trail's schema, defaults to `"carbonite_default"` 52 | """ 53 | @doc since: "0.4.0" 54 | @spec up(patch() | Range.t()) :: :ok 55 | @spec up(patch() | Range.t(), [up_option()]) :: :ok 56 | def up(patch_or_range, opts \\ []) when is_list(opts) do 57 | change(:up, patch_or_range, opts) 58 | end 59 | 60 | @type down_option :: {:carbonite_prefix, prefix()} | {:drop_schema, boolean()} 61 | 62 | @doc """ 63 | Rollback a migration. 64 | 65 | ## Options 66 | 67 | * `carbonite_prefix` defines the audit trail's schema, defaults to `"carbonite_default"` 68 | * `drop_schema` controls whether the initial migration deletes the schema during rollback 69 | """ 70 | @doc since: "0.4.0" 71 | @spec down(patch() | Range.t()) :: :ok 72 | @spec down(patch() | Range.t(), [down_option()]) :: :ok 73 | def down(patch_or_range, opts \\ []) when is_list(opts) do 74 | change(:down, patch_or_range, opts) 75 | end 76 | 77 | defp change(direction, %Range{} = range, opts) do 78 | for patch <- range do 79 | change(direction, patch, opts) 80 | end 81 | end 82 | 83 | defp change(direction, patch, opts) when is_integer(patch) do 84 | module = Module.concat([__MODULE__, "V#{patch}"]) 85 | 86 | apply(module, direction, [opts]) 87 | end 88 | 89 | # ------------------------------- trigger setup ---------------------------------- 90 | 91 | @default_table_prefix "public" 92 | 93 | @type trigger_option :: {:table_prefix, prefix()} | {:carbonite_prefix, prefix()} 94 | @type create_trigger_option :: trigger_option() | {:initially, :deferred | :immediate} 95 | 96 | @doc """ 97 | Installs a change capture trigger on a table. 98 | 99 | ## Options 100 | 101 | * `table_prefix` is the name of the schema the table lives in 102 | * `carbonite_prefix` is the schema of the audit trail, defaults to `"carbonite_default"` 103 | * `initially` can be either of `:immediate` or `:deferred`, defaults to `:immediate` 104 | 105 | ## Deferred triggers 106 | 107 | You may create the trigger with `initially: :deferred` option to make it run at the end of your 108 | database transactions. In combination with application logic that only conditionally inserts 109 | the `Carbonite.Transaction`, this can be used to avoid storing "empty" transactions, i.e. 110 | `Carbonite.Transaction` records without associated `Carbonite.Change`. 111 | 112 | Please be aware that this is not recommended by the Carbonite team. Instead we recommend to 113 | retain the empty transactions and optionally remove them in a preprocessing step or your 114 | outbox mechanism. Make sure you know how deferred triggers work before using this option. 115 | """ 116 | @doc since: "0.4.0" 117 | @spec create_trigger(table_name()) :: :ok 118 | @spec create_trigger(table_name(), [create_trigger_option()]) :: :ok 119 | def create_trigger(table_name, opts \\ []) do 120 | table_prefix = Keyword.get(opts, :table_prefix, @default_table_prefix) 121 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 122 | initially = Keyword.get(opts, :initially, :immediate) 123 | 124 | validate_initially_option!(initially) 125 | 126 | """ 127 | CREATE CONSTRAINT TRIGGER capture_changes_into_#{carbonite_prefix}_trigger 128 | AFTER INSERT OR UPDATE OR DELETE 129 | ON "#{table_prefix}"."#{table_name}" 130 | DEFERRABLE INITIALLY #{initially} 131 | FOR EACH ROW 132 | EXECUTE PROCEDURE #{carbonite_prefix}.capture_changes(); 133 | """ 134 | |> squish_and_execute() 135 | 136 | """ 137 | INSERT INTO #{carbonite_prefix}.triggers ( 138 | id, 139 | table_prefix, 140 | table_name, 141 | inserted_at, 142 | updated_at 143 | ) VALUES ( 144 | NEXTVAL('#{carbonite_prefix}.triggers_id_seq'), 145 | '#{table_prefix}', 146 | '#{table_name}', 147 | NOW(), 148 | NOW() 149 | ); 150 | """ 151 | |> squish_and_execute() 152 | 153 | :ok 154 | end 155 | 156 | defp validate_initially_option!(initially) when initially in [:immediate, :deferred], do: :ok 157 | 158 | defp validate_initially_option!(initially) do 159 | raise("initially must be one of [:immediate, :deferred], but is #{inspect(initially)}") 160 | end 161 | 162 | @doc """ 163 | Removes a change capture trigger from a table. 164 | 165 | ## Options 166 | 167 | * `table_prefix` is the name of the schema the table lives in 168 | * `carbonite_prefix` is the schema of the audit trail, defaults to `"carbonite_default"` 169 | """ 170 | @doc since: "0.4.0" 171 | @spec drop_trigger(table_name()) :: :ok 172 | @spec drop_trigger(table_name(), [trigger_option()]) :: :ok 173 | def drop_trigger(table_name, opts \\ []) do 174 | table_prefix = Keyword.get(opts, :table_prefix, @default_table_prefix) 175 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 176 | 177 | """ 178 | DELETE FROM #{carbonite_prefix}.triggers 179 | WHERE table_prefix = '#{table_prefix}' 180 | AND table_name = '#{table_name}'; 181 | """ 182 | |> squish_and_execute() 183 | 184 | """ 185 | DROP TRIGGER capture_changes_into_#{carbonite_prefix}_trigger 186 | ON "#{table_prefix}"."#{table_name}"; 187 | """ 188 | |> squish_and_execute() 189 | 190 | :ok 191 | end 192 | 193 | @type trigger_config_key :: 194 | :table_prefix 195 | | :primary_key_columns 196 | | :excluded_columns 197 | | :filtered_columns 198 | | :mode 199 | | :store_changed_from 200 | 201 | @doc """ 202 | Allows to update a trigger configuration option for a given table. 203 | 204 | This function builds an SQL UPDATE statement that can be used within a database migration to 205 | update a setting stored in Carbonite's `triggers` table without using the `Carbonite.Trigger` 206 | schema or other application-level code that is prone to change over time. This helps to ensure 207 | that your data migrations continue to function, regardless of future updates to Carbonite. 208 | 209 | ## Configuration values 210 | 211 | * `primary_key_columns` is a list of columns that form the primary key of the table 212 | (defaults to `["id"]`, set to `[]` to disable) 213 | * `excluded_columns` is a list of columns to exclude from change captures 214 | * `filtered_columns` is a list of columns that appear as '[FILTERED]' in the data 215 | * `store_changed_from` is a boolean defining whether the `changed_from` field should be filled 216 | * `mode` is either `:capture` or `:ignore` and defines the default behaviour of the trigger 217 | 218 | ## Example 219 | 220 | Carbonite.Migrations.put_trigger_config("rabbits", :excluded_columns, ["name"]) 221 | 222 | ## Options 223 | 224 | * `table_prefix` is the name of the schema the table lives in 225 | * `carbonite_prefix` is the schema of the audit trail, defaults to `"carbonite_default"` 226 | """ 227 | @doc since: "0.4.0" 228 | @spec put_trigger_config(table_name(), trigger_config_key(), any(), [trigger_option()]) :: :ok 229 | def put_trigger_config(table_name, key, value, opts \\ []) 230 | 231 | def put_trigger_config(table_name, key, value, opts) 232 | when key in [:primary_key_columns, :excluded_columns, :filtered_columns] do 233 | do_put_trigger_config(table_name, key, column_list(value), opts) 234 | end 235 | 236 | def put_trigger_config(table_name, :mode, value, opts) when value in [:capture, :ignore] do 237 | do_put_trigger_config(table_name, :mode, "'#{value}'", opts) 238 | end 239 | 240 | def put_trigger_config(table_name, :store_changed_from, value, opts) when is_boolean(value) do 241 | do_put_trigger_config(table_name, :store_changed_from, value, opts) 242 | end 243 | 244 | defp do_put_trigger_config(table_name, field, value, opts) do 245 | table_prefix = Keyword.get(opts, :table_prefix, @default_table_prefix) 246 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 247 | 248 | """ 249 | UPDATE #{carbonite_prefix}.triggers 250 | SET #{field} = #{value}, updated_at = NOW() 251 | WHERE table_prefix = '#{table_prefix}' 252 | AND table_name = '#{table_name}'; 253 | """ 254 | |> squish_and_execute() 255 | 256 | :ok 257 | end 258 | 259 | # ------------------------------- outbox setup ----------------------------------- 260 | 261 | @type outbox_name :: String.t() 262 | @type outbox_option :: {:carbonite_prefix, prefix()} 263 | 264 | @doc """ 265 | Inserts an outbox record into the database. 266 | 267 | ## Options 268 | 269 | * `carbonite_prefix` is the schema of the audit trail, defaults to `"carbonite_default"` 270 | """ 271 | @doc since: "0.4.0" 272 | @spec create_outbox(outbox_name()) :: :ok 273 | @spec create_outbox(outbox_name(), [outbox_option()]) :: :ok 274 | def create_outbox(outbox_name, opts \\ []) do 275 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 276 | 277 | """ 278 | INSERT INTO #{carbonite_prefix}.outboxes ( 279 | name, 280 | inserted_at, 281 | updated_at 282 | ) VALUES ( 283 | '#{outbox_name}', 284 | NOW(), 285 | NOW() 286 | ); 287 | """ 288 | |> squish_and_execute() 289 | 290 | :ok 291 | end 292 | 293 | @doc """ 294 | Removes an outbox record. 295 | 296 | ## Options 297 | 298 | * `carbonite_prefix` is the schema of the audit trail, defaults to `"carbonite_default"` 299 | """ 300 | @doc since: "0.4.0" 301 | @spec drop_outbox(outbox_name()) :: :ok 302 | @spec drop_outbox(outbox_name(), [outbox_option()]) :: :ok 303 | def drop_outbox(outbox_name, opts \\ []) do 304 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 305 | 306 | """ 307 | DELETE FROM #{carbonite_prefix}.outboxes 308 | WHERE name = '#{outbox_name}'; 309 | """ 310 | |> squish_and_execute() 311 | 312 | :ok 313 | end 314 | 315 | # ------------------------------ data migrations --------------------------------- 316 | 317 | @type insert_migration_transaction_option :: {:carbonite_prefix, prefix()} | {:meta, map()} 318 | 319 | @doc """ 320 | Inserts a transaction for a data migration. 321 | 322 | The transaction's `meta` attribute is populated with 323 | 324 | {"type": "migration", direction: "up"} 325 | 326 | ... and additionally the `name` from the parameters. 327 | 328 | ## Example 329 | 330 | defmodule MyApp.Repo.Migrations.SomeDataMigration do 331 | use Ecto.Migration 332 | 333 | import Carbonite.Migrations 334 | 335 | def change do 336 | insert_migration_transaction("some-data-migration") 337 | 338 | execute("UPDATE ...", "...") 339 | end 340 | end 341 | 342 | This works the same for `up/0`/`down/0`-style migrations: 343 | 344 | defmodule MyApp.Repo.Migrations.SomeDataMigration do 345 | use Ecto.Migration 346 | 347 | import Carbonite.Migrations 348 | 349 | def up do 350 | insert_migration_transaction("some-data-migration") 351 | 352 | execute("INSERT ...") 353 | end 354 | 355 | def down do 356 | insert_migration_transaction("some-data-migration") 357 | 358 | execute("DELETE ...") 359 | end 360 | end 361 | 362 | ## Options 363 | 364 | * `carbonite_prefix` is the schema of the audit trail, defaults to `"carbonite_default"` 365 | * `meta` is a map with additional meta information to store on the transaction 366 | """ 367 | @spec insert_migration_transaction(name :: String.t(), [insert_migration_transaction_option()]) :: 368 | :ok 369 | def insert_migration_transaction(name, opts \\ []) do 370 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 371 | meta = Keyword.get(opts, :meta, %{}) 372 | 373 | meta = 374 | %{type: "migration", direction: direction(), name: to_string(name)} 375 | |> Map.merge(meta) 376 | |> Jason.encode!() 377 | 378 | statement = 379 | """ 380 | INSERT INTO #{carbonite_prefix}.transactions (meta, inserted_at) 381 | VALUES ('#{meta}'::jsonb, NOW()); 382 | """ 383 | |> squish() 384 | 385 | # Needed because `change/0` requires a rollback statement. In our case irrelevant, as we're 386 | # inserting a transaction in both directions (and the `direction` value is set dynamically). 387 | execute(statement, statement) 388 | end 389 | 390 | @doc """ 391 | Defines a `c:Ecto.Migration.after_begin/0` implementation for a data migration. 392 | 393 | See `insert_migration_transaction/1` for options. 394 | 395 | This determines the `name` of the transaction from the migration's module name. 396 | 397 | {"direction": "up", "name": "my_app/repo/migrations/example", "type": "migration"} 398 | 399 | ## Example 400 | 401 | defmodule MyApp.Repo.Migrations.SomeDataMigration do 402 | use Ecto.Migration 403 | 404 | import Carbonite.Migrations 405 | insert_migration_transaction_after_begin() 406 | 407 | def change do 408 | execute("UPDATE ...") 409 | end 410 | end 411 | 412 | Alternatively, if you define your own migration template module: 413 | 414 | defmodule MyApp.DataMigration do 415 | defmacro __using__ do 416 | quote do 417 | use Ecto.Migration 418 | 419 | import Carbonite.Migrations 420 | insert_migration_transaction_after_begin() 421 | end 422 | end 423 | end 424 | """ 425 | @doc since: "0.5.0" 426 | defmacro insert_migration_transaction_after_begin(opts \\ []) do 427 | opts = Keyword.take(opts, [:carbonite_prefix, :meta]) 428 | 429 | quote do 430 | @behaviour Ecto.Migration 431 | 432 | @impl Ecto.Migration 433 | def after_begin do 434 | __MODULE__ 435 | |> Macro.underscore() 436 | |> Carbonite.Migrations.insert_migration_transaction(unquote(opts)) 437 | end 438 | end 439 | end 440 | end 441 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/helper.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.Helper do 4 | @moduledoc false 5 | 6 | import Ecto.Migration, only: [execute: 1] 7 | 8 | # Removes surrounding and consecutive whitespace from SQL to improve readability in console. 9 | @spec squish(String.t()) :: String.t() 10 | def squish(statement) do 11 | statement 12 | |> String.replace(~r/[[:space:]]+/, " ") 13 | |> String.trim() 14 | end 15 | 16 | # Removes surrounding and consecutive whitespace from SQL to improve readability in console. 17 | @spec squish_and_execute(String.t()) :: :ok 18 | def squish_and_execute(statement) do 19 | statement 20 | |> squish() 21 | |> execute() 22 | end 23 | 24 | # Joins a list of atoms/strings to a `{'bar', 'foo', ...}` (ordered) SQL array expression. 25 | @spec column_list(nil | [atom() | String.t()]) :: String.t() 26 | def column_list(nil), do: "'{}'" 27 | def column_list(value), do: "'{#{do_column_list(value)}}'" 28 | 29 | defp do_column_list(value) do 30 | value 31 | |> List.wrap() 32 | |> Enum.map(&to_string/1) 33 | |> Enum.sort() 34 | |> Enum.map_join(", ", &"\"#{&1}\"") 35 | end 36 | 37 | # Locks a table in exclusive mode for constraint modifications & co. 38 | @spec lock_table(binary(), binary()) :: :ok 39 | def lock_table(prefix, table) do 40 | squish_and_execute("LOCK TABLE #{prefix}.#{table} IN EXCLUSIVE MODE;") 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v1.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V1 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | 9 | @type prefix :: binary() 10 | 11 | @type up_option :: {:carbonite_prefix, prefix()} 12 | 13 | @spec create_set_transaction_id_procedure(prefix()) :: :ok 14 | def create_set_transaction_id_procedure(prefix) do 15 | """ 16 | CREATE OR REPLACE FUNCTION #{prefix}.set_transaction_id() RETURNS TRIGGER AS 17 | $body$ 18 | BEGIN 19 | NEW.id = COALESCE(NEW.id, pg_current_xact_id()); 20 | RETURN NEW; 21 | END 22 | $body$ 23 | LANGUAGE plpgsql; 24 | """ 25 | |> squish_and_execute() 26 | end 27 | 28 | @spec create_capture_changes_procedure(prefix()) :: :ok 29 | def create_capture_changes_procedure(prefix) do 30 | """ 31 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 32 | $body$ 33 | DECLARE 34 | trigger_row #{prefix}.triggers; 35 | change_row #{prefix}.changes; 36 | pk_source RECORD; 37 | col_name VARCHAR; 38 | pk_col_val VARCHAR; 39 | old_field RECORD; 40 | BEGIN 41 | /* load trigger config */ 42 | SELECT * 43 | INTO trigger_row 44 | FROM #{prefix}.triggers 45 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 46 | 47 | IF 48 | (trigger_row.mode = 'ignore' AND (trigger_row.override_transaction_id IS NULL OR trigger_row.override_transaction_id != pg_current_xact_id())) OR 49 | (trigger_row.mode = 'capture' AND trigger_row.override_transaction_id = pg_current_xact_id()) 50 | THEN 51 | RETURN NULL; 52 | END IF; 53 | 54 | /* instantiate change row */ 55 | change_row = ROW( 56 | NEXTVAL('#{prefix}.changes_id_seq'), 57 | pg_current_xact_id(), 58 | LOWER(TG_OP::TEXT), 59 | TG_TABLE_SCHEMA::TEXT, 60 | TG_TABLE_NAME::TEXT, 61 | NULL, 62 | NULL, 63 | '{}' 64 | ); 65 | 66 | /* build table_pk */ 67 | IF trigger_row.primary_key_columns != '{}' THEN 68 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 69 | pk_source := NEW; 70 | ELSIF (TG_OP = 'DELETE') THEN 71 | pk_source := OLD; 72 | END IF; 73 | 74 | change_row.table_pk := '{}'; 75 | 76 | FOREACH col_name IN ARRAY trigger_row.primary_key_columns LOOP 77 | EXECUTE 'SELECT $1.' || col_name || '::text' USING pk_source INTO pk_col_val; 78 | change_row.table_pk := change_row.table_pk || pk_col_val; 79 | END LOOP; 80 | END IF; 81 | 82 | /* fill in changed data */ 83 | IF (TG_OP = 'UPDATE') THEN 84 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 85 | 86 | FOR old_field IN SELECT * FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) LOOP 87 | IF NOT change_row.data @> jsonb_build_object(old_field.key, old_field.value) 88 | THEN change_row.changed := change_row.changed || old_field.key::VARCHAR; 89 | END IF; 90 | END LOOP; 91 | 92 | IF change_row.changed = '{}' THEN 93 | /* All changed fields are ignored. Skip this update. */ 94 | RETURN NULL; 95 | END IF; 96 | ELSIF (TG_OP = 'DELETE') THEN 97 | change_row.data = to_jsonb(OLD.*) - trigger_row.excluded_columns; 98 | ELSIF (TG_OP = 'INSERT') THEN 99 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 100 | END IF; 101 | 102 | /* filtered columns */ 103 | FOREACH col_name IN ARRAY trigger_row.filtered_columns LOOP 104 | change_row.data = jsonb_set(change_row.data, ('{' || col_name || '}')::text[], jsonb('"[FILTERED]"')); 105 | END LOOP; 106 | 107 | /* insert, fail gracefully unless transaction record present */ 108 | BEGIN 109 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 110 | EXCEPTION WHEN foreign_key_violation THEN 111 | RAISE '% on table %.% without prior INSERT into #{prefix}.transactions', 112 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 113 | END; 114 | 115 | RETURN NULL; 116 | END; 117 | $body$ 118 | LANGUAGE plpgsql; 119 | """ 120 | |> squish_and_execute() 121 | end 122 | 123 | @spec create_triggers_table_index(prefix()) :: :ok 124 | @spec create_triggers_table_index(prefix(), atom()) :: :ok 125 | def create_triggers_table_index(prefix, override_transaction_id \\ :override_transaction_id) do 126 | create( 127 | index("triggers", [:table_prefix, :table_name], 128 | name: "table_index", 129 | unique: true, 130 | include: [ 131 | :primary_key_columns, 132 | :excluded_columns, 133 | :filtered_columns, 134 | :mode, 135 | override_transaction_id 136 | ], 137 | prefix: prefix 138 | ) 139 | ) 140 | 141 | :ok 142 | end 143 | 144 | @impl true 145 | @spec up([up_option()]) :: :ok 146 | def up(opts) do 147 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 148 | 149 | # ---------------- Schema -------------------- 150 | 151 | execute("CREATE SCHEMA IF NOT EXISTS #{prefix}") 152 | 153 | # -------------- Transactions ---------------- 154 | 155 | create table("transactions", primary_key: false, prefix: prefix) do 156 | add(:id, :xid8, null: false, primary_key: true) 157 | add(:meta, :map, null: false, default: %{}) 158 | add(:processed_at, :utc_datetime_usec) 159 | add(:inserted_at, :utc_datetime_usec, null: false) 160 | end 161 | 162 | create( 163 | index("transactions", [:inserted_at], 164 | where: "processed_at IS NULL", 165 | prefix: prefix 166 | ) 167 | ) 168 | 169 | create_set_transaction_id_procedure(prefix) 170 | 171 | """ 172 | CREATE TRIGGER set_transaction_id_trigger 173 | BEFORE INSERT 174 | ON #{prefix}.transactions 175 | FOR EACH ROW 176 | EXECUTE PROCEDURE #{prefix}.set_transaction_id(); 177 | """ 178 | |> squish_and_execute() 179 | 180 | # ---------------- Changes ------------------- 181 | 182 | execute("CREATE TYPE #{prefix}.change_op AS ENUM('insert', 'update', 'delete');") 183 | 184 | create table("changes", primary_key: false, prefix: prefix) do 185 | add(:id, :bigserial, null: false, primary_key: true) 186 | 187 | add( 188 | :transaction_id, 189 | references(:transactions, 190 | on_delete: :delete_all, 191 | on_update: :update_all, 192 | column: :id, 193 | type: :xid8, 194 | prefix: prefix 195 | ), 196 | null: false 197 | ) 198 | 199 | add(:op, :"#{prefix}.change_op", null: false) 200 | add(:table_prefix, :string, null: false) 201 | add(:table_name, :string, null: false) 202 | add(:table_pk, {:array, :string}, null: true) 203 | add(:data, :jsonb, null: false) 204 | add(:changed, {:array, :string}, null: false) 205 | end 206 | 207 | create(index("changes", [:transaction_id], prefix: prefix)) 208 | create(index("changes", [:table_prefix, :table_name, :table_pk], prefix: prefix)) 209 | 210 | # ---------------- Triggers ------------------ 211 | 212 | execute("CREATE TYPE #{prefix}.trigger_mode AS ENUM('capture', 'ignore');") 213 | 214 | create table("triggers", primary_key: false, prefix: prefix) do 215 | add(:id, :bigserial, null: false, primary_key: true) 216 | add(:table_prefix, :string, null: false) 217 | add(:table_name, :string, null: false) 218 | add(:primary_key_columns, {:array, :string}, null: false) 219 | add(:excluded_columns, {:array, :string}, null: false) 220 | add(:filtered_columns, {:array, :string}, null: false) 221 | add(:mode, :"#{prefix}.trigger_mode", null: false) 222 | add(:override_transaction_id, :xid8, null: true) 223 | add(:inserted_at, :utc_datetime_usec, null: false) 224 | add(:updated_at, :utc_datetime_usec, null: false) 225 | end 226 | 227 | create_triggers_table_index(prefix) 228 | 229 | # ------------- Capture Function ------------- 230 | 231 | create_capture_changes_procedure(prefix) 232 | 233 | :ok 234 | end 235 | 236 | @type down_option :: {:carbonite_prefix, prefix()} | {:drop_schema, boolean()} 237 | 238 | @impl true 239 | @spec down([down_option()]) :: :ok 240 | def down(opts) when is_list(opts) do 241 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 242 | 243 | execute("DROP FUNCTION #{prefix}.capture_changes;") 244 | 245 | drop(table("triggers", prefix: prefix)) 246 | execute("DROP TYPE #{prefix}.trigger_mode;") 247 | 248 | drop(table("changes", prefix: prefix)) 249 | execute("DROP TYPE #{prefix}.change_op;") 250 | 251 | drop(table("transactions", prefix: prefix)) 252 | execute("DROP FUNCTION #{prefix}.set_transaction_id;") 253 | 254 | if Keyword.get(opts, :drop_schema, true) do 255 | execute("DROP SCHEMA #{prefix};") 256 | end 257 | 258 | :ok 259 | end 260 | end 261 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v10.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V10 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.{V1, V8} 9 | 10 | @type prefix :: binary() 11 | 12 | @type up_option :: {:carbonite_prefix, prefix()} 13 | 14 | @spec create_capture_changes_procedure(prefix) :: :ok 15 | def create_capture_changes_procedure(prefix) do 16 | """ 17 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 18 | $body$ 19 | DECLARE 20 | trigger_row RECORD; 21 | change_row #{prefix}.changes; 22 | BEGIN 23 | /* load trigger config */ 24 | WITH settings AS (SELECT NULLIF(current_setting('#{prefix}.override_mode', TRUE), '')::TEXT AS override_mode) 25 | SELECT 26 | primary_key_columns, 27 | excluded_columns, 28 | filtered_columns, 29 | CASE 30 | WHEN settings.override_mode = 'override' AND mode = 'ignore' THEN 'capture' 31 | WHEN settings.override_mode = 'override' AND mode = 'capture' THEN 'ignore' 32 | ELSE COALESCE(settings.override_mode, mode::text) 33 | END AS mode, 34 | store_changed_from 35 | INTO trigger_row 36 | FROM #{prefix}.triggers 37 | JOIN settings ON TRUE 38 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 39 | 40 | /* skip if ignored */ 41 | IF (trigger_row.mode = 'ignore') THEN 42 | RETURN NULL; 43 | END IF; 44 | 45 | /* instantiate change row */ 46 | change_row := ROW( 47 | NEXTVAL('#{prefix}.changes_id_seq'), 48 | pg_current_xact_id(), 49 | LOWER(TG_OP::TEXT), 50 | TG_TABLE_SCHEMA::TEXT, 51 | TG_TABLE_NAME::TEXT, 52 | NULL, 53 | NULL, 54 | '{}', 55 | NULL, 56 | NULL 57 | ); 58 | 59 | /* collect table pk */ 60 | IF trigger_row.primary_key_columns != '{}' THEN 61 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 62 | SELECT #{prefix}.record_dynamic_varchar_agg(NEW, trigger_row.primary_key_columns) 63 | INTO change_row.table_pk; 64 | ELSIF (TG_OP = 'DELETE') THEN 65 | SELECT #{prefix}.record_dynamic_varchar_agg(OLD, trigger_row.primary_key_columns) 66 | INTO change_row.table_pk; 67 | END IF; 68 | END IF; 69 | 70 | /* collect version data */ 71 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 72 | SELECT to_jsonb(NEW.*) - trigger_row.excluded_columns 73 | INTO change_row.data; 74 | ELSIF (TG_OP = 'DELETE') THEN 75 | SELECT to_jsonb(OLD.*) - trigger_row.excluded_columns 76 | INTO change_row.data; 77 | END IF; 78 | 79 | /* change tracking for UPDATEs */ 80 | IF (TG_OP = 'UPDATE') THEN 81 | change_row.changed_from = '{}'::JSONB; 82 | 83 | SELECT jsonb_object_agg(before.key, before.value) 84 | FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) AS before 85 | WHERE (change_row.data->before.key)::JSONB != before.value 86 | INTO change_row.changed_from; 87 | 88 | SELECT ARRAY(SELECT jsonb_object_keys(change_row.changed_from)) 89 | INTO change_row.changed; 90 | 91 | /* skip persisting this update if nothing has changed */ 92 | IF change_row.changed = '{}' THEN 93 | RETURN NULL; 94 | END IF; 95 | 96 | /* persisting the old data is opt-in, discard if not configured. */ 97 | IF trigger_row.store_changed_from IS FALSE THEN 98 | change_row.changed_from := NULL; 99 | END IF; 100 | END IF; 101 | 102 | /* filtered columns */ 103 | SELECT #{prefix}.jsonb_redact_keys(change_row.data, trigger_row.filtered_columns) 104 | INTO change_row.data; 105 | 106 | IF change_row.changed_from IS NOT NULL THEN 107 | SELECT #{prefix}.jsonb_redact_keys(change_row.changed_from, trigger_row.filtered_columns) 108 | INTO change_row.changed_from; 109 | END IF; 110 | 111 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 112 | BEGIN 113 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 114 | 115 | /* verify that xact_id matches */ 116 | IF NOT 117 | EXISTS( 118 | SELECT 1 FROM #{prefix}.transactions 119 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 120 | ) 121 | THEN 122 | RAISE USING ERRCODE = 'foreign_key_violation'; 123 | END IF; 124 | 125 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 126 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 127 | RAISE '(carbonite) % on table %.% without prior INSERT into #{prefix}.transactions', 128 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 129 | END; 130 | 131 | RETURN NULL; 132 | END; 133 | $body$ 134 | LANGUAGE plpgsql; 135 | """ 136 | |> squish_and_execute() 137 | 138 | :ok 139 | end 140 | 141 | @impl true 142 | @spec up([up_option()]) :: :ok 143 | def up(opts) do 144 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 145 | 146 | create_capture_changes_procedure(prefix) 147 | 148 | # Drops the table_index automatically. 149 | alter table(:triggers, prefix: prefix) do 150 | remove(:override_xact_id) 151 | end 152 | 153 | # This trigger is in fact rarely used when the triggers table is small. But let's keep it 154 | # in case someone has a lot of triggers in there. 155 | create( 156 | index("triggers", [:table_prefix, :table_name], 157 | name: "triggers_table_index", 158 | unique: true, 159 | include: [ 160 | :primary_key_columns, 161 | :excluded_columns, 162 | :filtered_columns, 163 | :mode, 164 | :store_changed_from 165 | ], 166 | prefix: prefix 167 | ) 168 | ) 169 | 170 | :ok 171 | end 172 | 173 | @type down_option :: {:carbonite_prefix, prefix()} 174 | 175 | @impl true 176 | @spec down([down_option()]) :: :ok 177 | def down(opts) do 178 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 179 | 180 | alter table(:triggers, prefix: prefix) do 181 | add(:override_xact_id, :xid8, null: true) 182 | end 183 | 184 | execute("DROP INDEX #{prefix}.triggers_table_index;") 185 | V1.create_triggers_table_index(prefix, :override_xact_id) 186 | 187 | V8.create_capture_changes_procedure(prefix) 188 | 189 | :ok 190 | end 191 | end 192 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v11.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V11 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.{V10, V8} 9 | 10 | @type prefix :: binary() 11 | 12 | @spec create_capture_changes_procedure(prefix) :: :ok 13 | def create_capture_changes_procedure(prefix) do 14 | """ 15 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 16 | $body$ 17 | DECLARE 18 | trigger_row RECORD; 19 | change_row #{prefix}.changes; 20 | 21 | pk_source RECORD; 22 | pk_col VARCHAR; 23 | pk_col_val VARCHAR; 24 | BEGIN 25 | /* load trigger config */ 26 | WITH settings AS (SELECT NULLIF(current_setting('#{prefix}.override_mode', TRUE), '')::TEXT AS override_mode) 27 | SELECT 28 | primary_key_columns, 29 | excluded_columns, 30 | filtered_columns, 31 | CASE 32 | WHEN settings.override_mode = 'override' AND mode = 'ignore' THEN 'capture' 33 | WHEN settings.override_mode = 'override' AND mode = 'capture' THEN 'ignore' 34 | ELSE COALESCE(settings.override_mode, mode::text) 35 | END AS mode, 36 | store_changed_from 37 | INTO trigger_row 38 | FROM #{prefix}.triggers 39 | JOIN settings ON TRUE 40 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 41 | 42 | /* skip if ignored */ 43 | IF (trigger_row.mode = 'ignore') THEN 44 | RETURN NULL; 45 | END IF; 46 | 47 | /* instantiate change row */ 48 | change_row := ROW( 49 | NEXTVAL('#{prefix}.changes_id_seq'), 50 | pg_current_xact_id(), 51 | LOWER(TG_OP::TEXT), 52 | TG_TABLE_SCHEMA::TEXT, 53 | TG_TABLE_NAME::TEXT, 54 | NULL, 55 | NULL, 56 | '{}', 57 | NULL, 58 | NULL 59 | ); 60 | 61 | /* collect table pk */ 62 | IF trigger_row.primary_key_columns != '{}' THEN 63 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 64 | pk_source := NEW; 65 | ELSIF (TG_OP = 'DELETE') THEN 66 | pk_source := OLD; 67 | END IF; 68 | 69 | change_row.table_pk = '{}'; 70 | FOREACH pk_col IN ARRAY trigger_row.primary_key_columns LOOP 71 | EXECUTE 'SELECT $1.' || quote_ident(pk_col) || '::TEXT' USING pk_source INTO pk_col_val; 72 | change_row.table_pk := change_row.table_pk || pk_col_val; 73 | END LOOP; 74 | END IF; 75 | 76 | /* collect version data */ 77 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 78 | SELECT to_jsonb(NEW.*) - trigger_row.excluded_columns 79 | INTO change_row.data; 80 | ELSIF (TG_OP = 'DELETE') THEN 81 | SELECT to_jsonb(OLD.*) - trigger_row.excluded_columns 82 | INTO change_row.data; 83 | END IF; 84 | 85 | /* change tracking for UPDATEs */ 86 | IF (TG_OP = 'UPDATE') THEN 87 | change_row.changed_from = '{}'::JSONB; 88 | 89 | SELECT jsonb_object_agg(before.key, before.value) 90 | FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) AS before 91 | WHERE (change_row.data->before.key)::JSONB != before.value 92 | INTO change_row.changed_from; 93 | 94 | SELECT ARRAY(SELECT jsonb_object_keys(change_row.changed_from)) 95 | INTO change_row.changed; 96 | 97 | /* skip persisting this update if nothing has changed */ 98 | IF change_row.changed = '{}' THEN 99 | RETURN NULL; 100 | END IF; 101 | 102 | /* persisting the old data is opt-in, discard if not configured. */ 103 | IF trigger_row.store_changed_from IS FALSE THEN 104 | change_row.changed_from := NULL; 105 | END IF; 106 | END IF; 107 | 108 | /* filtered columns */ 109 | SELECT #{prefix}.jsonb_redact_keys(change_row.data, trigger_row.filtered_columns) 110 | INTO change_row.data; 111 | 112 | IF change_row.changed_from IS NOT NULL THEN 113 | SELECT #{prefix}.jsonb_redact_keys(change_row.changed_from, trigger_row.filtered_columns) 114 | INTO change_row.changed_from; 115 | END IF; 116 | 117 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 118 | BEGIN 119 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 120 | 121 | /* verify that xact_id matches */ 122 | IF NOT 123 | EXISTS( 124 | SELECT 1 FROM #{prefix}.transactions 125 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 126 | ) 127 | THEN 128 | RAISE USING ERRCODE = 'foreign_key_violation'; 129 | END IF; 130 | 131 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 132 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 133 | RAISE '(carbonite) % on table %.% without prior INSERT into #{prefix}.transactions', 134 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 135 | END; 136 | 137 | RETURN NULL; 138 | END; 139 | $body$ 140 | LANGUAGE plpgsql; 141 | """ 142 | |> squish_and_execute() 143 | 144 | :ok 145 | end 146 | 147 | @type up_option :: {:carbonite_prefix, prefix()} 148 | 149 | @impl true 150 | @spec up([up_option()]) :: :ok 151 | def up(opts) do 152 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 153 | 154 | create_capture_changes_procedure(prefix) 155 | execute("DROP FUNCTION #{prefix}.record_dynamic_varchar_agg;") 156 | execute("DROP FUNCTION #{prefix}.record_dynamic_varchar;") 157 | 158 | :ok 159 | end 160 | 161 | @type down_option :: {:carbonite_prefix, prefix()} 162 | 163 | @impl true 164 | @spec down([down_option()]) :: :ok 165 | def down(opts) do 166 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 167 | 168 | V8.create_record_dynamic_varchar_procedure(prefix) 169 | V8.create_record_dynamic_varchar_agg_procedure(prefix) 170 | V10.create_capture_changes_procedure(prefix) 171 | 172 | :ok 173 | end 174 | end 175 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v12.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V12 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.V11 9 | 10 | @type prefix :: binary() 11 | 12 | defp create_capture_changes_procedure(prefix) do 13 | """ 14 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 15 | $body$ 16 | DECLARE 17 | trigger_row RECORD; 18 | change_row #{prefix}.changes; 19 | 20 | pk_source RECORD; 21 | pk_col VARCHAR; 22 | pk_col_val VARCHAR; 23 | BEGIN 24 | /* load trigger config */ 25 | WITH settings AS (SELECT NULLIF(current_setting('#{prefix}.override_mode', TRUE), '')::TEXT AS override_mode) 26 | SELECT 27 | primary_key_columns, 28 | excluded_columns, 29 | filtered_columns, 30 | CASE 31 | WHEN settings.override_mode = 'override' AND mode = 'ignore' THEN 'capture' 32 | WHEN settings.override_mode = 'override' AND mode = 'capture' THEN 'ignore' 33 | ELSE COALESCE(settings.override_mode, mode::text) 34 | END AS mode, 35 | store_changed_from 36 | INTO trigger_row 37 | FROM #{prefix}.triggers 38 | JOIN settings ON TRUE 39 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 40 | 41 | IF (trigger_row IS NULL) THEN 42 | RAISE '(carbonite) % on table %.% but no trigger record in #{prefix}.triggers', 43 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'no_data_found'; 44 | END IF; 45 | 46 | /* skip if ignored */ 47 | IF (trigger_row.mode = 'ignore') THEN 48 | RETURN NULL; 49 | END IF; 50 | 51 | /* instantiate change row */ 52 | change_row := ROW( 53 | NEXTVAL('#{prefix}.changes_id_seq'), 54 | pg_current_xact_id(), 55 | LOWER(TG_OP::TEXT), 56 | TG_TABLE_SCHEMA::TEXT, 57 | TG_TABLE_NAME::TEXT, 58 | NULL, 59 | NULL, 60 | '{}', 61 | NULL, 62 | NULL 63 | ); 64 | 65 | /* collect table pk */ 66 | IF trigger_row.primary_key_columns != '{}' THEN 67 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 68 | pk_source := NEW; 69 | ELSIF (TG_OP = 'DELETE') THEN 70 | pk_source := OLD; 71 | END IF; 72 | 73 | change_row.table_pk = '{}'; 74 | FOREACH pk_col IN ARRAY trigger_row.primary_key_columns LOOP 75 | EXECUTE 'SELECT $1.' || quote_ident(pk_col) || '::TEXT' USING pk_source INTO pk_col_val; 76 | change_row.table_pk := change_row.table_pk || pk_col_val; 77 | END LOOP; 78 | END IF; 79 | 80 | /* collect version data */ 81 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 82 | SELECT to_jsonb(NEW.*) - trigger_row.excluded_columns 83 | INTO change_row.data; 84 | ELSIF (TG_OP = 'DELETE') THEN 85 | SELECT to_jsonb(OLD.*) - trigger_row.excluded_columns 86 | INTO change_row.data; 87 | END IF; 88 | 89 | /* change tracking for UPDATEs */ 90 | IF (TG_OP = 'UPDATE') THEN 91 | change_row.changed_from = '{}'::JSONB; 92 | 93 | SELECT jsonb_object_agg(before.key, before.value) 94 | FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) AS before 95 | WHERE (change_row.data->before.key)::JSONB != before.value 96 | INTO change_row.changed_from; 97 | 98 | SELECT ARRAY(SELECT jsonb_object_keys(change_row.changed_from)) 99 | INTO change_row.changed; 100 | 101 | /* skip persisting this update if nothing has changed */ 102 | IF change_row.changed = '{}' THEN 103 | RETURN NULL; 104 | END IF; 105 | 106 | /* persisting the old data is opt-in, discard if not configured. */ 107 | IF trigger_row.store_changed_from IS FALSE THEN 108 | change_row.changed_from := NULL; 109 | END IF; 110 | END IF; 111 | 112 | /* filtered columns */ 113 | SELECT #{prefix}.jsonb_redact_keys(change_row.data, trigger_row.filtered_columns) 114 | INTO change_row.data; 115 | 116 | IF change_row.changed_from IS NOT NULL THEN 117 | SELECT #{prefix}.jsonb_redact_keys(change_row.changed_from, trigger_row.filtered_columns) 118 | INTO change_row.changed_from; 119 | END IF; 120 | 121 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 122 | BEGIN 123 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 124 | 125 | /* verify that xact_id matches */ 126 | IF NOT 127 | EXISTS( 128 | SELECT 1 FROM #{prefix}.transactions 129 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 130 | ) 131 | THEN 132 | RAISE USING ERRCODE = 'foreign_key_violation'; 133 | END IF; 134 | 135 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 136 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 137 | RAISE '(carbonite) % on table %.% without prior INSERT into #{prefix}.transactions', 138 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 139 | END; 140 | 141 | RETURN NULL; 142 | END; 143 | $body$ 144 | LANGUAGE plpgsql; 145 | """ 146 | |> squish_and_execute() 147 | 148 | :ok 149 | end 150 | 151 | @type up_option :: {:carbonite_prefix, prefix()} 152 | 153 | @impl true 154 | @spec up([up_option()]) :: :ok 155 | def up(opts) do 156 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 157 | 158 | create_capture_changes_procedure(prefix) 159 | 160 | :ok 161 | end 162 | 163 | @type down_option :: {:carbonite_prefix, prefix()} 164 | 165 | @impl true 166 | @spec down([down_option()]) :: :ok 167 | def down(opts) do 168 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 169 | 170 | V11.create_capture_changes_procedure(prefix) 171 | 172 | :ok 173 | end 174 | end 175 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v2.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V2 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | 9 | @type prefix :: binary() 10 | 11 | @type up_option :: {:carbonite_prefix, prefix()} 12 | 13 | @impl true 14 | @spec up([up_option()]) :: :ok 15 | def up(opts) do 16 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 17 | 18 | alter table("triggers", primary_key: false, prefix: prefix) do 19 | modify(:primary_key_columns, {:array, :string}, null: false, default: ["id"]) 20 | modify(:excluded_columns, {:array, :string}, null: false, default: []) 21 | modify(:filtered_columns, {:array, :string}, null: false, default: []) 22 | modify(:mode, :"#{prefix}.trigger_mode", null: false, default: "capture") 23 | end 24 | 25 | :ok 26 | end 27 | 28 | @impl true 29 | @spec down(any()) :: :ok 30 | def down(_opts) do 31 | :ok 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v3.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V3 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | 9 | @type prefix :: binary() 10 | 11 | @type up_option :: {:carbonite_prefix, prefix()} 12 | 13 | @impl true 14 | @spec up([up_option()]) :: :ok 15 | def up(opts) do 16 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 17 | 18 | create table("outboxes", primary_key: false, prefix: prefix) do 19 | add(:name, :string, null: false, primary_key: true) 20 | add(:memo, :map, null: false, default: "{}") 21 | add(:last_transaction_id, :xid8, null: false, default: "0") 22 | add(:inserted_at, :utc_datetime_usec, null: false) 23 | add(:updated_at, :utc_datetime_usec, null: false) 24 | end 25 | 26 | alter table("transactions", prefix: prefix) do 27 | remove(:processed_at, :utc_datetime_usec) 28 | end 29 | 30 | :ok 31 | end 32 | 33 | @type down_option :: {:carbonite_prefix, prefix()} 34 | 35 | @impl true 36 | @spec down([down_option()]) :: :ok 37 | def down(opts) do 38 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 39 | 40 | alter table("transactions", prefix: prefix) do 41 | add(:processed_at, :utc_datetime_usec, null: true) 42 | end 43 | 44 | create( 45 | index("transactions", [:inserted_at], 46 | where: "processed_at IS NULL", 47 | prefix: prefix 48 | ) 49 | ) 50 | 51 | drop(table("outboxes", prefix: prefix)) 52 | 53 | :ok 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v4.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V4 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.V1 9 | 10 | @type prefix :: binary() 11 | 12 | @type up_option :: {:carbonite_prefix, prefix()} 13 | 14 | # This buffer ensures that the sequence is initialized to a value hopefully greater than the 15 | # current xact id, even if new `transactions` records have been inserted during the migration. 16 | @xact_id_buffer 5000 17 | 18 | @spec create_capture_changes_procedure(prefix()) :: :ok 19 | def create_capture_changes_procedure(prefix) do 20 | """ 21 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 22 | $body$ 23 | DECLARE 24 | trigger_row #{prefix}.triggers; 25 | change_row #{prefix}.changes; 26 | pk_source RECORD; 27 | col_name VARCHAR; 28 | pk_col_val VARCHAR; 29 | old_field RECORD; 30 | BEGIN 31 | /* load trigger config */ 32 | SELECT * 33 | INTO trigger_row 34 | FROM #{prefix}.triggers 35 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 36 | 37 | IF 38 | (trigger_row.mode = 'ignore' AND (trigger_row.override_xact_id IS NULL OR trigger_row.override_xact_id != pg_current_xact_id())) OR 39 | (trigger_row.mode = 'capture' AND trigger_row.override_xact_id = pg_current_xact_id()) 40 | THEN 41 | RETURN NULL; 42 | END IF; 43 | 44 | /* instantiate change row */ 45 | change_row = ROW( 46 | NEXTVAL('#{prefix}.changes_id_seq'), 47 | pg_current_xact_id(), 48 | LOWER(TG_OP::TEXT), 49 | TG_TABLE_SCHEMA::TEXT, 50 | TG_TABLE_NAME::TEXT, 51 | NULL, 52 | NULL, 53 | '{}', 54 | NULL 55 | ); 56 | 57 | /* build table_pk */ 58 | IF trigger_row.primary_key_columns != '{}' THEN 59 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 60 | pk_source := NEW; 61 | ELSIF (TG_OP = 'DELETE') THEN 62 | pk_source := OLD; 63 | END IF; 64 | 65 | change_row.table_pk := '{}'; 66 | 67 | FOREACH col_name IN ARRAY trigger_row.primary_key_columns LOOP 68 | EXECUTE 'SELECT $1.' || col_name || '::TEXT' USING pk_source INTO pk_col_val; 69 | change_row.table_pk := change_row.table_pk || pk_col_val; 70 | END LOOP; 71 | END IF; 72 | 73 | /* fill in changed data */ 74 | IF (TG_OP = 'UPDATE') THEN 75 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 76 | 77 | FOR old_field IN SELECT * FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) LOOP 78 | IF NOT change_row.data @> jsonb_build_object(old_field.key, old_field.value) 79 | THEN change_row.changed := change_row.changed || old_field.key::VARCHAR; 80 | END IF; 81 | END LOOP; 82 | 83 | IF change_row.changed = '{}' THEN 84 | /* All changed fields are ignored. Skip this update. */ 85 | RETURN NULL; 86 | END IF; 87 | ELSIF (TG_OP = 'DELETE') THEN 88 | change_row.data = to_jsonb(OLD.*) - trigger_row.excluded_columns; 89 | ELSIF (TG_OP = 'INSERT') THEN 90 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 91 | END IF; 92 | 93 | /* filtered columns */ 94 | FOREACH col_name IN ARRAY trigger_row.filtered_columns LOOP 95 | change_row.data = jsonb_set(change_row.data, ('{' || col_name || '}')::TEXT[], jsonb('"[FILTERED]"')); 96 | END LOOP; 97 | 98 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 99 | BEGIN 100 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 101 | 102 | /* verify that xact_id matches */ 103 | IF NOT 104 | EXISTS( 105 | SELECT 1 FROM #{prefix}.transactions 106 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 107 | ) 108 | THEN 109 | RAISE USING ERRCODE = 'foreign_key_violation'; 110 | END IF; 111 | 112 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 113 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 114 | RAISE '% on table %.% without prior INSERT into #{prefix}.transactions', 115 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 116 | END; 117 | 118 | RETURN NULL; 119 | END; 120 | $body$ 121 | LANGUAGE plpgsql; 122 | """ 123 | |> squish_and_execute() 124 | 125 | :ok 126 | end 127 | 128 | @spec create_set_transaction_id_procedure(prefix()) :: :ok 129 | def create_set_transaction_id_procedure(prefix) do 130 | """ 131 | CREATE OR REPLACE FUNCTION #{prefix}.set_transaction_id() RETURNS TRIGGER AS 132 | $body$ 133 | BEGIN 134 | BEGIN 135 | /* verify that no previous INSERT within current transaction (with same id) */ 136 | IF 137 | EXISTS( 138 | SELECT 1 FROM #{prefix}.transactions 139 | WHERE id = COALESCE(NEW.id, CURRVAL('#{prefix}.transactions_id_seq')) 140 | AND xact_id = COALESCE(NEW.xact_id, pg_current_xact_id()) 141 | ) 142 | THEN 143 | NEW.id = COALESCE(NEW.id, CURRVAL('#{prefix}.transactions_id_seq')); 144 | END IF; 145 | EXCEPTION WHEN object_not_in_prerequisite_state THEN 146 | /* when NEXTVAL has never been called within session, we're good */ 147 | END; 148 | 149 | NEW.id = COALESCE(NEW.id, NEXTVAL('#{prefix}.transactions_id_seq')); 150 | NEW.xact_id = COALESCE(NEW.xact_id, pg_current_xact_id()); 151 | 152 | RETURN NEW; 153 | END 154 | $body$ 155 | LANGUAGE plpgsql; 156 | """ 157 | |> squish_and_execute() 158 | 159 | :ok 160 | end 161 | 162 | @impl true 163 | @spec up([up_option()]) :: :ok 164 | def up(opts) do 165 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 166 | 167 | lock_changes(prefix) 168 | 169 | # ------------- Change constraints ----------- 170 | 171 | temporarily_drop_fk_on_changes(prefix, fn -> 172 | rename_id_column(prefix, "transactions", :id, :xact_id) 173 | rename_id_column(prefix, "changes", :transaction_id, :transaction_xact_id) 174 | 175 | squish_and_execute("ALTER TABLE #{prefix}.transactions DROP CONSTRAINT transactions_pkey;") 176 | 177 | squish_and_execute( 178 | "ALTER TABLE #{prefix}.transactions ADD PRIMARY KEY (id) INCLUDE (xact_id);" 179 | ) 180 | end) 181 | 182 | temporarily_drop_default_on_outboxes(prefix, "0", fn -> 183 | change_type(prefix, "outboxes", "last_transaction_id", "BIGINT") 184 | end) 185 | 186 | # ------------- New ID sequence -------------- 187 | 188 | %Postgrex.Result{rows: [[seq_start_with]]} = 189 | repo().query!("SELECT pg_current_xact_id()::TEXT::BIGINT + #{@xact_id_buffer};") 190 | 191 | """ 192 | CREATE SEQUENCE #{prefix}.transactions_id_seq 193 | START WITH #{seq_start_with} 194 | OWNED BY #{prefix}.transactions.id; 195 | """ 196 | |> squish_and_execute() 197 | 198 | create_set_transaction_id_procedure(prefix) 199 | 200 | # ------------- override_xact_id ------------- 201 | 202 | rename(table("triggers", prefix: prefix), :override_transaction_id, to: :override_xact_id) 203 | 204 | # ------------- Capture Function ------------- 205 | 206 | create_capture_changes_procedure(prefix) 207 | 208 | :ok 209 | end 210 | 211 | @type down_option :: {:carbonite_prefix, prefix()} 212 | 213 | @impl true 214 | @spec down([down_option()]) :: :ok 215 | def down(opts) do 216 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 217 | 218 | lock_changes(prefix) 219 | 220 | # ------------- Change constraints ----------- 221 | 222 | temporarily_drop_fk_on_changes(prefix, fn -> 223 | squish_and_execute("ALTER TABLE #{prefix}.transactions DROP CONSTRAINT transactions_pkey;") 224 | squish_and_execute("ALTER TABLE #{prefix}.transactions ADD PRIMARY KEY (xact_id);") 225 | 226 | revert_id_column(prefix, "changes", :transaction_xact_id, :transaction_id) 227 | revert_id_column(prefix, "transactions", :xact_id, :id) 228 | end) 229 | 230 | temporarily_drop_default_on_outboxes(prefix, "0", fn -> 231 | change_type(prefix, "outboxes", "last_transaction_id", "BIGINT") 232 | end) 233 | 234 | # ------------- override_xact_id ------------- 235 | 236 | rename(table("triggers", prefix: prefix), :override_xact_id, to: :override_transaction_id) 237 | 238 | # ------------ Restore functions ------------- 239 | 240 | V1.create_set_transaction_id_procedure(prefix) 241 | V1.create_capture_changes_procedure(prefix) 242 | 243 | :ok 244 | end 245 | 246 | defp lock_changes(prefix) do 247 | squish_and_execute("LOCK TABLE #{prefix}.changes IN EXCLUSIVE MODE;") 248 | end 249 | 250 | defp rename_id_column(prefix, table, from, to) do 251 | rename(table(table, prefix: prefix), from, to: to) 252 | 253 | alter table(table, prefix: prefix) do 254 | add(from, :bigint, null: true) 255 | end 256 | 257 | squish_and_execute("UPDATE #{prefix}.#{table} SET #{from} = #{to}::TEXT::BIGINT;") 258 | 259 | alter table(table, prefix: prefix) do 260 | modify(from, :bigint, null: false) 261 | end 262 | end 263 | 264 | defp revert_id_column(prefix, table, from, to) do 265 | alter table(table, prefix: prefix) do 266 | remove(to) 267 | end 268 | 269 | rename(table(table, prefix: prefix), from, to: to) 270 | end 271 | 272 | defp change_type(prefix, table, column, type) do 273 | """ 274 | ALTER TABLE #{prefix}.#{table} 275 | ALTER COLUMN #{column} 276 | SET DATA TYPE #{type} 277 | USING #{column}::text::#{type}; 278 | """ 279 | |> squish_and_execute() 280 | end 281 | 282 | defp temporarily_drop_fk_on_changes(prefix, callback) do 283 | """ 284 | ALTER TABLE #{prefix}.changes 285 | DROP CONSTRAINT changes_transaction_id_fkey; 286 | """ 287 | |> squish_and_execute() 288 | 289 | callback.() 290 | 291 | """ 292 | ALTER TABLE #{prefix}.changes 293 | ADD CONSTRAINT changes_transaction_id_fkey 294 | FOREIGN KEY (transaction_id) 295 | REFERENCES #{prefix}.transactions 296 | ON DELETE CASCADE 297 | ON UPDATE CASCADE; 298 | """ 299 | |> squish_and_execute() 300 | end 301 | 302 | defp temporarily_drop_default_on_outboxes(prefix, default, callback) do 303 | """ 304 | ALTER TABLE #{prefix}.outboxes 305 | ALTER COLUMN last_transaction_id 306 | DROP DEFAULT; 307 | """ 308 | |> squish_and_execute() 309 | 310 | callback.() 311 | 312 | """ 313 | ALTER TABLE #{prefix}.outboxes 314 | ALTER COLUMN last_transaction_id 315 | SET DEFAULT #{default}; 316 | """ 317 | |> squish_and_execute() 318 | end 319 | end 320 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v5.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V5 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.V4 9 | 10 | @type prefix :: binary() 11 | 12 | @type up_option :: {:carbonite_prefix, prefix()} 13 | 14 | @spec create_capture_changes_procedure(prefix()) :: :ok 15 | def create_capture_changes_procedure(prefix) do 16 | """ 17 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 18 | $body$ 19 | DECLARE 20 | trigger_row #{prefix}.triggers; 21 | change_row #{prefix}.changes; 22 | pk_source RECORD; 23 | col_name VARCHAR; 24 | pk_col_val VARCHAR; 25 | old_field RECORD; 26 | old_field_jsonb JSONB; 27 | BEGIN 28 | /* load trigger config */ 29 | SELECT * 30 | INTO trigger_row 31 | FROM #{prefix}.triggers 32 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 33 | 34 | IF 35 | (trigger_row.mode = 'ignore' AND (trigger_row.override_xact_id IS NULL OR trigger_row.override_xact_id != pg_current_xact_id())) OR 36 | (trigger_row.mode = 'capture' AND trigger_row.override_xact_id = pg_current_xact_id()) 37 | THEN 38 | RETURN NULL; 39 | END IF; 40 | 41 | /* instantiate change row */ 42 | change_row = ROW( 43 | NEXTVAL('#{prefix}.changes_id_seq'), 44 | pg_current_xact_id(), 45 | LOWER(TG_OP::TEXT), 46 | TG_TABLE_SCHEMA::TEXT, 47 | TG_TABLE_NAME::TEXT, 48 | NULL, 49 | NULL, 50 | '{}', 51 | NULL, 52 | NULL 53 | ); 54 | 55 | /* build table_pk */ 56 | IF trigger_row.primary_key_columns != '{}' THEN 57 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 58 | pk_source := NEW; 59 | ELSIF (TG_OP = 'DELETE') THEN 60 | pk_source := OLD; 61 | END IF; 62 | 63 | change_row.table_pk := '{}'; 64 | 65 | FOREACH col_name IN ARRAY trigger_row.primary_key_columns LOOP 66 | EXECUTE 'SELECT $1.' || col_name || '::TEXT' USING pk_source INTO pk_col_val; 67 | change_row.table_pk := change_row.table_pk || pk_col_val; 68 | END LOOP; 69 | END IF; 70 | 71 | /* fill in changed data */ 72 | IF (TG_OP = 'UPDATE') THEN 73 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 74 | change_row.changed_from = '{}'::JSONB; 75 | 76 | FOR old_field_jsonb 77 | IN SELECT jsonb_build_object(key, value) 78 | FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) 79 | LOOP 80 | IF NOT change_row.data @> old_field_jsonb THEN 81 | change_row.changed_from := change_row.changed_from || old_field_jsonb; 82 | END IF; 83 | END LOOP; 84 | 85 | change_row.changed := ARRAY(SELECT jsonb_object_keys(change_row.changed_from)); 86 | 87 | IF change_row.changed = '{}' THEN 88 | /* All changed fields are ignored. Skip this update. */ 89 | RETURN NULL; 90 | END IF; 91 | 92 | /* Persisting the old data is opt-in, discard if not configured. */ 93 | IF trigger_row.store_changed_from IS FALSE THEN 94 | change_row.changed_from := NULL; 95 | END IF; 96 | ELSIF (TG_OP = 'DELETE') THEN 97 | change_row.data = to_jsonb(OLD.*) - trigger_row.excluded_columns; 98 | ELSIF (TG_OP = 'INSERT') THEN 99 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 100 | END IF; 101 | 102 | /* filtered columns */ 103 | FOREACH col_name IN ARRAY trigger_row.filtered_columns LOOP 104 | change_row.data = jsonb_set(change_row.data, ('{' || col_name || '}')::TEXT[], jsonb('"[FILTERED]"')); 105 | END LOOP; 106 | 107 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 108 | BEGIN 109 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 110 | 111 | /* verify that xact_id matches */ 112 | IF NOT 113 | EXISTS( 114 | SELECT 1 FROM #{prefix}.transactions 115 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 116 | ) 117 | THEN 118 | RAISE USING ERRCODE = 'foreign_key_violation'; 119 | END IF; 120 | 121 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 122 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 123 | RAISE '% on table %.% without prior INSERT into #{prefix}.transactions', 124 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 125 | END; 126 | 127 | RETURN NULL; 128 | END; 129 | $body$ 130 | LANGUAGE plpgsql; 131 | """ 132 | |> squish_and_execute() 133 | 134 | :ok 135 | end 136 | 137 | @impl true 138 | @spec up([up_option()]) :: :ok 139 | def up(opts) do 140 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 141 | 142 | lock_table(prefix, "changes") 143 | 144 | # ------------- `changed_from` --------------- 145 | 146 | alter table(:changes, prefix: prefix) do 147 | add(:changed_from, :jsonb, null: true) 148 | end 149 | 150 | alter table(:triggers, prefix: prefix) do 151 | add(:store_changed_from, :boolean, default: false, null: false) 152 | end 153 | 154 | # ------------- Capture Function ------------- 155 | 156 | create_capture_changes_procedure(prefix) 157 | 158 | :ok 159 | end 160 | 161 | @type down_option :: {:carbonite_prefix, prefix()} 162 | 163 | @impl true 164 | @spec down([down_option()]) :: :ok 165 | def down(opts) do 166 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 167 | 168 | lock_table(prefix, "changes") 169 | 170 | # ------------- `changed_from` ------------ 171 | 172 | alter table(:changes, prefix: prefix) do 173 | remove(:changed_from) 174 | end 175 | 176 | alter table(:triggers, prefix: prefix) do 177 | remove(:store_changed_from) 178 | end 179 | 180 | # ------------ Restore functions ------------- 181 | 182 | V4.create_capture_changes_procedure(prefix) 183 | 184 | :ok 185 | end 186 | end 187 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v6.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V6 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.V5 9 | 10 | @type prefix :: binary() 11 | 12 | @type up_option :: {:carbonite_prefix, prefix()} 13 | 14 | @spec create_capture_changes_procedure(prefix()) :: :ok 15 | def create_capture_changes_procedure(prefix) do 16 | """ 17 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 18 | $body$ 19 | DECLARE 20 | trigger_row #{prefix}.triggers; 21 | change_row #{prefix}.changes; 22 | pk_source RECORD; 23 | col_name VARCHAR; 24 | pk_col_val VARCHAR; 25 | old_value JSONB; 26 | BEGIN 27 | /* load trigger config */ 28 | SELECT * 29 | INTO trigger_row 30 | FROM #{prefix}.triggers 31 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 32 | 33 | IF 34 | (trigger_row.mode = 'ignore' AND (trigger_row.override_xact_id IS NULL OR trigger_row.override_xact_id != pg_current_xact_id())) OR 35 | (trigger_row.mode = 'capture' AND trigger_row.override_xact_id = pg_current_xact_id()) 36 | THEN 37 | RETURN NULL; 38 | END IF; 39 | 40 | /* instantiate change row */ 41 | change_row = ROW( 42 | NEXTVAL('#{prefix}.changes_id_seq'), 43 | pg_current_xact_id(), 44 | LOWER(TG_OP::TEXT), 45 | TG_TABLE_SCHEMA::TEXT, 46 | TG_TABLE_NAME::TEXT, 47 | NULL, 48 | NULL, 49 | '{}', 50 | NULL, 51 | NULL 52 | ); 53 | 54 | /* build table_pk */ 55 | IF trigger_row.primary_key_columns != '{}' THEN 56 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 57 | pk_source := NEW; 58 | ELSIF (TG_OP = 'DELETE') THEN 59 | pk_source := OLD; 60 | END IF; 61 | 62 | change_row.table_pk := '{}'; 63 | 64 | FOREACH col_name IN ARRAY trigger_row.primary_key_columns LOOP 65 | EXECUTE 'SELECT $1.' || col_name || '::TEXT' USING pk_source INTO pk_col_val; 66 | change_row.table_pk := change_row.table_pk || pk_col_val; 67 | END LOOP; 68 | END IF; 69 | 70 | /* fill in changed data */ 71 | IF (TG_OP = 'UPDATE') THEN 72 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 73 | change_row.changed_from = '{}'::JSONB; 74 | 75 | FOR col_name, old_value 76 | IN SELECT * FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) 77 | LOOP 78 | IF (change_row.data->col_name)::JSONB != old_value THEN 79 | change_row.changed_from := jsonb_set(change_row.changed_from, ARRAY[col_name], old_value); 80 | END IF; 81 | END LOOP; 82 | 83 | change_row.changed := ARRAY(SELECT jsonb_object_keys(change_row.changed_from)); 84 | 85 | IF change_row.changed = '{}' THEN 86 | /* All changed fields are ignored. Skip this update. */ 87 | RETURN NULL; 88 | END IF; 89 | 90 | /* Persisting the old data is opt-in, discard if not configured. */ 91 | IF trigger_row.store_changed_from IS FALSE THEN 92 | change_row.changed_from := NULL; 93 | END IF; 94 | ELSIF (TG_OP = 'DELETE') THEN 95 | change_row.data = to_jsonb(OLD.*) - trigger_row.excluded_columns; 96 | ELSIF (TG_OP = 'INSERT') THEN 97 | change_row.data = to_jsonb(NEW.*) - trigger_row.excluded_columns; 98 | END IF; 99 | 100 | /* filtered columns */ 101 | FOREACH col_name IN ARRAY trigger_row.filtered_columns LOOP 102 | change_row.data = jsonb_set(change_row.data, ('{' || col_name || '}')::TEXT[], jsonb('"[FILTERED]"')); 103 | END LOOP; 104 | 105 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 106 | BEGIN 107 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 108 | 109 | /* verify that xact_id matches */ 110 | IF NOT 111 | EXISTS( 112 | SELECT 1 FROM #{prefix}.transactions 113 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 114 | ) 115 | THEN 116 | RAISE USING ERRCODE = 'foreign_key_violation'; 117 | END IF; 118 | 119 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 120 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 121 | RAISE '% on table %.% without prior INSERT into #{prefix}.transactions', 122 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 123 | END; 124 | 125 | RETURN NULL; 126 | END; 127 | $body$ 128 | LANGUAGE plpgsql; 129 | """ 130 | |> squish_and_execute() 131 | 132 | :ok 133 | end 134 | 135 | @impl true 136 | @spec up([up_option()]) :: :ok 137 | def up(opts) do 138 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 139 | 140 | create_capture_changes_procedure(prefix) 141 | 142 | :ok 143 | end 144 | 145 | @type down_option :: {:carbonite_prefix, prefix()} 146 | 147 | @impl true 148 | @spec down([down_option()]) :: :ok 149 | def down(opts) do 150 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 151 | 152 | V5.create_capture_changes_procedure(prefix) 153 | 154 | :ok 155 | end 156 | end 157 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v7.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V7 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | 9 | @type prefix :: binary() 10 | 11 | @type up_option :: {:carbonite_prefix, prefix()} | {:concurrently, boolean()} 12 | 13 | @impl true 14 | @spec up([up_option()]) :: :ok 15 | def up(opts) do 16 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 17 | concurrently = Keyword.get(opts, :concurrently, false) 18 | 19 | # Ecto.Migration.rename/2 does not yet support :prefix 20 | # https://github.com/elixir-ecto/ecto_sql/pull/573 21 | 22 | """ 23 | ALTER INDEX #{prefix}.changes_transaction_id_index 24 | RENAME TO changes_transaction_xact_id_index; 25 | """ 26 | |> squish_and_execute() 27 | 28 | create(index(:changes, [:transaction_id], prefix: prefix, concurrently: concurrently)) 29 | 30 | :ok 31 | end 32 | 33 | @type down_option :: {:carbonite_prefix, prefix()} 34 | 35 | @impl true 36 | @spec down([down_option()]) :: :ok 37 | def down(opts) do 38 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 39 | 40 | drop(index(:changes, [:transaction_id], prefix: prefix)) 41 | 42 | """ 43 | ALTER INDEX #{prefix}.changes_transaction_xact_id_index 44 | RENAME TO changes_transaction_id_index; 45 | """ 46 | |> squish_and_execute() 47 | 48 | :ok 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v8.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V8 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.V6 9 | 10 | @type prefix :: binary() 11 | 12 | @type up_option :: {:carbonite_prefix, prefix()} 13 | 14 | @spec create_record_dynamic_varchar_procedure(prefix) :: :ok 15 | def create_record_dynamic_varchar_procedure(prefix) do 16 | """ 17 | CREATE OR REPLACE FUNCTION #{prefix}.record_dynamic_varchar(source RECORD, col VARCHAR) 18 | RETURNS VARCHAR AS 19 | $body$ 20 | DECLARE 21 | result VARCHAR; 22 | BEGIN 23 | EXECUTE 'SELECT $1.' || quote_ident(col) || '::TEXT' USING source INTO result; 24 | 25 | RETURN result; 26 | END; 27 | $body$ 28 | LANGUAGE plpgsql; 29 | """ 30 | |> squish_and_execute() 31 | end 32 | 33 | @spec create_record_dynamic_varchar_agg_procedure(prefix) :: :ok 34 | def create_record_dynamic_varchar_agg_procedure(prefix) do 35 | """ 36 | CREATE OR REPLACE FUNCTION #{prefix}.record_dynamic_varchar_agg(source RECORD, cols VARCHAR[]) 37 | RETURNS VARCHAR[] AS 38 | $body$ 39 | DECLARE 40 | col VARCHAR; 41 | result VARCHAR[]; 42 | BEGIN 43 | result := '{}'; 44 | 45 | FOREACH col IN ARRAY cols LOOP 46 | result := result || (SELECT #{prefix}.record_dynamic_varchar(source, col)); 47 | END LOOP; 48 | 49 | RETURN result; 50 | END; 51 | $body$ 52 | LANGUAGE plpgsql; 53 | """ 54 | |> squish_and_execute() 55 | end 56 | 57 | defp create_jsonb_redact_keys_procedure(prefix) do 58 | """ 59 | CREATE OR REPLACE FUNCTION #{prefix}.jsonb_redact_keys(source JSONB, keys VARCHAR[]) RETURNS JSONB AS 60 | $body$ 61 | DECLARE 62 | keys_intersect VARCHAR[]; 63 | key VARCHAR; 64 | BEGIN 65 | SELECT ARRAY( 66 | SELECT UNNEST(ARRAY(SELECT jsonb_object_keys(source))) 67 | INTERSECT 68 | SELECT UNNEST(keys) 69 | ) INTO keys_intersect; 70 | 71 | FOREACH key IN ARRAY keys_intersect LOOP 72 | source := jsonb_set(source, ('{' || key || '}')::TEXT[], jsonb('"[FILTERED]"')); 73 | END LOOP; 74 | 75 | RETURN source; 76 | END; 77 | $body$ 78 | LANGUAGE plpgsql; 79 | """ 80 | |> squish_and_execute() 81 | end 82 | 83 | @spec create_capture_changes_procedure(prefix) :: :ok 84 | def create_capture_changes_procedure(prefix) do 85 | """ 86 | CREATE OR REPLACE FUNCTION #{prefix}.capture_changes() RETURNS TRIGGER AS 87 | $body$ 88 | DECLARE 89 | trigger_row #{prefix}.triggers; 90 | change_row #{prefix}.changes; 91 | BEGIN 92 | /* load trigger config */ 93 | SELECT * 94 | INTO trigger_row 95 | FROM #{prefix}.triggers 96 | WHERE table_prefix = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME; 97 | 98 | IF 99 | (trigger_row.mode = 'ignore' AND (trigger_row.override_xact_id IS NULL OR trigger_row.override_xact_id != pg_current_xact_id())) OR 100 | (trigger_row.mode = 'capture' AND trigger_row.override_xact_id = pg_current_xact_id()) 101 | THEN 102 | RETURN NULL; 103 | END IF; 104 | 105 | /* instantiate change row */ 106 | change_row := ROW( 107 | NEXTVAL('#{prefix}.changes_id_seq'), 108 | pg_current_xact_id(), 109 | LOWER(TG_OP::TEXT), 110 | TG_TABLE_SCHEMA::TEXT, 111 | TG_TABLE_NAME::TEXT, 112 | NULL, 113 | NULL, 114 | '{}', 115 | NULL, 116 | NULL 117 | ); 118 | 119 | /* collect table pk */ 120 | IF trigger_row.primary_key_columns != '{}' THEN 121 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 122 | SELECT #{prefix}.record_dynamic_varchar_agg(NEW, trigger_row.primary_key_columns) 123 | INTO change_row.table_pk; 124 | ELSIF (TG_OP = 'DELETE') THEN 125 | SELECT #{prefix}.record_dynamic_varchar_agg(OLD, trigger_row.primary_key_columns) 126 | INTO change_row.table_pk; 127 | END IF; 128 | END IF; 129 | 130 | /* collect version data */ 131 | IF (TG_OP IN ('INSERT', 'UPDATE')) THEN 132 | SELECT to_jsonb(NEW.*) - trigger_row.excluded_columns 133 | INTO change_row.data; 134 | ELSIF (TG_OP = 'DELETE') THEN 135 | SELECT to_jsonb(OLD.*) - trigger_row.excluded_columns 136 | INTO change_row.data; 137 | END IF; 138 | 139 | /* change tracking for UPDATEs */ 140 | IF (TG_OP = 'UPDATE') THEN 141 | change_row.changed_from = '{}'::JSONB; 142 | 143 | SELECT jsonb_object_agg(before.key, before.value) 144 | FROM jsonb_each(to_jsonb(OLD.*) - trigger_row.excluded_columns) AS before 145 | WHERE (change_row.data->before.key)::JSONB != before.value 146 | INTO change_row.changed_from; 147 | 148 | SELECT ARRAY(SELECT jsonb_object_keys(change_row.changed_from)) 149 | INTO change_row.changed; 150 | 151 | /* skip persisting this update if nothing has changed */ 152 | IF change_row.changed = '{}' THEN 153 | RETURN NULL; 154 | END IF; 155 | 156 | /* persisting the old data is opt-in, discard if not configured. */ 157 | IF trigger_row.store_changed_from IS FALSE THEN 158 | change_row.changed_from := NULL; 159 | END IF; 160 | END IF; 161 | 162 | /* filtered columns */ 163 | SELECT #{prefix}.jsonb_redact_keys(change_row.data, trigger_row.filtered_columns) 164 | INTO change_row.data; 165 | 166 | IF change_row.changed_from IS NOT NULL THEN 167 | SELECT #{prefix}.jsonb_redact_keys(change_row.changed_from, trigger_row.filtered_columns) 168 | INTO change_row.changed_from; 169 | END IF; 170 | 171 | /* insert, fail gracefully unless transaction record present or NEXTVAL has never been called */ 172 | BEGIN 173 | change_row.transaction_id = CURRVAL('#{prefix}.transactions_id_seq'); 174 | 175 | /* verify that xact_id matches */ 176 | IF NOT 177 | EXISTS( 178 | SELECT 1 FROM #{prefix}.transactions 179 | WHERE id = change_row.transaction_id AND xact_id = change_row.transaction_xact_id 180 | ) 181 | THEN 182 | RAISE USING ERRCODE = 'foreign_key_violation'; 183 | END IF; 184 | 185 | INSERT INTO #{prefix}.changes VALUES (change_row.*); 186 | EXCEPTION WHEN foreign_key_violation OR object_not_in_prerequisite_state THEN 187 | RAISE '% on table %.% without prior INSERT into #{prefix}.transactions', 188 | TG_OP, TG_TABLE_SCHEMA, TG_TABLE_NAME USING ERRCODE = 'foreign_key_violation'; 189 | END; 190 | 191 | RETURN NULL; 192 | END; 193 | $body$ 194 | LANGUAGE plpgsql; 195 | """ 196 | |> squish_and_execute() 197 | 198 | :ok 199 | end 200 | 201 | @impl true 202 | @spec up([up_option()]) :: :ok 203 | def up(opts) do 204 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 205 | 206 | create_record_dynamic_varchar_procedure(prefix) 207 | create_record_dynamic_varchar_agg_procedure(prefix) 208 | create_jsonb_redact_keys_procedure(prefix) 209 | create_capture_changes_procedure(prefix) 210 | 211 | :ok 212 | end 213 | 214 | @type down_option :: {:carbonite_prefix, prefix()} 215 | 216 | @impl true 217 | @spec down([down_option()]) :: :ok 218 | def down(opts) do 219 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 220 | 221 | V6.create_capture_changes_procedure(prefix) 222 | execute("DROP FUNCTION #{prefix}.jsonb_redact_keys;") 223 | execute("DROP FUNCTION #{prefix}.record_dynamic_varchar_agg;") 224 | execute("DROP FUNCTION #{prefix}.record_dynamic_varchar;") 225 | 226 | :ok 227 | end 228 | end 229 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/v9.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.V9 do 4 | @moduledoc false 5 | 6 | use Ecto.Migration 7 | use Carbonite.Migrations.Version 8 | alias Carbonite.Migrations.V4 9 | 10 | @type prefix :: binary() 11 | 12 | @spec create_set_transaction_id_procedure(prefix()) :: :ok 13 | def create_set_transaction_id_procedure(prefix) do 14 | """ 15 | CREATE OR REPLACE FUNCTION #{prefix}.set_transaction_id() RETURNS TRIGGER AS 16 | $body$ 17 | BEGIN 18 | BEGIN 19 | /* verify that no previous INSERT within current transaction (with same id) */ 20 | IF 21 | EXISTS( 22 | WITH constants AS ( 23 | SELECT 24 | COALESCE(NEW.id, CURRVAL('#{prefix}.transactions_id_seq')) AS id, 25 | COALESCE(NEW.xact_id, pg_current_xact_id()) AS xact_id 26 | ) 27 | SELECT 1 FROM #{prefix}.transactions 28 | JOIN constants 29 | ON constants.id = transactions.id 30 | AND constants.xact_id = transactions.xact_id 31 | ) 32 | THEN 33 | NEW.id = COALESCE(NEW.id, CURRVAL('#{prefix}.transactions_id_seq')); 34 | END IF; 35 | EXCEPTION WHEN object_not_in_prerequisite_state THEN 36 | /* when NEXTVAL has never been called within session, we're good */ 37 | END; 38 | 39 | NEW.id = COALESCE(NEW.id, NEXTVAL('#{prefix}.transactions_id_seq')); 40 | NEW.xact_id = COALESCE(NEW.xact_id, pg_current_xact_id()); 41 | 42 | RETURN NEW; 43 | END 44 | $body$ 45 | LANGUAGE plpgsql; 46 | """ 47 | |> squish_and_execute() 48 | 49 | :ok 50 | end 51 | 52 | @type up_option :: {:carbonite_prefix, prefix()} 53 | 54 | @impl true 55 | @spec up([up_option()]) :: :ok 56 | def up(opts) do 57 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 58 | 59 | create_set_transaction_id_procedure(prefix) 60 | 61 | :ok 62 | end 63 | 64 | @type down_option :: {:carbonite_prefix, prefix()} 65 | 66 | @impl true 67 | @spec down([down_option()]) :: :ok 68 | def down(opts) do 69 | prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 70 | 71 | V4.create_set_transaction_id_procedure(prefix) 72 | 73 | :ok 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/carbonite/migrations/version.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Migrations.Version do 4 | @moduledoc false 5 | 6 | @callback up(keyword()) :: :ok 7 | @callback down(keyword()) :: :ok 8 | 9 | defmacro __using__(_) do 10 | quote do 11 | import Carbonite.Migrations.Helper 12 | import Carbonite.Prefix 13 | 14 | @behaviour Carbonite.Migrations.Version 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/carbonite/multi.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Multi do 4 | @moduledoc """ 5 | This module provides functions for dealing with audit trails in the context of Ecto.Multi. 6 | """ 7 | 8 | @moduledoc since: "0.2.0" 9 | 10 | alias Carbonite.Trigger 11 | alias Ecto.Multi 12 | 13 | @type prefix :: binary() 14 | @type params :: map() 15 | 16 | @type prefix_option :: {:carbonite_prefix, prefix()} 17 | 18 | @doc """ 19 | Adds an insert operation for a `Carbonite.Transaction` to an `Ecto.Multi`. 20 | 21 | Multi step is called `:carbonite_transaction`. 22 | 23 | See `Carbonite.insert_transaction/3` for options. 24 | 25 | ## Example 26 | 27 | Ecto.Multi.new() 28 | |> Carbonite.Multi.insert_transaction(%{meta: %{type: "create_rabbit"}}) 29 | |> Ecto.Multi.insert(:rabbit, fn _ -> Rabbit.changeset(%{}) end) 30 | """ 31 | @doc since: "0.2.0" 32 | @spec insert_transaction(Multi.t()) :: Multi.t() 33 | @spec insert_transaction(Multi.t(), params()) :: Multi.t() 34 | @spec insert_transaction(Multi.t(), params(), [prefix_option()]) :: Multi.t() 35 | def insert_transaction(%Multi{} = multi, params \\ %{}, opts \\ []) do 36 | Multi.run(multi, :carbonite_transaction, fn repo, _state -> 37 | Carbonite.insert_transaction(repo, params, opts) 38 | end) 39 | end 40 | 41 | @doc """ 42 | Adds a operation to an `Ecto.Multi` to fetch the changes of the current transaction. 43 | 44 | Useful for returning all transaction changes to the caller. 45 | 46 | Multi step is called `:carbonite_changes`. 47 | 48 | See `Carbonite.fetch_changes/2` for options. 49 | 50 | ## Example 51 | 52 | Ecto.Multi.new() 53 | |> Carbonite.Multi.insert_transaction(%{meta: %{type: "create_rabbit"}}) 54 | |> Ecto.Multi.insert(:rabbit, fn _ -> Rabbit.changeset(%{}) end) 55 | |> Carbonite.Multi.fetch_changes() 56 | """ 57 | @doc since: "0.5.0" 58 | @spec fetch_changes(Multi.t()) :: Multi.t() 59 | @spec fetch_changes(Multi.t(), [prefix_option()]) :: Multi.t() 60 | def fetch_changes(%Multi{} = multi, opts \\ []) do 61 | Multi.run(multi, :carbonite_changes, fn repo, _state -> 62 | Carbonite.fetch_changes(repo, opts) 63 | end) 64 | end 65 | 66 | @doc """ 67 | Sets the current transaction to "override mode" for all tables in the audit log. 68 | 69 | See `Carbonite.override_mode/2` for options. 70 | """ 71 | @doc since: "0.2.0" 72 | @spec override_mode(Multi.t()) :: Multi.t() 73 | @spec override_mode(Multi.t(), [{:to, Trigger.mode()} | prefix_option()]) :: Multi.t() 74 | def override_mode(%Multi{} = multi, opts \\ []) do 75 | Multi.run(multi, :carbonite_triggers, fn repo, _state -> 76 | {:ok, Carbonite.override_mode(repo, opts)} 77 | end) 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/carbonite/outbox.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Outbox do 4 | @moduledoc """ 5 | A `Carbonite.Outbox` stores metadata for outboxes like the last processed transaction. 6 | 7 | The `last_transaction_id` field defaults to zero, indicating that nothing has been 8 | processed yet. 9 | """ 10 | 11 | @moduledoc since: "0.4.0" 12 | 13 | use Carbonite.Schema 14 | import Ecto.Changeset 15 | 16 | @primary_key false 17 | 18 | @type name() :: String.t() 19 | @type memo() :: map() 20 | 21 | @type t :: %__MODULE__{ 22 | name: name(), 23 | last_transaction_id: non_neg_integer(), 24 | memo: memo(), 25 | inserted_at: DateTime.t(), 26 | updated_at: DateTime.t() 27 | } 28 | 29 | schema "outboxes" do 30 | field(:name, :string, primary_key: true) 31 | field(:last_transaction_id, :integer) 32 | field(:memo, :map) 33 | 34 | timestamps() 35 | end 36 | 37 | @doc """ 38 | Builds an update changeset. 39 | """ 40 | @doc since: "0.4.0" 41 | @spec changeset(__MODULE__.t(), params :: map()) :: Ecto.Changeset.t() 42 | def changeset(%__MODULE__{} = outbox, params) do 43 | outbox 44 | |> cast(params, [:last_transaction_id, :memo]) 45 | |> validate_required([:memo]) 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/carbonite/prefix.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Prefix do 4 | @moduledoc false 5 | 6 | defmacro default_prefix, do: "carbonite_default" 7 | end 8 | -------------------------------------------------------------------------------- /lib/carbonite/query.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Query do 4 | @moduledoc """ 5 | This module provides query functions for retrieving audit trails from the database. 6 | """ 7 | 8 | @moduledoc since: "0.2.0" 9 | 10 | import Ecto.Query 11 | import Carbonite.Prefix 12 | alias Carbonite.{Change, Outbox, Transaction, Trigger} 13 | 14 | @type prefix :: binary() 15 | @type disabled :: nil | false 16 | 17 | @type prefix_option :: {:carbonite_prefix, prefix()} 18 | @type preload_option :: {:preload, boolean()} 19 | @type order_by_option :: {:order_by, false | nil | term()} 20 | 21 | @type transactions_option :: prefix_option() | preload_option() 22 | 23 | # Must be a macro as when queryable is passed as a variable, Ecto won't allow setting the 24 | # prefix. Only literal schema modules will allow to have their prefixes updated. 25 | defmacrop from_with_prefix(queryable, opts) do 26 | quote do 27 | carbonite_prefix = Keyword.get(unquote(opts), :carbonite_prefix, default_prefix()) 28 | 29 | from(unquote(queryable), prefix: ^to_string(carbonite_prefix)) 30 | end 31 | end 32 | 33 | @doc """ 34 | Returns an `t:Ecto.Query.t/0` that can be used to select transactions from the database. 35 | 36 | ## Examples 37 | 38 | Carbonite.Query.transactions() 39 | |> MyApp.Repo.all() 40 | 41 | # Preload changes 42 | Carbonite.Query.transactions(preload: true) 43 | |> MyApp.Repo.all() 44 | 45 | ## Options 46 | 47 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 48 | * `preload` - can be used to preload the changes, defaults to `false` 49 | """ 50 | 51 | @doc since: "0.3.1" 52 | @spec transactions() :: Ecto.Query.t() 53 | @spec transactions([transactions_option()]) :: Ecto.Query.t() 54 | def transactions(opts \\ []) do 55 | from_with_prefix(Transaction, opts) 56 | |> maybe_preload(opts, :changes, from_with_prefix(Change, opts)) 57 | end 58 | 59 | @type current_transaction_option :: prefix_option() | preload_option() 60 | 61 | @doc """ 62 | Returns an `t:Ecto.Query.t/0` that can be used to select or delete the "current" transaction. 63 | 64 | This function is useful when your tests run in a database transaction using Ecto's SQL sandbox. 65 | 66 | ## Example: Asserting on the current transaction 67 | 68 | When you insert your `Carbonite.Transaction` record somewhere inside your domain logic, you do 69 | not wish to return it to the caller only to be able to assert on its attributes in tests. This 70 | example shows how you could assert on the metadata inserted. 71 | 72 | # Test running inside Ecto's SQL sandbox. 73 | test "my test" do 74 | some_operation_with_a_transaction() 75 | 76 | assert current_transaction_meta() == %{"type" => "some_operation"} 77 | end 78 | 79 | defp current_transaction_meta do 80 | Carbonite.Query.current_transaction() 81 | |> MyApp.Repo.one!() 82 | |> Map.fetch(:meta) 83 | end 84 | 85 | ## Options 86 | 87 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 88 | * `preload` - can be used to preload the changes, defaults to `false` 89 | """ 90 | @doc since: "0.2.0" 91 | @spec current_transaction() :: Ecto.Query.t() 92 | @spec current_transaction([current_transaction_option()]) :: Ecto.Query.t() 93 | def current_transaction(opts \\ []) do 94 | carbonite_prefix = Keyword.get(opts, :carbonite_prefix, default_prefix()) 95 | 96 | from_with_prefix(Transaction, opts) 97 | |> where( 98 | [t], 99 | t.id == fragment("CURRVAL(CONCAT(?::VARCHAR, '.transactions_id_seq'))", ^carbonite_prefix) 100 | ) 101 | |> where([t], t.xact_id == fragment("pg_current_xact_id()")) 102 | |> maybe_preload(opts, :changes, from_with_prefix(Change, opts)) 103 | end 104 | 105 | # Returns all triggers. 106 | @doc false 107 | @spec triggers() :: Ecto.Query.t() 108 | def triggers(opts \\ []) do 109 | from_with_prefix(Trigger, opts) 110 | end 111 | 112 | @doc """ 113 | Returns an `t:Ecto.Query.t/0` that selects a outbox by name. 114 | 115 | ## Options 116 | 117 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 118 | """ 119 | @doc since: "0.4.0" 120 | @spec outbox(Outbox.name()) :: Ecto.Query.t() 121 | @spec outbox(Outbox.name(), [prefix_option()]) :: Ecto.Query.t() 122 | def outbox(outbox_name, opts \\ []) do 123 | from_with_prefix(Outbox, opts) 124 | |> where([o], o.name == ^outbox_name) 125 | end 126 | 127 | @type outbox_queue_option :: 128 | prefix_option() 129 | | preload_option() 130 | | {:min_age, non_neg_integer() | disabled()} 131 | | {:limit, non_neg_integer() | disabled()} 132 | 133 | @doc """ 134 | Returns an `t:Ecto.Query.t/0` that selects the next batch of transactions for an outbox. 135 | 136 | * Transactions are ordered by their ID ascending, so *roughly* in order of insertion. 137 | 138 | ## Options 139 | 140 | * `min_age` - the minimum age of a record, defaults to 300 seconds (set nil to disable) 141 | * `limit` - limits the query in size, defaults to 100 (set nil to disable) 142 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 143 | * `preload` - can be used to preload the changes, defaults to `true` 144 | """ 145 | @doc since: "0.4.0" 146 | @spec outbox_queue(Outbox.t()) :: Ecto.Query.t() 147 | @spec outbox_queue(Outbox.t(), [outbox_queue_option()]) :: Ecto.Query.t() 148 | def outbox_queue(%Outbox{last_transaction_id: last_processed_tx_id}, opts \\ []) do 149 | opts = Keyword.put_new(opts, :preload, true) 150 | 151 | from_with_prefix(Transaction, opts) 152 | |> where([t], t.id > ^last_processed_tx_id) 153 | |> maybe_apply(opts, :limit, 100, fn q, bs -> limit(q, ^bs) end) 154 | |> maybe_apply(opts, :min_age, 300, &where_inserted_at_lt/2) 155 | |> maybe_preload(opts, :changes, from_with_prefix(Change, opts)) 156 | |> order_by({:asc, :id}) 157 | end 158 | 159 | @type outbox_done_option :: prefix_option() | {:min_age, non_neg_integer() | disabled()} 160 | 161 | @doc """ 162 | Returns an `t:Ecto.Query.t/0` that selects all completely processed transactions. 163 | 164 | * If no outbox exists, this query returns all transactions. 165 | * If one or more outboxes exist, this query returns all transactions with an ID less than the 166 | minimum of the `last_transaction_id` attributes of the outboxes. 167 | * Transactions are not ordered. 168 | 169 | ## Options 170 | 171 | * `min_age` - the minimum age of a record, defaults to 300 seconds (set nil to disable) 172 | * `carbonite_prefix` - defines the audit trail's schema, defaults to `"carbonite_default"` 173 | """ 174 | @doc since: "0.4.0" 175 | @spec outbox_done() :: Ecto.Query.t() 176 | @spec outbox_done([outbox_done_option()]) :: Ecto.Query.t() 177 | def outbox_done(opts \\ []) do 178 | # NOTE: The query below has a non-optimal query plan, but expressing it differently makes 179 | # it a bit convoluted (e.g., fetching the min `last_transaction_id` or `MAX_INT` if that 180 | # does not exist and then filtering by <= that number), so we keep the `ALL()` for now. 181 | 182 | outbox_query = 183 | Outbox 184 | |> from_with_prefix(opts) 185 | |> select([o], o.last_transaction_id) 186 | 187 | Transaction 188 | |> from_with_prefix(opts) 189 | |> where([t], t.id <= all(outbox_query)) 190 | |> maybe_apply(opts, :min_age, 300, &where_inserted_at_lt/2) 191 | end 192 | 193 | @default_table_prefix "public" 194 | 195 | @type changes_option :: 196 | prefix_option() | preload_option() | order_by_option() | {:table_prefix, prefix()} 197 | 198 | @doc """ 199 | Returns an `t:Ecto.Query.t/0` that can be used to select changes for a single record. 200 | 201 | Given an `t:Ecto.Schema.t/0` struct, this function builds a query that fetches all changes 202 | recorded for it from the database, ordered ascending by their ID (i.e., roughly by 203 | insertion date descending). 204 | 205 | ## Example 206 | 207 | %MyApp.Rabbit{id: 1} 208 | |> Carbonite.Query.changes() 209 | |> MyApp.Repo.all() 210 | 211 | ## Options 212 | 213 | * `carbonite_prefix` defines the audit trail's schema, defaults to `"carbonite_default"` 214 | * `table_prefix` allows to override the table prefix, defaults to schema prefix of the record 215 | * `preload` can be used to preload the transaction 216 | * `order_by` allows to override the ordering, defaults to `{:asc, :id}` 217 | """ 218 | @doc since: "0.2.0" 219 | @spec changes(record :: Ecto.Schema.t()) :: Ecto.Query.t() 220 | @spec changes(record :: Ecto.Schema.t(), [changes_option()]) :: Ecto.Query.t() 221 | def changes(%schema{__meta__: %Ecto.Schema.Metadata{}} = record, opts \\ []) do 222 | table_prefix = 223 | Keyword.get_lazy(opts, :table_prefix, fn -> 224 | schema.__schema__(:prefix) || @default_table_prefix 225 | end) 226 | 227 | table_name = schema.__schema__(:source) 228 | 229 | table_pk = 230 | for pk_col <- Enum.sort(schema.__schema__(:primary_key)) do 231 | record |> Map.fetch!(pk_col) |> to_string() 232 | end 233 | 234 | from_with_prefix(Change, opts) 235 | |> where([c], c.table_prefix == ^table_prefix) 236 | |> where([c], c.table_name == ^table_name) 237 | |> where([c], c.table_pk == ^table_pk) 238 | |> maybe_preload(opts, :transaction, from_with_prefix(Transaction, opts)) 239 | |> maybe_order_by(opts) 240 | end 241 | 242 | defp maybe_apply(queryable, opts, key, default, fun) do 243 | if value = Keyword.get(opts, key, default) do 244 | fun.(queryable, value) 245 | else 246 | queryable 247 | end 248 | end 249 | 250 | defp maybe_order_by(queryable, opts) do 251 | case Keyword.get(opts, :order_by, {:asc, :id}) do 252 | order_by when order_by in [false, nil] -> 253 | queryable 254 | 255 | value -> 256 | order_by(queryable, ^value) 257 | end 258 | end 259 | 260 | defp maybe_preload(queryable, opts, association, preload_query) do 261 | case Keyword.get(opts, :preload, false) do 262 | preload when preload in [false, nil] -> 263 | queryable 264 | 265 | true -> 266 | preload(queryable, [{^association, ^preload_query}]) 267 | end 268 | end 269 | 270 | defp where_inserted_at_lt(queryable, min_age) do 271 | max_inserted_at = DateTime.add(DateTime.utc_now(), -1 * min_age, :second) 272 | 273 | where(queryable, [t], t.inserted_at <= ^max_inserted_at) 274 | end 275 | end 276 | -------------------------------------------------------------------------------- /lib/carbonite/schema.ex: -------------------------------------------------------------------------------- 1 | defmodule Carbonite.Schema do 2 | @moduledoc false 3 | 4 | defmacro __using__(_opts) do 5 | quote do 6 | use Ecto.Schema 7 | require Carbonite.Prefix 8 | 9 | @schema_prefix Carbonite.Prefix.default_prefix() 10 | 11 | @timestamps_opts [type: :utc_datetime_usec] 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/carbonite/transaction.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Transaction do 4 | @moduledoc """ 5 | A `Carbonite.Transaction` is the binding link between change records of tables. 6 | 7 | As such, it contains a set of optional metadata that describes the transaction. 8 | """ 9 | 10 | @moduledoc since: "0.1.0" 11 | 12 | use Carbonite.Schema 13 | import Ecto.Changeset 14 | 15 | if Code.ensure_loaded?(Jason.Encoder) do 16 | @derive {Jason.Encoder, only: [:id, :meta, :inserted_at, :changes]} 17 | end 18 | 19 | @primary_key false 20 | 21 | @type meta :: map() 22 | 23 | @type id :: non_neg_integer() 24 | 25 | @type t :: %__MODULE__{ 26 | id: id(), 27 | xact_id: non_neg_integer(), 28 | meta: meta(), 29 | inserted_at: DateTime.t(), 30 | changes: Ecto.Association.NotLoaded.t() | [Carbonite.Change.t()] 31 | } 32 | 33 | schema "transactions" do 34 | field(:id, :integer, primary_key: true) 35 | field(:xact_id, :integer) 36 | field(:meta, :map, default: %{}) 37 | 38 | timestamps(updated_at: false) 39 | 40 | has_many(:changes, Carbonite.Change, references: :id) 41 | end 42 | 43 | @meta_pdict_key :carbonite_meta 44 | 45 | @doc """ 46 | Stores a piece of metadata in the process dictionary. 47 | 48 | This can be useful in situations where you want to record a value at a system boundary (say, 49 | the user's `account_id`) without having to pass it through to the database transaction. 50 | 51 | Returns the currently stored metadata. 52 | """ 53 | @doc since: "0.2.0" 54 | @spec put_meta(key :: any(), value :: any()) :: meta() 55 | def put_meta(key, value) do 56 | meta = Map.put(current_meta(), key, value) 57 | Process.put(@meta_pdict_key, meta) 58 | meta 59 | end 60 | 61 | @doc """ 62 | Returns the currently stored metadata. 63 | """ 64 | @doc since: "0.2.0" 65 | @spec current_meta() :: meta() 66 | def current_meta do 67 | Process.get(@meta_pdict_key) || %{} 68 | end 69 | 70 | @doc """ 71 | Builds a changeset for a new `Carbonite.Transaction`. 72 | 73 | The `:meta` map from the params will be merged with the metadata currently stored in the 74 | process dictionary. 75 | """ 76 | @doc since: "0.2.0" 77 | @spec changeset() :: Ecto.Changeset.t() 78 | @spec changeset(params :: map()) :: Ecto.Changeset.t() 79 | def changeset(params \\ %{}) do 80 | %__MODULE__{} 81 | |> cast(params, [:meta]) 82 | |> merge_current_meta() 83 | end 84 | 85 | defp merge_current_meta(changeset) do 86 | meta = Map.merge(current_meta(), get_field(changeset, :meta)) 87 | 88 | put_change(changeset, :meta, meta) 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/carbonite/trigger.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Trigger do 4 | @moduledoc """ 5 | A `Carbonite.Trigger` stores per table configuration for the change capture trigger. 6 | """ 7 | 8 | @moduledoc since: "0.1.0" 9 | 10 | use Carbonite.Schema 11 | 12 | @primary_key {:id, :id, autogenerate: true} 13 | 14 | @type id :: non_neg_integer() 15 | @type mode :: :capture | :ignore 16 | 17 | @type t :: %__MODULE__{ 18 | id: id(), 19 | table_name: String.t(), 20 | table_prefix: String.t(), 21 | primary_key_columns: [String.t()], 22 | excluded_columns: [String.t()], 23 | filtered_columns: [String.t()], 24 | store_changed_from: boolean(), 25 | mode: mode(), 26 | inserted_at: DateTime.t(), 27 | updated_at: DateTime.t() 28 | } 29 | 30 | schema "triggers" do 31 | field(:table_prefix, :string) 32 | field(:table_name, :string) 33 | field(:primary_key_columns, {:array, :string}) 34 | field(:excluded_columns, {:array, :string}) 35 | field(:filtered_columns, {:array, :string}) 36 | field(:store_changed_from, :boolean) 37 | field(:mode, Ecto.Enum, values: [:capture, :ignore]) 38 | 39 | timestamps() 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/mix/tasks/carbonite.gen.initial_migration.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Carbonite.Gen.InitialMigration do 2 | use Mix.Task 3 | 4 | import Macro, only: [camelize: 1, underscore: 1] 5 | import Mix.Generator 6 | import Mix.Ecto, except: [migrations_path: 1] 7 | import Ecto.Migrator, only: [migrations_path: 1] 8 | 9 | @shortdoc "Generates the initial migration to install Carbonite data structures" 10 | 11 | @moduledoc """ 12 | Generates a sample migration that runs all migrations currently contained within Carbonite. 13 | """ 14 | 15 | @doc false 16 | @dialyzer {:no_return, run: 1} 17 | 18 | def run(args) do 19 | no_umbrella!("carbonite.gen.initial_migration") 20 | repos = parse_repo(args) 21 | name = "install_carbonite" 22 | 23 | Enum.each(repos, fn repo -> 24 | ensure_repo(repo, args) 25 | path = Path.relative_to(migrations_path(repo), Mix.Project.app_path()) 26 | file = Path.join(path, "#{timestamp()}_#{underscore(name)}.exs") 27 | create_directory(path) 28 | 29 | assigns = [mod: Module.concat([repo, Migrations, camelize(name)])] 30 | 31 | create_file(file, migration_template(assigns)) 32 | end) 33 | end 34 | 35 | defp timestamp do 36 | {{y, m, d}, {hh, mm, ss}} = :calendar.universal_time() 37 | "#{y}#{pad(m)}#{pad(d)}#{pad(hh)}#{pad(mm)}#{pad(ss)}" 38 | end 39 | 40 | defp pad(i), do: i |> to_string() |> String.pad_leading(2, "0") 41 | 42 | embed_template(:migration, """ 43 | defmodule <%= inspect @mod %> do 44 | use Ecto.Migration 45 | 46 | def up do 47 | # If you like to install Carbonite's tables into a different schema, add the 48 | # carbonite_prefix option. 49 | # 50 | # Carbonite.Migrations.up(1, carbonite_prefix: "carbonite_other") 51 | 52 | Carbonite.Migrations.up(<%= Carbonite.Migrations.initial_patch() %>..<%= Carbonite.Migrations.current_patch() %>) 53 | 54 | # Install a trigger for a table: 55 | # 56 | # Carbonite.Migrations.create_trigger("rabbits") 57 | # Carbonite.Migrations.create_trigger("rabbits", table_prefix: "animals") 58 | # Carbonite.Migrations.create_trigger("rabbits", carbonite_prefix: "carbonite_other") 59 | 60 | # Configure trigger options: 61 | # 62 | # Carbonite.Migrations.put_trigger_config("rabbits", :primary_key_columns, ["compound", "key"]) 63 | # Carbonite.Migrations.put_trigger_config("rabbits", :excluded_columns, ["private"]) 64 | # Carbonite.Migrations.put_trigger_config("rabbits", :filtered_columns, ["private"]) 65 | # Carbonite.Migrations.put_trigger_config("rabbits", :mode, :ignore) 66 | 67 | # If you wish to insert an initial outbox: 68 | # 69 | # Carbonite.Migrations.create_outbox("rabbits") 70 | # Carbonite.Migrations.create_outbox("rabbits", carbonite_prefix: "carbonite_other") 71 | 72 | end 73 | 74 | def down do 75 | # Remove trigger from a table: 76 | # 77 | # Carbonite.Migrations.drop_trigger("rabbits") 78 | # Carbonite.Migrations.drop_trigger("rabbits", table_prefix: "animals") 79 | # Carbonite.Migrations.drop_trigger("rabbits", carbonite_prefix: "carbonite_other") 80 | 81 | Carbonite.Migrations.down(<%= Carbonite.Migrations.current_patch() %>..<%= Carbonite.Migrations.initial_patch() %>) 82 | end 83 | end 84 | """) 85 | end 86 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.MixProject do 4 | use Mix.Project 5 | 6 | @version "0.15.0" 7 | 8 | def project do 9 | [ 10 | app: :carbonite, 11 | version: @version, 12 | elixir: "~> 1.11", 13 | start_permanent: Mix.env() == :prod, 14 | elixirc_paths: elixirc_paths(Mix.env()), 15 | deps: deps(), 16 | aliases: aliases(), 17 | preferred_cli_env: [lint: :test], 18 | dialyzer: [ 19 | plt_add_apps: [:mix, :ex_unit, :jason], 20 | plt_core_path: "_plts", 21 | plt_file: {:no_warn, "_plts/carbonite.plt"} 22 | ], 23 | 24 | # hex.pm 25 | package: package(), 26 | description: "Audit trails for Elixir/PostgreSQL based on triggers", 27 | 28 | # hexdocs.pm 29 | name: "Carbonite", 30 | source_url: "https://github.com/bitcrowd/carbonite", 31 | homepage_url: "https://github.com/bitcrowd/carbonite", 32 | docs: [ 33 | main: "Carbonite", 34 | logo: ".logo_for_docs.png", 35 | extras: ["CHANGELOG.md": [title: "Changelog"], LICENSE: [title: "License"]], 36 | source_ref: "v#{@version}", 37 | source_url: "https://github.com/bitcrowd/carbonite", 38 | formatters: ["html"], 39 | skip_undefined_reference_warnings_on: ["CHANGELOG.md"] 40 | ] 41 | ] 42 | end 43 | 44 | defp package do 45 | [ 46 | maintainers: ["@bitcrowd"], 47 | licenses: ["Apache-2.0"], 48 | links: %{github: "https://github.com/bitcrowd/carbonite"} 49 | ] 50 | end 51 | 52 | def application do 53 | [ 54 | extra_applications: [:logger] 55 | ] 56 | end 57 | 58 | defp elixirc_paths(env) when env in [:dev, :test], do: ["lib", "test/support"] 59 | defp elixirc_paths(_env), do: ["lib"] 60 | 61 | defp deps do 62 | [ 63 | {:ecto_sql, "~> 3.10"}, 64 | {:jason, "~> 1.2"}, 65 | {:postgrex, "~> 0.15 and >= 0.15.11"}, 66 | {:credo, "~> 1.7", only: [:dev, :test], runtime: false}, 67 | {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}, 68 | {:ex_doc, "> 0.0.0", only: [:dev, :test], runtime: false} 69 | ] 70 | end 71 | 72 | defp aliases do 73 | [ 74 | lint: [ 75 | "format --check-formatted", 76 | "credo --strict", 77 | "dialyzer --format dialyxir" 78 | ], 79 | "ecto.reset": [ 80 | "ecto.drop", 81 | "ecto.create", 82 | "ecto.migrate" 83 | ] 84 | ] 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunt": {:hex, :bunt, "0.2.1", "e2d4792f7bc0ced7583ab54922808919518d0e57ee162901a16a1b6664ef3b14", [:mix], [], "hexpm", "a330bfb4245239787b15005e66ae6845c9cd524a288f0d141c148b02603777a5"}, 3 | "credo": {:hex, :credo, "1.7.0", "6119bee47272e85995598ee04f2ebbed3e947678dee048d10b5feca139435f75", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "6839fcf63d1f0d1c0f450abc8564a57c43d644077ab96f2934563e68b8a769d7"}, 4 | "db_connection": {:hex, :db_connection, "2.5.0", "bb6d4f30d35ded97b29fe80d8bd6f928a1912ca1ff110831edcd238a1973652c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c92d5ba26cd69ead1ff7582dbb860adeedfff39774105a4f1c92cbb654b55aa2"}, 5 | "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, 6 | "dialyxir": {:hex, :dialyxir, "1.4.1", "a22ed1e7bd3a3e3f197b68d806ef66acb61ee8f57b3ac85fc5d57354c5482a93", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "84b795d6d7796297cca5a3118444b80c7d94f7ce247d49886e7c291e1ae49801"}, 7 | "earmark_parser": {:hex, :earmark_parser, "1.4.35", "437773ca9384edf69830e26e9e7b2e0d22d2596c4a6b17094a3b29f01ea65bb8", [:mix], [], "hexpm", "8652ba3cb85608d0d7aa2d21b45c6fad4ddc9a1f9a1f1b30ca3a246f0acc33f6"}, 8 | "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"}, 9 | "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"}, 10 | "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, 11 | "ex_doc": {:hex, :ex_doc, "0.30.6", "5f8b54854b240a2b55c9734c4b1d0dd7bdd41f71a095d42a70445c03cf05a281", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bd48f2ddacf4e482c727f9293d9498e0881597eae6ddc3d9562bd7923375109f"}, 12 | "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, 13 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 14 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 15 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 16 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, 17 | "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, 18 | "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, 19 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 20 | } 21 | -------------------------------------------------------------------------------- /test/capture_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule CaptureTest do 4 | use ExUnit.Case, async: true 5 | import Ecto.Adapters.SQL, only: [query!: 2] 6 | alias Carbonite.TestRepo 7 | alias Ecto.Adapters.SQL.Sandbox 8 | 9 | setup do 10 | :ok = Sandbox.checkout(TestRepo) 11 | end 12 | 13 | defp query!(statement), do: query!(TestRepo, statement) 14 | 15 | defp insert_transaction do 16 | query!("INSERT INTO carbonite_default.transactions (inserted_at) VALUES (NOW());") 17 | end 18 | 19 | defp insert_jack(table \\ "rabbits") do 20 | query!("INSERT INTO #{table} (name, age) VALUES ('Jack', 99);") 21 | end 22 | 23 | defp upsert_jack(new_name, do_clause) do 24 | query!(""" 25 | INSERT INTO rabbits (id, name) VALUES (#{last_rabbit_id()}, '#{new_name}') 26 | ON CONFLICT (id) DO #{do_clause} 27 | """) 28 | end 29 | 30 | defp select_changes do 31 | "SELECT * FROM carbonite_default.changes ORDER BY id ASC;" 32 | |> query!() 33 | |> postgrex_result_to_structs() 34 | end 35 | 36 | defp select_rabbits(table \\ "rabbits") do 37 | "SELECT * FROM #{table} ORDER BY id DESC;" 38 | |> query!() 39 | |> postgrex_result_to_structs() 40 | end 41 | 42 | defp last_rabbit_id do 43 | select_rabbits() 44 | |> List.last() 45 | |> Map.fetch!("id") 46 | |> to_string() 47 | end 48 | 49 | defp postgrex_result_to_structs(%Postgrex.Result{columns: columns, rows: rows}) do 50 | Enum.map(rows, fn row -> 51 | columns 52 | |> Enum.zip(row) 53 | |> Map.new() 54 | end) 55 | end 56 | 57 | defp transaction_with_simulated_commit(fun) do 58 | TestRepo.transaction(fn -> 59 | fun.() 60 | 61 | # We simulate the end of the transaction by setting the constraint to immediate now. 62 | # Otherwise when Ecto's SQL sandbox rolls back the transaction, our trigger never fires. 63 | query!("SET CONSTRAINTS ALL IMMEDIATE;") 64 | end) 65 | end 66 | 67 | describe "change capture trigger" do 68 | test "INSERTs on tables are tracked as changes" do 69 | TestRepo.transaction(fn -> 70 | insert_transaction() 71 | insert_jack() 72 | end) 73 | 74 | assert [ 75 | %{ 76 | "id" => _, 77 | "transaction_id" => _, 78 | "table_prefix" => "public", 79 | "table_name" => "rabbits", 80 | "op" => "insert", 81 | "changed" => [], 82 | "changed_from" => nil, 83 | "data" => %{"id" => _, "name" => "Jack"} 84 | } 85 | ] = select_changes() 86 | end 87 | 88 | test "UPDATEs on tables are tracked as changes" do 89 | TestRepo.transaction(fn -> 90 | insert_transaction() 91 | insert_jack() 92 | query!("UPDATE rabbits SET name = 'Jane' WHERE name = 'Jack';") 93 | end) 94 | 95 | assert [ 96 | %{ 97 | "op" => "insert", 98 | "changed" => [], 99 | "changed_from" => nil, 100 | "data" => %{"id" => _, "name" => "Jack"} 101 | }, 102 | %{ 103 | "op" => "update", 104 | "changed" => ["name"], 105 | "changed_from" => %{"name" => "Jack"}, 106 | "data" => %{"id" => _, "name" => "Jane"} 107 | } 108 | ] = select_changes() 109 | end 110 | 111 | test "UPDATEs on tables are not tracked when data remains the same" do 112 | TestRepo.transaction(fn -> 113 | insert_transaction() 114 | insert_jack() 115 | query!("UPDATE rabbits SET name = 'Jack';") 116 | end) 117 | 118 | assert [%{"op" => "insert"}] = select_changes() 119 | end 120 | 121 | test "changes to arrays are detected correctly" do 122 | TestRepo.transaction(fn -> 123 | insert_transaction() 124 | insert_jack() 125 | 126 | # Additive array changes weren't detected correctly previously. 127 | query!("UPDATE rabbits SET carrots = '{carrot1}';") 128 | end) 129 | 130 | assert [ 131 | _insert, 132 | %{"op" => "update", "changed_from" => %{"carrots" => []}, "changed" => ["carrots"]} 133 | ] = select_changes() 134 | end 135 | 136 | test "changed_from tracking is optional" do 137 | TestRepo.transaction(fn -> 138 | insert_transaction() 139 | insert_jack() 140 | query!("UPDATE rabbits SET name = 'Jane' WHERE name = 'Jack';") 141 | query!("UPDATE carbonite_default.triggers SET store_changed_from = FALSE;") 142 | query!("UPDATE rabbits SET name = 'Jiff' WHERE name = 'Jane';") 143 | end) 144 | 145 | assert [ 146 | # INSERT has no changed_from 147 | %{"changed_from" => nil}, 148 | # store_changed_from is true for rabbits (in migration) 149 | %{"changed_from" => %{"name" => "Jack"}}, 150 | # demonstrating that it is optional 151 | %{"changed_from" => nil} 152 | ] = select_changes() 153 | end 154 | 155 | test "INSERT ON CONFLICT NOTHING is not tracked" do 156 | TestRepo.transaction(fn -> 157 | insert_transaction() 158 | insert_jack() 159 | upsert_jack("Jack", "NOTHING") 160 | end) 161 | 162 | assert [%{"op" => "insert"}] = select_changes() 163 | end 164 | 165 | test "INSERT ON CONFLICT SET ... is not tracked when data remains the same" do 166 | TestRepo.transaction(fn -> 167 | insert_transaction() 168 | insert_jack() 169 | upsert_jack("Jack", "UPDATE SET name = excluded.name;") 170 | end) 171 | 172 | assert [%{"op" => "insert"}] = select_changes() 173 | end 174 | 175 | test "INSERT ON CONFLICT SET ... is tracked when data is changed" do 176 | TestRepo.transaction(fn -> 177 | insert_transaction() 178 | insert_jack() 179 | upsert_jack("Jane", "UPDATE SET name = excluded.name;") 180 | end) 181 | 182 | assert [ 183 | %{"op" => "insert"}, 184 | %{ 185 | "op" => "update", 186 | "changed" => ["name"], 187 | "data" => %{"id" => _, "name" => "Jane"} 188 | } 189 | ] = select_changes() 190 | end 191 | 192 | test "DELETEs on tables are tracked as changes" do 193 | TestRepo.transaction(fn -> 194 | insert_transaction() 195 | insert_jack() 196 | query!("DELETE FROM rabbits WHERE name = 'Jack';") 197 | end) 198 | 199 | assert [ 200 | %{ 201 | "op" => "insert", 202 | "changed" => [], 203 | "changed_from" => nil, 204 | "data" => %{"id" => _, "name" => "Jack"} 205 | }, 206 | %{ 207 | "op" => "delete", 208 | "changed" => [], 209 | "changed_from" => nil, 210 | "data" => %{"id" => _, "name" => "Jack"} 211 | } 212 | ] = select_changes() 213 | end 214 | 215 | test "table primary key is written for INSERTs" do 216 | TestRepo.transaction(fn -> 217 | insert_transaction() 218 | insert_jack() 219 | end) 220 | 221 | rabbit_id = last_rabbit_id() 222 | 223 | assert [%{"table_pk" => [^rabbit_id]}] = select_changes() 224 | end 225 | 226 | test "table primary key is written for UPDATEs" do 227 | TestRepo.transaction(fn -> 228 | insert_transaction() 229 | insert_jack() 230 | query!("UPDATE rabbits SET name = 'Jane' WHERE name = 'Jack';") 231 | end) 232 | 233 | rabbit_id = last_rabbit_id() 234 | 235 | assert [ 236 | %{"table_pk" => [^rabbit_id]}, 237 | %{"table_pk" => [^rabbit_id]} 238 | ] = select_changes() 239 | end 240 | 241 | test "table primary key is written for DELETEs" do 242 | {:ok, rabbit_id} = 243 | TestRepo.transaction(fn -> 244 | insert_transaction() 245 | insert_jack() 246 | rabbit_id = last_rabbit_id() 247 | query!("DELETE FROM rabbits WHERE name = 'Jack';") 248 | rabbit_id 249 | end) 250 | 251 | assert [ 252 | %{"table_pk" => [^rabbit_id]}, 253 | %{"table_pk" => [^rabbit_id]} 254 | ] = select_changes() 255 | end 256 | 257 | test "table_pk is NULL when primary_key_columns is empty" do 258 | query!("UPDATE carbonite_default.triggers SET primary_key_columns = '{}';") 259 | 260 | TestRepo.transaction(fn -> 261 | insert_transaction() 262 | insert_jack() 263 | end) 264 | 265 | assert [%{"table_pk" => nil}] = select_changes() 266 | end 267 | 268 | test "a friendly error is raised when transaction is not inserted or is inserted too late" do 269 | msg = 270 | "ERROR 23503 (foreign_key_violation) (carbonite) INSERT on table public.rabbits " <> 271 | "without prior INSERT into carbonite_default.transactions" 272 | 273 | # Case 1: Session has never called `NEXTVAL` -> `CURRVAL` fails. 274 | 275 | assert_raise Postgrex.Error, msg, fn -> 276 | TestRepo.transaction(&insert_jack/0) 277 | end 278 | 279 | # Case 2: Previous transaction (in session) has used `NEXTVAL` -> FK violation. 280 | 281 | query!("SELECT NEXTVAL('carbonite_default.transactions_id_seq');") 282 | 283 | assert_raise Postgrex.Error, msg, fn -> 284 | TestRepo.transaction(&insert_jack/0) 285 | end 286 | end 287 | 288 | test "a (not quite as) friendly error is raised when transaction is inserted twice" do 289 | TestRepo.transaction(fn -> 290 | insert_transaction() 291 | 292 | assert_raise Postgrex.Error, 293 | ~r/duplicate key value violates unique constraint "transactions_pkey"/, 294 | fn -> 295 | insert_transaction() 296 | end 297 | end) 298 | end 299 | 300 | test "initially deferred trigger allows late transaction insertion" do 301 | transaction_with_simulated_commit(fn -> 302 | # deferred_rabbits have a trigger with INITIALLY DEFERRED constraint, so we can insert 303 | # a record before inserting the transaction. 304 | insert_jack("deferred_rabbits") 305 | insert_transaction() 306 | end) 307 | 308 | assert [%{"table_name" => "deferred_rabbits"}] = select_changes() 309 | end 310 | 311 | test "initially deferred trigger still requires a transaction to be inserted" do 312 | msg = 313 | "ERROR 23503 (foreign_key_violation) (carbonite) INSERT on table public.deferred_rabbits " <> 314 | "without prior INSERT into carbonite_default.transactions" 315 | 316 | assert_raise Postgrex.Error, msg, fn -> 317 | transaction_with_simulated_commit(fn -> 318 | insert_jack("deferred_rabbits") 319 | end) 320 | end 321 | 322 | assert select_rabbits("deferred_rabbits") == [] 323 | end 324 | end 325 | 326 | describe "default mode / override mode" do 327 | test "override mode can set the mode explicitly" do 328 | TestRepo.transaction(fn -> 329 | query!("SET LOCAL carbonite_default.override_mode = 'ignore';") 330 | 331 | insert_jack() 332 | end) 333 | 334 | assert select_changes() == [] 335 | end 336 | 337 | test "override mode can reverse the configured mode" do 338 | TestRepo.transaction(fn -> 339 | query!("SET LOCAL carbonite_default.override_mode = 'override';") 340 | 341 | insert_jack() 342 | end) 343 | 344 | assert select_changes() == [] 345 | end 346 | 347 | test "default mode can be set to ignore" do 348 | # This test exists because we had a bug with the ignore mode 349 | # when the override_xact_id was NULL 350 | TestRepo.transaction(fn -> 351 | query!(""" 352 | UPDATE carbonite_default.triggers SET mode = 'ignore'; 353 | """) 354 | 355 | insert_jack() 356 | end) 357 | 358 | assert select_changes() == [] 359 | end 360 | end 361 | 362 | describe "excluded columns" do 363 | test "excluded columns do not appear in captured data" do 364 | TestRepo.transaction(fn -> 365 | insert_transaction() 366 | insert_jack() 367 | end) 368 | 369 | assert [%{"data" => data}] = select_changes() 370 | refute Map.has_key?(data, "age") 371 | end 372 | 373 | test "UPDATEs on only excluded fields are not tracked" do 374 | TestRepo.transaction(fn -> 375 | insert_transaction() 376 | insert_jack() 377 | query!("UPDATE rabbits SET age = 100 WHERE name = 'Jack';") 378 | end) 379 | 380 | assert [%{"age" => 100}] = select_rabbits() 381 | assert [%{"op" => "insert"}] = select_changes() 382 | end 383 | end 384 | 385 | describe "filtered columns" do 386 | test "appear as [FILTERED] in the data" do 387 | TestRepo.transaction(fn -> 388 | query!(""" 389 | UPDATE carbonite_default.triggers SET filtered_columns = '{name}'; 390 | """) 391 | 392 | insert_transaction() 393 | insert_jack() 394 | end) 395 | 396 | assert [%{"data" => %{"name" => "[FILTERED]"}}] = select_changes() 397 | end 398 | 399 | test "appear as [FILTERED] in the changed_from" do 400 | TestRepo.transaction(fn -> 401 | query!(""" 402 | UPDATE carbonite_default.triggers SET filtered_columns = '{name}'; 403 | """) 404 | 405 | insert_transaction() 406 | insert_jack() 407 | query!("UPDATE rabbits SET name = 'Jane' WHERE name = 'Jack';") 408 | end) 409 | 410 | assert [ 411 | _insert, 412 | %{"changed_from" => %{"name" => "[FILTERED]"}} 413 | ] = select_changes() 414 | end 415 | 416 | test "unknown columns in filtered_columns do not affect the result" do 417 | TestRepo.transaction(fn -> 418 | query!(""" 419 | UPDATE carbonite_default.triggers SET filtered_columns = '{doesnotexist}'; 420 | """) 421 | 422 | insert_transaction() 423 | insert_jack() 424 | end) 425 | 426 | assert [%{"data" => data}] = select_changes() 427 | refute Map.has_key?(data, "doesnotexist") 428 | end 429 | end 430 | 431 | describe "quoted identifiers" do 432 | test "invalid or reserved table prefixes and names are quoted correctly" do 433 | TestRepo.transaction(fn -> 434 | insert_transaction() 435 | query!(~s|INSERT INTO "default"."rabbits;" (name, age) VALUES ('Jåck', 99);|) 436 | end) 437 | 438 | assert [%{"op" => "insert"}] = select_changes() 439 | end 440 | end 441 | 442 | test "missing trigger row lets the trigger function error gracefully" do 443 | msg = 444 | "ERROR P0002 (no_data_found) (carbonite) INSERT on table public.rabbits " <> 445 | "but no trigger record in carbonite_default.triggers" 446 | 447 | TestRepo.transaction(fn -> 448 | insert_transaction() 449 | query!("DELETE FROM carbonite_default.triggers;") 450 | 451 | assert_raise Postgrex.Error, msg, fn -> 452 | insert_jack() 453 | end 454 | end) 455 | end 456 | end 457 | -------------------------------------------------------------------------------- /test/carbonite/change_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.ChangeTest do 4 | use Carbonite.APICase, async: true 5 | alias Carbonite.Change 6 | 7 | describe "Jason.Encoder implementation" do 8 | test "Carbonite.Change can be encoded to JSON" do 9 | json = 10 | %Change{ 11 | id: 1, 12 | op: :update, 13 | table_prefix: "default", 14 | table_name: "rabbits", 15 | table_pk: ["1"], 16 | data: %{"name" => "Jack"}, 17 | changed: ["name"], 18 | changed_from: %{"name" => "Jane"} 19 | } 20 | |> Jason.encode!() 21 | |> Jason.decode!() 22 | 23 | assert json == 24 | %{ 25 | "changed" => ["name"], 26 | "changed_from" => %{"name" => "Jane"}, 27 | "data" => %{"name" => "Jack"}, 28 | "id" => 1, 29 | "op" => "update", 30 | "table_name" => "rabbits", 31 | "table_pk" => ["1"], 32 | "table_prefix" => "default" 33 | } 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/carbonite/migrations_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.MigrationsTest do 4 | use ExUnit.Case, async: false 5 | import ExUnit.CaptureLog 6 | import Ecto.Query, only: [order_by: 2] 7 | 8 | defmodule UnboxedTestRepo do 9 | use Ecto.Repo, 10 | otp_app: :carbonite, 11 | adapter: Ecto.Adapters.Postgres 12 | 13 | @impl Ecto.Repo 14 | def init(_context, config) do 15 | test_repo_config = 16 | :carbonite 17 | |> Application.fetch_env!(Carbonite.TestRepo) 18 | |> Keyword.delete(:pool) 19 | 20 | {:ok, Keyword.merge(config, test_repo_config)} 21 | end 22 | end 23 | 24 | defmodule InsertMigrato do 25 | use Ecto.Migration 26 | import Carbonite.Migrations 27 | 28 | insert_migration_transaction_after_begin() 29 | 30 | def up do 31 | execute("INSERT INTO rabbits (name, age) VALUES ('Migrato', 180)") 32 | end 33 | 34 | def down do 35 | execute("DELETE FROM rabbits WHERE name = 'Migrato'") 36 | end 37 | end 38 | 39 | setup do 40 | start_supervised!(UnboxedTestRepo) 41 | 42 | :ok 43 | end 44 | 45 | defp up_and_down_migration(migration, up_cb, down_cb) do 46 | capture_log(fn -> 47 | try do 48 | Ecto.Migrator.run(UnboxedTestRepo, [{0, migration}], :up, all: true) 49 | 50 | up_cb.() 51 | 52 | Ecto.Migrator.run(UnboxedTestRepo, [{0, migration}], :down, all: true) 53 | 54 | down_cb.() 55 | after 56 | # Cleanup. Here because `start_supervised!` stops the Repo too early for an `on_exit`. 57 | delete_all_transactions() 58 | end 59 | end) 60 | end 61 | 62 | defp select_all_transactions do 63 | Carbonite.Query.transactions(preload: true) 64 | |> order_by({:asc, :inserted_at}) 65 | |> UnboxedTestRepo.all() 66 | end 67 | 68 | defp delete_all_transactions do 69 | UnboxedTestRepo.delete_all(Carbonite.Query.transactions()) 70 | end 71 | 72 | describe "insert_migration_transaction_after_begin/1" do 73 | test "makes the migration automatically insert a transaction" do 74 | up_and_down_migration( 75 | InsertMigrato, 76 | fn -> 77 | assert [ 78 | %{ 79 | changes: [%{op: :insert, table_name: "rabbits"}], 80 | meta: %{ 81 | "direction" => "up", 82 | "name" => "carbonite/migrations_test/insert_migrato", 83 | "type" => "migration" 84 | } 85 | } 86 | ] = select_all_transactions() 87 | end, 88 | fn -> 89 | assert [_, %{meta: %{"direction" => "down"}}] = select_all_transactions() 90 | end 91 | ) 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /test/carbonite/multi_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.MultiTest do 4 | use Carbonite.APICase, async: true 5 | import Carbonite.Multi 6 | alias Carbonite.{Rabbit, TestRepo} 7 | 8 | describe "insert_transaction/3" do 9 | test "inserts a transaction within an Ecto.Multi" do 10 | assert {:ok, _} = 11 | Ecto.Multi.new() 12 | |> insert_transaction() 13 | |> Ecto.Multi.put(:params, %{name: "Jack", age: 99}) 14 | |> Ecto.Multi.insert(:rabbit, &Rabbit.create_changeset(&1.params)) 15 | |> TestRepo.transaction() 16 | end 17 | end 18 | 19 | describe "override_mode/2" do 20 | test "enables override mode for the current transaction" do 21 | assert {:ok, _} = 22 | Ecto.Multi.new() 23 | |> override_mode() 24 | |> Ecto.Multi.put(:params, %{name: "Jack", age: 99}) 25 | |> Ecto.Multi.insert(:rabbit, &Rabbit.create_changeset(&1.params)) 26 | |> TestRepo.transaction() 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/carbonite/outbox_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.OutboxTest do 4 | use Carbonite.APICase, async: true 5 | import Carbonite.Outbox 6 | alias Carbonite.Outbox 7 | 8 | describe "changeset/2" do 9 | test "casts attributes" do 10 | cs = changeset(%Outbox{}, %{last_transaction_id: 500_000, memo: %{"foo" => "bar"}}) 11 | assert cs.valid? 12 | assert cs.changes.last_transaction_id == 500_000 13 | assert cs.changes.memo == %{"foo" => "bar"} 14 | end 15 | 16 | test "requires presence of memo" do 17 | cs = changeset(%Outbox{}, %{last_transaction_id: 500_000, memo: nil}) 18 | refute cs.valid? 19 | assert [{:memo, {_msg, [validation: :required]}}] = cs.errors 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/carbonite/query_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.QueryTest do 4 | use Carbonite.APICase, async: true 5 | alias Carbonite.{Change, Outbox, Query, Rabbit, TestRepo, Transaction} 6 | 7 | defp insert_rabbits(_) do 8 | {:ok, results} = 9 | Ecto.Multi.new() 10 | |> Carbonite.Multi.insert_transaction(%{meta: %{type: "rabbits_inserted"}}) 11 | |> Ecto.Multi.put(:params, %{name: "Jack", age: 99}) 12 | |> Ecto.Multi.insert(:rabbit, &Rabbit.create_changeset(&1.params)) 13 | |> Ecto.Multi.put(:params2, %{name: "Lily", age: 172}) 14 | |> Ecto.Multi.insert(:rabbit2, &Rabbit.create_changeset(&1.params2)) 15 | |> TestRepo.transaction() 16 | 17 | Map.take(results, [:rabbit, :rabbit2]) 18 | end 19 | 20 | describe "transactions/2" do 21 | setup [:insert_past_transactions, :insert_rabbits] 22 | 23 | test "fetches all transactions" do 24 | assert length(TestRepo.all(Query.transactions())) == 4 25 | end 26 | 27 | test "can preload changes alongside the transaction" do 28 | assert [%Transaction{changes: changes} | _] = 29 | TestRepo.all(Query.transactions(preload: true)) 30 | 31 | assert is_list(changes) 32 | end 33 | 34 | test "carbonite_prefix option works as expected" do 35 | assert TestRepo.all(Query.transactions(carbonite_prefix: "alternate_test_schema")) == [] 36 | end 37 | 38 | test "accepts an atom as prefix" do 39 | assert TestRepo.all(Query.transactions(carbonite_prefix: :alternate_test_schema)) == [] 40 | end 41 | end 42 | 43 | describe "current_transaction/2" do 44 | setup [:insert_past_transactions, :insert_rabbits] 45 | 46 | test "can fetch the current transaction when inside SQL sandbox", %{ 47 | transactions: transactions 48 | } do 49 | assert %Transaction{id: id} = TestRepo.one!(Query.current_transaction()) 50 | 51 | assert id not in ids(transactions) 52 | end 53 | 54 | test "can preload changes alongside the transaction" do 55 | assert %Transaction{changes: [%Change{}, %Change{}]} = 56 | TestRepo.one!(Query.current_transaction(preload: true)) 57 | end 58 | 59 | test "can be used to erase the current transaction" do 60 | assert TestRepo.count(Transaction) == 4 61 | assert TestRepo.count(Change) == 2 62 | 63 | TestRepo.delete_all(Query.current_transaction()) 64 | 65 | assert TestRepo.count(Transaction) == 3 66 | assert TestRepo.count(Change) == 0 67 | end 68 | 69 | test "can be used to update the current transaction" do 70 | transaction = Query.current_transaction() |> TestRepo.one() 71 | 72 | transaction 73 | |> Ecto.Changeset.cast(%{meta: Map.put(transaction.meta, "foo", "bar")}, [:meta]) 74 | |> TestRepo.update!() 75 | 76 | assert %{"type" => "rabbits_inserted", "foo" => "bar"} = TestRepo.reload(transaction).meta 77 | end 78 | end 79 | 80 | describe "outbox/1" do 81 | defp outbox(name) do 82 | name 83 | |> Query.outbox() 84 | |> TestRepo.one() 85 | end 86 | 87 | test "gets an outbox by name" do 88 | assert %Outbox{} = outbox("rabbits") 89 | refute outbox("doesnotexist") 90 | end 91 | end 92 | 93 | describe "outbox_queue/2" do 94 | setup [:insert_past_transactions] 95 | 96 | defp outbox_queue(opts \\ []) do 97 | outbox = get_rabbits_outbox() 98 | 99 | outbox 100 | |> Query.outbox_queue(opts) 101 | |> TestRepo.all() 102 | end 103 | 104 | test "filters by id > last_transaction_id" do 105 | update_rabbits_outbox(%{last_transaction_id: 200_000}) 106 | 107 | assert ids(outbox_queue()) == [300_000] 108 | end 109 | 110 | test "orders results by id" do 111 | assert ids(outbox_queue()) == [100_000, 200_000, 300_000] 112 | end 113 | 114 | test "can limit the limit" do 115 | assert ids(outbox_queue(limit: 1)) == [100_000] 116 | end 117 | 118 | test "can filter by min_age" do 119 | assert ids(outbox_queue(min_age: 9_000)) == [100_000] 120 | end 121 | 122 | test "preloads changes by default" do 123 | assert [%Transaction{changes: []} | _] = outbox_queue() 124 | end 125 | 126 | test "can not preload changes" do 127 | assert [%Transaction{changes: %Ecto.Association.NotLoaded{}} | _] = 128 | outbox_queue(preload: false) 129 | end 130 | end 131 | 132 | describe "outbox_done/1" do 133 | setup [:insert_past_transactions, :insert_transaction_in_alternate_schema] 134 | 135 | defp outbox_done(opts \\ []) do 136 | opts 137 | |> Query.outbox_done() 138 | |> TestRepo.all() 139 | end 140 | 141 | test "filters by id >= MIN(outboxes.last_transaction_id)" do 142 | assert ids(outbox_done()) == [] 143 | 144 | update_rabbits_outbox(%{last_transaction_id: 200_000}) 145 | 146 | assert ids(outbox_done()) == [100_000, 200_000] 147 | 148 | TestRepo.insert!( 149 | %Outbox{name: "second", last_transaction_id: 100_000}, 150 | prefix: :carbonite_default 151 | ) 152 | 153 | assert ids(outbox_done()) == [100_000] 154 | 155 | TestRepo.delete_all(Outbox, prefix: :carbonite_default) 156 | 157 | assert ids(outbox_done()) == [100_000, 200_000, 300_000] 158 | end 159 | 160 | test "can filter by min_age" do 161 | update_rabbits_outbox(%{last_transaction_id: 200_000}) 162 | 163 | assert ids(outbox_done(min_age: 9_000)) == [100_000] 164 | end 165 | 166 | test "carbonite_prefix option works as expected" do 167 | assert ids(outbox_done(carbonite_prefix: "alternate_test_schema")) == [] 168 | 169 | update_alternate_outbox(%{last_transaction_id: 1_000}) 170 | 171 | assert ids(outbox_done(carbonite_prefix: "alternate_test_schema")) == [666] 172 | end 173 | end 174 | 175 | describe "changes/2" do 176 | setup [:insert_rabbits] 177 | 178 | defp changes(rabbit, opts \\ []) do 179 | rabbit 180 | |> Query.changes(opts) 181 | |> TestRepo.all() 182 | end 183 | 184 | test "queries changes for a given record given its struct", %{rabbit: rabbit} do 185 | assert [%Change{data: %{"name" => "Jack"}}] = changes(rabbit) 186 | end 187 | 188 | test "can disable ordering", %{rabbit: rabbit} do 189 | assert Map.get(Query.changes(rabbit, order_by: false), :order_bys) == [] 190 | end 191 | 192 | test "can set custom ordering", %{rabbit: rabbit} do 193 | rabbit 194 | |> Rabbit.rename_changeset("Gerda") 195 | |> TestRepo.update!() 196 | 197 | [_, _] = descending_ids = changes(rabbit, order_by: {:desc, :id}) 198 | 199 | assert Enum.reverse(descending_ids) == changes(rabbit) 200 | end 201 | 202 | test "can preload the transaction", %{rabbit: rabbit} do 203 | assert [%Change{transaction: %Transaction{}}] = changes(rabbit, preload: true) 204 | end 205 | 206 | test "can override the schema prefix", %{rabbit: rabbit} do 207 | assert changes(rabbit, table_prefix: "foo") == [] 208 | assert [%Change{}] = changes(rabbit, table_prefix: "public") 209 | end 210 | end 211 | end 212 | -------------------------------------------------------------------------------- /test/carbonite/transaction_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TransactionTest do 4 | use Carbonite.APICase, async: true 5 | import Carbonite.Transaction 6 | alias Carbonite.{Change, Transaction} 7 | 8 | describe "changeset/1" do 9 | test "transaction_changesets an Ecto.Changeset for a transaction" do 10 | %Ecto.Changeset{} = changeset = changeset() 11 | 12 | assert get_field(changeset, :meta) == %{} 13 | end 14 | 15 | test "allows setting metadata" do 16 | %Ecto.Changeset{} = changeset = changeset(%{meta: %{foo: 1}}) 17 | 18 | assert get_field(changeset, :meta) == %{foo: 1} 19 | end 20 | 21 | test "merges metadata from process dictionary" do 22 | Transaction.put_meta(:foo, 1) 23 | Transaction.put_meta(:bar, 1) 24 | %Ecto.Changeset{} = changeset = changeset(%{meta: %{foo: 2}}) 25 | 26 | assert get_field(changeset, :meta) == %{foo: 2, bar: 1} 27 | end 28 | end 29 | 30 | describe "Jason.Encoder implementation" do 31 | test "Transaction can be encoded to JSON" do 32 | json = 33 | %Transaction{ 34 | id: 1, 35 | meta: %{"foo" => 1}, 36 | inserted_at: ~U[2021-11-01T12:00:00Z], 37 | changes: [ 38 | %Change{ 39 | id: 1, 40 | op: :update, 41 | table_prefix: "default", 42 | table_name: "rabbits", 43 | table_pk: ["1"], 44 | data: %{"name" => "Jack"}, 45 | changed: ["name"], 46 | changed_from: %{"name" => "Jane"} 47 | } 48 | ] 49 | } 50 | |> Jason.encode!() 51 | |> Jason.decode!() 52 | 53 | assert json == 54 | %{ 55 | "changes" => [ 56 | %{ 57 | "changed" => ["name"], 58 | "changed_from" => %{"name" => "Jane"}, 59 | "data" => %{"name" => "Jack"}, 60 | "id" => 1, 61 | "op" => "update", 62 | "table_name" => "rabbits", 63 | "table_pk" => ["1"], 64 | "table_prefix" => "default" 65 | } 66 | ], 67 | "id" => 1, 68 | "inserted_at" => "2021-11-01T12:00:00Z", 69 | "meta" => %{"foo" => 1} 70 | } 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /test/carbonite_test.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule CarboniteTest do 4 | use Carbonite.APICase, async: true 5 | import Carbonite 6 | alias Carbonite.{Outbox, Rabbit, TestRepo, Transaction} 7 | alias Ecto.Adapters.SQL 8 | 9 | defp insert_jack do 10 | %{name: "Jack", age: 99} 11 | |> Rabbit.create_changeset() 12 | |> TestRepo.insert!() 13 | end 14 | 15 | describe "insert_transaction/3" do 16 | test "inserts a transaction" do 17 | assert {:ok, tx} = insert_transaction(TestRepo) 18 | 19 | assert tx.meta == %{} 20 | assert_almost_now(tx.inserted_at) 21 | 22 | assert is_integer(tx.id) 23 | assert %Ecto.Association.NotLoaded{} = tx.changes 24 | end 25 | 26 | test "allows setting metadata" do 27 | assert {:ok, tx} = insert_transaction(TestRepo, %{meta: %{foo: 1}}) 28 | 29 | # atoms deserialize to strings 30 | assert tx.meta == %{"foo" => 1} 31 | end 32 | 33 | test "merges metadata from process dictionary" do 34 | Carbonite.Transaction.put_meta(:foo, 1) 35 | 36 | assert {:ok, tx} = insert_transaction(TestRepo) 37 | 38 | assert tx.meta == %{"foo" => 1} 39 | end 40 | 41 | test "subsequent inserts in a single transaction are ignored" do 42 | # Since we're running in the SQL sandbox, both of these transactions 43 | # actually reference the same transaction id. 44 | 45 | assert {:ok, tx1} = insert_transaction(TestRepo, %{meta: %{foo: 1}}) 46 | assert {:ok, tx2} = insert_transaction(TestRepo, %{meta: %{foo: 2}}) 47 | 48 | assert tx1.meta == %{"foo" => 1} 49 | assert tx2.meta == %{"foo" => 1} 50 | end 51 | 52 | test "carbonite_prefix option works as expected" do 53 | assert {:ok, _tx} = 54 | insert_transaction(TestRepo, %{}, carbonite_prefix: "alternate_test_schema") 55 | 56 | assert %{num_rows: 1} = 57 | SQL.query!(TestRepo, "SELECT * FROM alternate_test_schema.transactions") 58 | end 59 | end 60 | 61 | describe "fetch_changes/2" do 62 | test "inserts a transaction" do 63 | TestRepo.transaction(fn -> 64 | insert_transaction(TestRepo) 65 | insert_jack() 66 | 67 | assert {:ok, [%Carbonite.Change{op: :insert}]} = fetch_changes(TestRepo) 68 | end) 69 | end 70 | 71 | test "carbonite_prefix option works as expected" do 72 | TestRepo.transaction(fn -> 73 | # Disable on primary test schema, enable on alternate test schema. 74 | override_mode(TestRepo) 75 | override_mode(TestRepo, carbonite_prefix: "alternate_test_schema") 76 | 77 | insert_transaction(TestRepo, %{}, carbonite_prefix: "alternate_test_schema") 78 | insert_jack() 79 | 80 | assert {:ok, [%Carbonite.Change{op: :insert}]} = 81 | fetch_changes(TestRepo, carbonite_prefix: "alternate_test_schema") 82 | end) 83 | end 84 | end 85 | 86 | describe "override_mode/2" do 87 | test "enables override mode for the current transaction" do 88 | assert override_mode(TestRepo) == :ok 89 | 90 | insert_jack() 91 | 92 | assert get_transactions() == [] 93 | end 94 | 95 | test "carbonite_prefix option works as expected" do 96 | insert_transaction(TestRepo) 97 | insert_jack() 98 | 99 | # Mode is :ignore in the alternate_test_schema, so override mode enables the trigger. 100 | assert override_mode(TestRepo, carbonite_prefix: "alternate_test_schema") == :ok 101 | 102 | assert_raise Postgrex.Error, ~r/without prior INSERT into alternate_test_schema/, fn -> 103 | insert_jack() 104 | end 105 | end 106 | 107 | test "can toggle override mode with the :to option" do 108 | assert override_mode(TestRepo, to: :ignore) == :ok 109 | 110 | insert_jack() 111 | 112 | assert override_mode(TestRepo, to: :capture) == :ok 113 | 114 | assert_raise Postgrex.Error, fn -> 115 | insert_jack() 116 | end 117 | end 118 | end 119 | 120 | describe "process/4" do 121 | setup [:insert_past_transactions, :insert_transaction_in_alternate_schema] 122 | 123 | test "starts at the last processed position (+1)" do 124 | update_rabbits_outbox(%{last_transaction_id: 200_000}) 125 | 126 | assert {:ok, _outbox} = 127 | process(TestRepo, "rabbits", fn [tx], _memo -> 128 | send(self(), tx.id) 129 | :cont 130 | end) 131 | 132 | refute_received 100_000 133 | refute_received 200_000 134 | assert_received 300_000 135 | end 136 | 137 | test "passes down batch query options" do 138 | assert {:ok, _outbox} = 139 | process(TestRepo, "rabbits", [min_age: 9_000], fn [tx], _memo -> 140 | send(self(), tx.id) 141 | :cont 142 | end) 143 | 144 | assert_received 100_000 145 | refute_received 200_000 146 | refute_received 300_000 147 | end 148 | 149 | test "remembers the last processed position" do 150 | assert {:ok, %Outbox{} = outbox} = 151 | process(TestRepo, "rabbits", fn _txs, _memo -> 152 | :cont 153 | end) 154 | 155 | assert outbox.last_transaction_id == 300_000 156 | end 157 | 158 | test "remembers the returned memo" do 159 | update_rabbits_outbox(%{memo: %{"foo" => 1}}) 160 | 161 | assert {:ok, %Outbox{} = outbox} = 162 | process(TestRepo, "rabbits", fn _txs, memo -> 163 | {:cont, memo: Map.put(memo, "foo", memo["foo"] + 1)} 164 | end) 165 | 166 | assert outbox.memo == %{"foo" => 4} 167 | end 168 | 169 | test "when halted discards the last chunk" do 170 | update_rabbits_outbox(%{last_transaction_id: 200_000, memo: %{"foo" => 1}}) 171 | 172 | assert {:halt, %Outbox{} = outbox} = 173 | process(TestRepo, "rabbits", fn _txs, _memo -> 174 | :halt 175 | end) 176 | 177 | assert outbox.last_transaction_id == 200_000 178 | assert outbox.memo == %{"foo" => 1} 179 | end 180 | 181 | test "when halted still allows to update the outbox" do 182 | assert {:halt, %Outbox{} = outbox} = 183 | process(TestRepo, "rabbits", fn _txs, _memo -> 184 | {:halt, last_transaction_id: 100_000, memo: %{"some" => "data"}} 185 | end) 186 | 187 | assert outbox.last_transaction_id == 100_000 188 | assert outbox.memo == %{"some" => "data"} 189 | end 190 | 191 | defp assert_chunks(opts, expected) do 192 | assert {:ok, %Outbox{} = outbox} = 193 | process(TestRepo, "rabbits", opts, fn txs, memo -> 194 | chunks = Map.get(memo, "chunks", []) 195 | {:cont, memo: Map.put(memo, "chunks", chunks ++ [ids(txs)])} 196 | end) 197 | 198 | assert outbox.memo == %{"chunks" => expected} 199 | end 200 | 201 | test "accepts a chunk option to send larger chunks to the process function" do 202 | assert_chunks([chunk: 2], [[100_000, 200_000], [300_000]]) 203 | end 204 | 205 | test "defaults to chunk size 1" do 206 | assert_chunks([], [[100_000], [200_000], [300_000]]) 207 | end 208 | 209 | test "continues querying until processable transactions are exhausted" do 210 | # Limit in place, but chunk is bigger than limit (so irrelevant). 211 | # Algorithm continues until query comes back empty. 212 | assert_chunks([chunk: 100, limit: 2], [[100_000, 200_000], [300_000]]) 213 | end 214 | 215 | test "accepts a filter function for refining the batch query" do 216 | filter = fn query -> 217 | # mimicking the "min_age" behaviour in a custom filter 218 | max_inserted_at = DateTime.utc_now() |> DateTime.add(-9_000) 219 | where(query, [t], t.inserted_at < ^max_inserted_at) 220 | end 221 | 222 | assert {:ok, _outbox} = 223 | process(TestRepo, "rabbits", [chunk: 100, filter: filter], fn txs, _memo -> 224 | assert ids(txs) == [100_000] 225 | :cont 226 | end) 227 | end 228 | 229 | test "carbonite_prefix option works as expected" do 230 | {:ok, outbox} = 231 | process(TestRepo, "alternate_outbox", [carbonite_prefix: "alternate_test_schema"], fn txs, 232 | _ -> 233 | assert ids(txs) == [666] 234 | :cont 235 | end) 236 | 237 | assert outbox.last_transaction_id == 666 238 | end 239 | end 240 | 241 | describe "purge/2" do 242 | setup [:insert_past_transactions, :insert_transaction_in_alternate_schema] 243 | 244 | setup do 245 | update_rabbits_outbox(%{last_transaction_id: 200_000}) 246 | 247 | :ok 248 | end 249 | 250 | test "deletes transactions that have been processed by all outboxes" do 251 | assert purge(TestRepo) == {:ok, 2} 252 | 253 | assert [%Transaction{id: 300_000}] = get_transactions() 254 | end 255 | 256 | test "passes down batch query options" do 257 | assert purge(TestRepo, min_age: 9_000) == {:ok, 1} 258 | 259 | assert [%Transaction{id: 200_000}, %Transaction{id: 300_000}] = get_transactions() 260 | end 261 | 262 | test "carbonite_prefix option works as expected" do 263 | assert [%Transaction{id: 666}] = get_transactions(carbonite_prefix: "alternate_test_schema") 264 | assert purge(TestRepo, carbonite_prefix: "alternate_test_schema") == {:ok, 0} 265 | 266 | update_alternate_outbox(%{last_transaction_id: 1_000}) 267 | 268 | assert purge(TestRepo, carbonite_prefix: "alternate_test_schema") == {:ok, 1} 269 | assert [] = get_transactions(carbonite_prefix: "alternate_test_schema") 270 | 271 | # Transactions/outboxes on other schema are not affected. 272 | assert purge(TestRepo) == {:ok, 2} 273 | end 274 | end 275 | 276 | defp assert_almost_now(datetime) do 277 | assert_in_delta DateTime.to_unix(datetime), DateTime.to_unix(DateTime.utc_now()), 1 278 | end 279 | end 280 | -------------------------------------------------------------------------------- /test/support/api_case.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.APICase do 4 | @moduledoc false 5 | 6 | use ExUnit.CaseTemplate 7 | import Ecto.Changeset, only: [change: 2] 8 | import Ecto.Query, only: [order_by: 2] 9 | alias Carbonite.{Query, TestRepo, Transaction} 10 | alias Ecto.Adapters.SQL.Sandbox 11 | 12 | using do 13 | quote do 14 | import Ecto 15 | import Ecto.Changeset 16 | import Ecto.Query 17 | import Carbonite.APICase 18 | end 19 | end 20 | 21 | setup tags do 22 | pid = Sandbox.start_owner!(Carbonite.TestRepo, shared: not tags[:async]) 23 | on_exit(fn -> Sandbox.stop_owner(pid) end) 24 | 25 | :ok 26 | end 27 | 28 | def insert_past_transactions(_) do 29 | transactions = 30 | [ 31 | %Transaction{id: 100_000, inserted_at: hours_ago(3)}, 32 | %Transaction{id: 200_000, inserted_at: hours_ago(2)}, 33 | %Transaction{id: 300_000, inserted_at: hours_ago(1)} 34 | ] 35 | |> Enum.map(fn tx -> 36 | TestRepo.insert!(tx, prefix: Carbonite.default_prefix()) 37 | end) 38 | 39 | %{transactions: transactions} 40 | end 41 | 42 | def insert_transaction_in_alternate_schema(_) do 43 | transaction_on_alternate_schema = 44 | TestRepo.insert!(%Transaction{id: 666, inserted_at: hours_ago(1)}, 45 | prefix: "alternate_test_schema" 46 | ) 47 | 48 | %{transaction_on_alternate_schema: transaction_on_alternate_schema} 49 | end 50 | 51 | defp hours_ago(n) do 52 | DateTime.utc_now() |> DateTime.add(n * -3600) 53 | end 54 | 55 | def get_rabbits_outbox do 56 | "rabbits" 57 | |> Query.outbox() 58 | |> TestRepo.one!() 59 | end 60 | 61 | def update_rabbits_outbox(attrs) do 62 | get_rabbits_outbox() 63 | |> change(attrs) 64 | |> TestRepo.update!() 65 | end 66 | 67 | def update_alternate_outbox(attrs) do 68 | "alternate_outbox" 69 | |> Query.outbox(carbonite_prefix: "alternate_test_schema") 70 | |> TestRepo.one!() 71 | |> change(attrs) 72 | |> TestRepo.update!() 73 | end 74 | 75 | def get_transactions(opts \\ []) do 76 | opts 77 | |> Query.transactions() 78 | |> order_by({:asc, :id}) 79 | |> TestRepo.all() 80 | end 81 | 82 | def ids(set), do: set |> Enum.map(& &1.id) |> Enum.sort() 83 | end 84 | -------------------------------------------------------------------------------- /test/support/rabbit.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.Rabbit do 4 | @moduledoc false 5 | 6 | use Ecto.Schema 7 | import Ecto.Changeset, only: [cast: 3, change: 2] 8 | 9 | schema "rabbits" do 10 | field(:name, :string) 11 | field(:age, :integer) 12 | field(:carrots, {:array, :string}) 13 | end 14 | 15 | def create_changeset(params) do 16 | cast(%__MODULE__{}, params, [:name, :age]) 17 | end 18 | 19 | def rename_changeset(%__MODULE__{} = rabbit, new_name) do 20 | change(rabbit, %{name: new_name}) 21 | end 22 | 23 | def age_changeset(%__MODULE__{} = rabbit) do 24 | change(rabbit, %{age: rabbit.age + 1}) 25 | end 26 | 27 | # Dirty little helper to make rabbits on the console. 28 | def make do 29 | Ecto.Multi.new() 30 | |> Carbonite.Multi.insert_transaction() 31 | |> Ecto.Multi.insert(:rabbit, fn _ -> create_changeset(%{age: 101, name: "Janet"}) end) 32 | |> Carbonite.TestRepo.transaction() 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/support/test_repo.ex: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TestRepo do 4 | @moduledoc false 5 | 6 | use Ecto.Repo, 7 | otp_app: :carbonite, 8 | adapter: Ecto.Adapters.Postgres 9 | 10 | import Ecto.Query, only: [from: 2] 11 | 12 | @spec count(module()) :: non_neg_integer() 13 | def count(schema) do 14 | schema 15 | |> from(select: count()) 16 | |> one!(prefix: Carbonite.default_prefix()) 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /test/support/test_repo/migrations/20210704201537_create_rabbits.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TestRepo.Migrations.CreateRabbits do 4 | use Ecto.Migration 5 | 6 | def change do 7 | create table(:rabbits) do 8 | add(:name, :string) 9 | add(:age, :integer) 10 | add(:carrots, {:array, :string}, default: "{}") 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /test/support/test_repo/migrations/20210704201627_install_carbonite.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TestRepo.Migrations.InstallCarbonite do 4 | use Ecto.Migration 5 | alias Carbonite.Migrations, as: M 6 | 7 | def up do 8 | M.up(M.patch_range()) 9 | M.create_trigger(:rabbits) 10 | M.put_trigger_config(:rabbits, :excluded_columns, ["age"]) 11 | M.put_trigger_config(:rabbits, :store_changed_from, true) 12 | M.create_outbox("rabbits") 13 | end 14 | 15 | def down do 16 | M.drop_trigger(:rabbits) 17 | M.down(Enum.reverse(M.patch_range())) 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/support/test_repo/migrations/20221011201645_install_carbonite_alternate_test_schema.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TestRepo.Migrations.InstallCarboniteAlternateTestSchema do 4 | use Ecto.Migration 5 | alias Carbonite.Migrations, as: M 6 | 7 | def up do 8 | M.up(M.patch_range(), carbonite_prefix: "alternate_test_schema") 9 | 10 | M.create_trigger(:rabbits, carbonite_prefix: "alternate_test_schema") 11 | 12 | M.put_trigger_config(:rabbits, :mode, :ignore, carbonite_prefix: "alternate_test_schema") 13 | 14 | M.create_outbox("alternate_outbox", 15 | carbonite_prefix: "alternate_test_schema" 16 | ) 17 | end 18 | 19 | def down do 20 | M.drop_trigger(:rabbits, carbonite_prefix: "alternate_test_schema") 21 | 22 | M.down(Enum.reverse(M.patch_range()), carbonite_prefix: "alternate_test_schema") 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /test/support/test_repo/migrations/20221102120000_create_deferred_rabbits.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TestRepo.Migrations.CreateDeferredRabbits do 4 | use Ecto.Migration 5 | 6 | def change do 7 | create table(:deferred_rabbits) do 8 | add(:name, :string) 9 | add(:age, :integer) 10 | add(:carrots, {:array, :string}, default: "{}") 11 | end 12 | 13 | Carbonite.Migrations.create_trigger(:deferred_rabbits, initially: :deferred) 14 | end 15 | 16 | def down do 17 | Carbonite.Migrations.drop_trigger(:deferred_rabbits) 18 | 19 | drop(table(:deferred_rabbits)) 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/support/test_repo/migrations/20250101140000_create_weird_character_rabbits.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | defmodule Carbonite.TestRepo.Migrations.CreateWeirdCharacterRabbits do 4 | use Ecto.Migration 5 | alias Carbonite.Migrations, as: M 6 | 7 | def change do 8 | execute(~s|CREATE SCHEMA "default";|, ~s|DROP SCHEMA "default";|) 9 | 10 | create table("rabbits;", prefix: "default") do 11 | add(:name, :string) 12 | add(:age, :integer) 13 | add(:carrots, {:array, :string}, default: "{}") 14 | end 15 | 16 | M.create_trigger("rabbits;", table_prefix: "default") 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | 3 | Carbonite.TestRepo.start_link() 4 | Ecto.Adapters.SQL.Sandbox.mode(Carbonite.TestRepo, :manual) 5 | 6 | ExUnit.start() 7 | --------------------------------------------------------------------------------