├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── NOTICE ├── README.cn.md ├── README.md ├── expected ├── actions.out ├── array.out ├── binData.out ├── binary.out ├── boolean.out ├── identity.out ├── json.out ├── numeric.out ├── pkey.out ├── specval.out ├── specval_1.out ├── string.out ├── timestamptz.out ├── transaction.out ├── xml.out └── xml_1.out ├── logical.conf ├── sql ├── actions.sql ├── array.sql ├── binData.sql ├── binary.sql ├── boolean.sql ├── identity.sql ├── json.sql ├── numeric.sql ├── pkey.sql ├── specval.sql ├── string.sql ├── timestamptz.sql ├── transaction.sql └── xml.sql └── wal2mongo.c /.gitignore: -------------------------------------------------------------------------------- 1 | # regressio test outputs 2 | /regression.diffs 3 | /regression.out 4 | /results/ 5 | 6 | # Prerequisites 7 | *.d 8 | 9 | # Object files 10 | *.o 11 | *.ko 12 | *.obj 13 | *.elf 14 | 15 | # Linker output 16 | *.ilk 17 | *.map 18 | *.exp 19 | 20 | # Precompiled Headers 21 | *.gch 22 | *.pch 23 | 24 | # Libraries 25 | *.lib 26 | *.a 27 | *.la 28 | *.lo 29 | 30 | # Shared objects (inc. Windows DLLs) 31 | *.dll 32 | *.so 33 | *.so.* 34 | *.dylib 35 | 36 | # Executables 37 | *.exe 38 | *.app 39 | *.i*86 40 | *.x86_64 41 | *.hex 42 | 43 | # Debug files 44 | *.dSYM/ 45 | *.su 46 | *.idb 47 | *.pdb 48 | 49 | # Kernel Module Compile Results 50 | *.mod* 51 | *.cmd 52 | .tmp_versions/ 53 | modules.order 54 | Module.symvers 55 | Mkfile.old 56 | dkms.conf 57 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # run the testsuite on travis-ci.org 2 | --- 3 | # run once for each of these 4 | env: 5 | global: 6 | - enable_coverage=yes 7 | - PGVERSION=12 8 | 9 | language: C 10 | dist: bionic 11 | sudo: required 12 | 13 | before_install: 14 | - sudo apt-get update -qq 15 | - pip install --user cpp-coveralls 16 | 17 | install: 18 | # remove all existing postgresql 19 | - sudo rm -rf /etc/postgresql /var/lib/postgresql 20 | - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - 21 | - echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee /etc/apt/sources.list.d/pgdg.list 22 | - sudo apt update 23 | - sudo pip install cpp-coveralls 24 | - sudo apt -y install postgresql-12 postgresql-client-12 postgresql-server-dev-12 25 | - sudo apt-get install -y postgresql-common 26 | - sudo apt-get install -y bison flex libicu-dev libssl-dev git 27 | - sudo -u postgres createuser --superuser $USER 28 | - sudo sed -i 's/#wal_level = replica/wal_level = logical/g' /etc/postgresql/12/main/postgresql.conf 29 | - sudo service postgresql@12-main restart 30 | - git clone https://github.com/idrawone/wal2mongo.git 31 | 32 | script: 33 | - cd wal2mongo 34 | - export PATH=/usr/lib/postgresql/12/bin:$PATH 35 | - USE_PGXS=1 make 36 | - sudo USE_PGXS=1 make install 37 | - USE_PGXS=1 make installcheck-force 38 | - if test -s regression.diffs; then cat regression.diffs; cat results/string.out; cat expected/string.out; fi 39 | 40 | after_success: 41 | - coveralls --gcov-options '\-lp' 42 | 43 | deploy: 44 | provider: releases 45 | api_key: 46 | secure: "Cikugg6yUXIyHoRYN+oLzSbRNyJ5N7JR1IYkJjgBS2ZogqxMHhZszdYCTI7FyfdzzUA3qJLiARuYpOjtk/OxzZ1b4c1+kk4miwjeNEfSZFhEUfhPq9sAD+gBq/akCQhFUWf5OQiA/DQBivL0sVIxmHc8oE9YpMMKPze2WQZvQO/g6fctzkP2eKVA2/Mq+JO4uU/VCm0PTNKNSEnO54j+riKf3qsElOzJ9GX5kWl3gOV4UQoAn+GzvH7SwzB5iex3FXEtSPuW5d80LmtOb5vJB3YJH9Yk49DMhER5W6BBgcYC/BNgfpHjSeR5fCDV4pfmfkSLD6mV2Rsy2ke8NQDFeVXbr8SZtMsEAfdTb7M8ima+CtjoawBe/SwNxEA5QuhpuQGeAvfSJzAMU+dGvvajtC1dU6GehWvBcdbxiU7VcWjItfCrV9BVh3zCdo+1keAkpO078eeb5C8qiqK4yiJ6GN1S7jIx9Y6I9VeNLjOqIRuxjt4dqmT7LBNsksA1SGddvt5jzQXjGzV34P26b31aba0Xf0eCPqpMEcv28O7vB2wBB05TC+yjgUWBUrWvVz3sDcTrhBFxlZwQc2li37brIx5bHB4lG2SG8KoUatujYUuNP2mifJaW5lXv9ibIefPac+SNjG4A+lW7x35/vvVcwcOoFK9fgZav+Q8JbzbrPH8=" 47 | file_glob: true 48 | file: "*.so" 49 | skip_cleanup: true 50 | on: 51 | repo: HighgoSoftware/wal2mongo 52 | branch: release 53 | tags: true 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # contrib/wal2mongo/Makefile 2 | 3 | MODULES = wal2mongo 4 | PGFILEDESC = "wal2mongo - a logical decoding output plugin for MongoDB" 5 | 6 | REGRESS = binary actions transaction boolean numeric binData timestamptz \ 7 | array json string xml identity pkey specval 8 | 9 | #ISOLATION = mxact delayed_startup ondisk_startup concurrent_ddl_dml \ 10 | oldest_xmin snapshot_transfer 11 | 12 | REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/wal2mongo/logical.conf 13 | #ISOLATION_OPTS = --temp-config $(top_srcdir)/contrib/wal2mongo/logical.conf 14 | 15 | # Disabled because these tests require "wal_level=logical", which 16 | # typical installcheck users do not have (e.g. buildfarm clients). 17 | NO_INSTALLCHECK = 0 18 | 19 | ifdef USE_PGXS 20 | PG_CONFIG = pg_config 21 | PGXS := $(shell $(PG_CONFIG) --pgxs) 22 | include $(PGXS) 23 | else 24 | subdir = contrib/wal2mongo 25 | top_builddir = ../.. 26 | include $(top_builddir)/src/Makefile.global 27 | include $(top_srcdir)/contrib/contrib-global.mk 28 | endif 29 | 30 | # But it can nonetheless be very helpful to run tests on preexisting 31 | # installation, allow to do so, but only if requested explicitly. 32 | installcheck-force: 33 | $(pg_regress_installcheck) $(REGRESS) 34 | #$(pg_isolation_regress_installcheck) $(ISOLATION) 35 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 HighGo Software, Inc. (Canada). All Rights Reserved. 2 | 3 | This project contains software that is Copyright (c) 2020 HighGo Software, Inc. 4 | 5 | This project is licensed to you under the Apache License, Version 2.0 (the "License"). 6 | 7 | You may not use this project except in compliance with the License. 8 | 9 | This project may include a number of subcomponents with separate copyright notices 10 | and license terms. Your use of these subcomponents is subject to the terms and 11 | conditions of the subcomponent's license, as noted in the LICENSE file. 12 | -------------------------------------------------------------------------------- /README.cn.md: -------------------------------------------------------------------------------- 1 | ![License](https://img.shields.io/github/license/HighgoSoftware/wal2mongo) 2 | ![Build](https://travis-ci.com/HighgoSoftware/wal2mongo.svg?branch=release) 3 | [![Release](https://img.shields.io/github/v/tag/HighgoSoftware/wal2mongo?label=Release)](https://github.com/HighgoSoftware/wal2mongo/releases) 4 | 5 | ### 介绍 6 | `wal2mongo` 是一个PostgreSQL逻辑解码输出插件，用来输出mongo能够接受JSON格式，使PostgreSQL到MongoDB的逻辑复制更加容易。 7 | 8 | ### 使用条件 9 | 要使用wal2mongo逻辑解码输出插件，需要安装以下PostgreSQL服务器之一， 10 | * [PostgreSQL 12.x](https://www.postgresql.org/download) 11 | * [HighGo PostgreSQL Server 1.x](https://www.highgo.ca/products/highgo-postgresql-server) 12 | 13 | ### 编译，测试和安装 14 | `wal2mongo` 支持两种编译方式：一种是针对希望在PostgreSQL源代码树结构下管理 wal2mongo 代码的开发人员。 另一个是针对希望将wal2mongo集成到已编译好的PostgreSQL的开发人员或DBA。 15 | 16 | #### 类似Linux的环境中 17 | * 在PostgreSQL源代码树下构建 18 | ``` 19 | cd /path/to/postgres/contrib/ 20 | git clone https://github.com/HighgoSoftware/wal2mongo.git 21 | cd wal2mongo 22 | make 23 | make install 24 | make check 25 | ``` 26 | 27 | * 针对PostgreSQL二进制文件编译 28 | ``` 29 | mkdir sandbox 30 | cd sandbox 31 | git clone https://github.com/HighgoSoftware/wal2mongo.git 32 | cd wal2mongo 33 | ``` 34 | 35 | 将 “PATH” 指向现有的PostgreSQL二进制目录。 使用`PGHOST`和`PGPORT`指定一个PostgreSQL服务器和端口来运行installcheck-force测试。 36 | ``` 37 | $ export PATH=/path/to/postgres/bin:$PATH 38 | USE_PGXS=1 make 39 | USE_PGXS=1 make install 40 | USE_PGXS=1 make installcheck-force 41 | ``` 42 | 43 | #### 在Windows 7、10和2019服务器上 44 | * 在PostgreSQL源代码树下编译 45 | 1. 按照 [此处](https://www.postgresql.org/docs/12/install-windows-full.html) 的说明使用Microsoft Windows SDK设置编译环境. [Visual Studio 2019社区版](https://visualstudio.microsoft.com/downloads/) 足以编译Postgres 12.x和wal2mongo逻辑解码输出插件。 成功安装VS 2019之后，请下载 [`ActivePerl 5.28`](https://www.activestate.com/products/perl/downloads/), [`ActiveTcl 8.6`](https://www.activestate.com/products/tcl/downloads/) 和 [`GnuWin32 0.6.3`](https://sourceforge.net/projects/getgnuwin32/files/getgnuwin32/0.6.30/GetGnuWin32-0.6.3.exe/download) 然后使用默认设置安装即可。 46 | 47 | 2. 在环境变量管理面板中检查系统变量的 “ActivePerl”，“ActiveTcl” 和 “GnuWin32” 的二进制路径，如果不存在，请添加它们。 48 | 49 | 3. 编译并安装 50 | 51 | ``` 52 | cd \path\to\postgres\contrib\ 53 | git clone https://github.com/HighgoSoftware/wal2mongo.git 54 | cd \path\to\postgres\src\tools\msvc\ 55 | build 56 | install \path\to\install\foler\ 57 | ``` 58 | 4. 运行回归测试（注意，vcregress尚不支持单个扩展的回归测试） 59 | ``` 60 | vcregress contribcheck 61 | ``` 62 | * 针对PostgreSQL二进制文件编译 63 | 64 | 可以按照以下步骤使用Visual Studio 2019编译`wal2mongo` 65 | 1. 使用空项目 （empty project）模板创建一个新项目，例如，将项目名称设置为 `wal2mongo` 66 | 2. 右键单击 `wal2mongo` 项目->添加->新项目...，选择 `C ++ File`，但将其命名为`wal2mongo.c`。 67 | 3. 将来自github的所有c源代码粘贴到本地“ wal2mongo.c” 文件中。 68 | 4. 将 “PGDLLEXPORT” 添加到函数 “_PG_init” 和 “_PG_output_plugin_init”中，如下所示， 69 | ``` 70 | extern PGDLLEXPORT void _PG_init(void); 71 | extern PGDLLEXPORT void _PG_output_plugin_init(OutputPluginCallbacks* cb); 72 | ``` 73 | 5. 右键点击`wal2mongo`项目->属性，然后更改以下设置， 74 | 常规->配置类型“动态库（.dll）” 75 | 76 | 设置C / C++ ->代码生成->启用C++ exceptions 的设置改为`否` 77 | 78 | 设置C / C ++->高级->编译为`编译为C代码` 79 | 80 | 将链接器->清单文件->生成清单设置为“否” 81 | 82 | 将`postgres.lib`添加到链接器->输入->其他依赖项 83 | 84 | 根据PostgreSQL二进制文件的安装位置，按以下顺序将以下路径添加到 C / C++ ->常规->其他include目录 85 | 86 | ``` 87 | C:\Users\Administrator\Downloads\pg12.2\include\server\port\win32_msvc 88 | C:\Users\Administrator\Downloads\pg12.2\include\server\port\win32 89 | C:\Users\Administrator\Downloads\pg12.2\include\server 90 | C:\Users\Administrator\Downloads\pg12.2\include 91 | ``` 92 | 同样，根据PostgreSQL二进制文件的安装位置，在Linker-> General-> Additional Library Directories 中添加如下所示的库路径 93 | ``` 94 | C:\Users\Administrator\Downloads\pg12.2\lib 95 | ``` 96 | 6. 右键点击`wal2mongo`，然后点击`build`。 如果一切正常，则应该显示以下消息。 97 | ``` 98 | 1>wal2mongo.vcxproj -> C:\Users\Administrator\source\repos\wal2mongo\Debug\wal2mongo.dll 99 | 1>Done building project "wal2mongo.vcxproj". 100 | ========== Rebuild All: 1 succeeded, 0 failed, 0 skipped ========== 101 | ``` 102 | 7. 手动将 “wal2mongo.dll” 复制到安装了PostgreaSQL的 “lib”中，然后按照示例部分进行测试。 103 | 104 | 105 | ### 设置和配置 106 | 编辑PostgreSQl配置文件 “postgresql.conf”，并确保将 “wal_level” 设置为 “logical”，并且将 “max_replication_slots” 设置为至少1（默认设置为10）。 107 | 重新启动postgres服务。 108 | 109 | ### 选件 110 | 有待确定... 111 | 112 | ### 样例 113 | 下面是两种使用wal2mongo将数据从PostgreSQL复制到MongoDB的简单方法：一种使用psql console；另一个使用pg_recvlogical工具。 114 | 115 | #### 使用psql console 116 | * 创建一个逻辑插槽 117 | 例如，使用输出插件 “wal2mongo” 创建一个名为 “w2m_slot”的逻辑插槽。 118 | ``` 119 | postgres=# SELECT * FROM pg_create_logical_replication_slot('w2m_slot', 'wal2mongo'); 120 | slot_name | lsn 121 | -----------+------------ 122 | w2m_slot | 1/3CB04148 123 | (1 row) 124 | ``` 125 | 126 | * 检查刚创建的插槽 127 | ``` 128 | postgres=# SELECT slot_name, plugin, slot_type, database, active, restart_lsn, confirmed_flush_lsn FROM pg_replication_slots; 129 | slot_name | plugin | slot_type | database | active | restart_lsn | confirmed_flush_lsn 130 | -----------+-----------+-----------+----------+--------+-------------+--------------------- 131 | w2m_slot | wal2mongo | logical | postgres | f | 1/3CB04110 | 1/3CB04148 132 | (1 row) 133 | ``` 134 | 135 | * 创建一个表并插入数据 136 | ``` 137 | postgres=# CREATE TABLE books ( 138 | id SERIAL PRIMARY KEY, 139 | title VARCHAR(100) NOT NULL, 140 | author VARCHAR(100) NULL 141 | ); 142 | 143 | postgres=# insert into books 144 | (id, title, author) 145 | values 146 | (123, 'HG-PGSQL1.1', 'Highgo'); 147 | ``` 148 | 149 | * 查看有无任何改动 150 | ``` 151 | postgres=# SELECT * FROM pg_logical_slot_peek_changes('w2m_slot', NULL, NULL); 152 | lsn | xid | data 153 | ------------+------+------------------------------------------------------------------------ 154 | 1/3CB247F8 | 1793 | db.books.insertOne( { id:123, title:"HG-PGSQL1.1", author:"Highgo" } ) 155 | (1 row) 156 | ``` 157 | 158 | * 取得改动 159 | ``` 160 | postgres=# SELECT * FROM pg_logical_slot_get_changes('w2m_slot', NULL, NULL); 161 | lsn | xid | data 162 | ------------+------+------------------------------------------------------------------------ 163 | 1/3CB247F8 | 1793 | db.books.insertOne( { id:123, title:"HG-PGSQL1.1", author:"Highgo" } ) 164 | (1 row) 165 | ``` 166 | 167 | * 使用 mongo 客户端中复制数据（选项1） 168 | 169 | 登录 mongoDB，然后复制数据部分中的所有字符串，然后粘贴到 mongo 客户端上 170 | ``` 171 | > db.books.insertOne( { id:123, title:"HG-PGSQL1.1", author:"Highgo" } ) 172 | { 173 | "acknowledged" : true, 174 | "insertedId" : ObjectId("5e5ea92be9684c562aae5b7a") 175 | } 176 | ``` 177 | 178 | * 使用.js文件复制数据（选项2） 179 | 180 | 复制输出的所有字符串，然后将其粘贴到文件中，例如 test.js，然后使用mongo客户端导入文件 181 | ``` 182 | $ mongo < test.js 183 | MongoDB shell version v4.0.16 184 | connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb 185 | Implicit session: session { "id" : UUID("86ddf177-9704-43f9-9f66-31ac1f9f89e0") } 186 | MongoDB server version: 4.0.16 187 | { 188 | "acknowledged" : true, 189 | "insertedId" : ObjectId("5e5ea8f3bb2265ca8fa4b7ae") 190 | } 191 | bye 192 | ``` 193 | 194 | * 检查复制的数据 195 | ``` 196 | > db.books.find(); 197 | { "_id" : ObjectId("5e5ea8f3bb2265ca8fa4b7ae"), "id" : 123, "title" : "HG-PGSQL1.1", "author" : "Highgo" } 198 | > 199 | ``` 200 | 201 | * 删除逻辑插槽如果不再使用的话 202 | ``` 203 | postgres=# SELECT pg_drop_replication_slot('w2m_slot'); 204 | pg_drop_replication_slot 205 | -------------------------- 206 | 207 | (1 row) 208 | ``` 209 | 210 | #### 使用 pg_recvlogical 211 | * 创建一个逻辑插槽 212 | ``` 213 | $ pg_recvlogical -d postgres --slot w2m_slot2 --create-slot --plugin=wal2mongo 214 | ``` 215 | 216 | * 在终端1上启动pg_recvlogical 217 | ``` 218 | $ pg_recvlogical -d postgres --slot w2m_slot2 --start -f - 219 | ``` 220 | 或者让 pg_recvlogical 记录所有数据改动到一个文件，例如 221 | ``` 222 | $ pg_recvlogical -d postgres --slot w2m_slot2 --start -f test2.js 223 | ``` 224 | 225 | * 创建一个表并从终端2插入数据 226 | ``` 227 | postgres=# CREATE TABLE books ( 228 | id SERIAL PRIMARY KEY, 229 | title VARCHAR(100) NOT NULL, 230 | author VARCHAR(100) NULL 231 | ); 232 | 233 | postgres=# insert into books 234 | (id, title, author) 235 | values 236 | (124, 'HG-PGSQL1.2', 'Highgo'); 237 | ``` 238 | 239 | * 切换回终端1来检查更改 240 | 241 | 如下所示的一条记录应该显示在终端1或文件 test2.js 中， 242 | ``` 243 | db.books.insertOne( { id:124, title:"HG-PGSQL1.2", author:"Highgo" } ) 244 | ``` 245 | 246 | * 将数据复制到mongoDB 247 | 248 | * 删除逻辑插槽如果不再使用的话 249 | ``` 250 | $ pg_recvlogical -d postgres --slot w2m_slot2 --drop-slot 251 | ``` 252 | 253 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![License](https://img.shields.io/github/license/HighgoSoftware/wal2mongo) 2 | ![Build](https://travis-ci.com/HighgoSoftware/wal2mongo.svg?branch=release) 3 | [![Release](https://img.shields.io/github/v/tag/HighgoSoftware/wal2mongo?label=Release)](https://github.com/HighgoSoftware/wal2mongo/releases) 4 | 5 | ### Introduction 6 | `wal2mongo` is a PostgreSQL logical decoding output plugin designed to make the logical replication easier from `PostgreSQL` to `MongoDB` by formating the output to a JSON-like format accepted by `mongo`. 7 | 8 | ### Prerequisites 9 | To use wal2mongo logical decoding output plugin, either one of below PostgreSQL servers need to be installed, 10 | * [PostgreSQL 12.x](https://www.postgresql.org/download) 11 | * [HighGo PostgreSQL Server 1.x](https://www.highgo.ca/products/highgo-postgresql-server) 12 | 13 | ### Build, Test and Install 14 | `wal2mongo` is designed to support two typical ways for building PostgreSQL extension: one is for developers who want to manage `wal2mongo` source code under PostgreSQL source code tree structure; the other one is for developers or DBA who want to integrate `wal2mongo` to existing PostgreSQL binaries. 15 | 16 | #### On a Linux-like environment 17 | * Build under PostgreSQL Source Code Tree 18 | ``` 19 | cd /path/to/postgres/contrib/ 20 | git clone https://github.com/HighgoSoftware/wal2mongo.git 21 | cd wal2mongo 22 | make 23 | make install 24 | make check 25 | ``` 26 | 27 | * Build against PostgreSQL binaries 28 | ``` 29 | mkdir sandbox 30 | cd sandbox 31 | git clone https://github.com/HighgoSoftware/wal2mongo.git 32 | cd wal2mongo 33 | ``` 34 | 35 | Set the `PATH` to point to existing PostgreSQL binary directory. Use `PGHOST` and `PGPORT` to specify a PostgreSQL server and port to run the installcheck-force test against if different from default ones. 36 | ``` 37 | $ export PATH=/path/to/postgres/bin:$PATH 38 | USE_PGXS=1 make 39 | USE_PGXS=1 make install 40 | USE_PGXS=1 make installcheck-force 41 | ``` 42 | 43 | #### On Windows7, 10 and 2019 Server 44 | * Build under PostgreSQL Source Code Tree 45 | 1. Following the instruction [here](https://www.postgresql.org/docs/12/install-windows-full.html) to setup the build environment using Microsoft Windows SDK. The [Visual Studio 2019 Community](https://visualstudio.microsoft.com/downloads/) is enough for building Postgres 12.x and wal2mongo logical decoding output plugin. After VS 2019 has been installed successfully, download [`ActivePerl 5.28`](https://www.activestate.com/products/perl/downloads/), [`ActiveTcl 8.6`](https://www.activestate.com/products/tcl/downloads/) and [`GnuWin32 0.6.3`](https://sourceforge.net/projects/getgnuwin32/files/getgnuwin32/0.6.30/GetGnuWin32-0.6.3.exe/download) and install them with the default setting would be enough. 46 | 47 | 2. Check the binaries path for `ActivePerl`, `ActiveTcl` and `GnuWin32` for System variables in Environment variables management panel, if not exist then add them in. 48 | 49 | 3. Build and install 50 | 51 | ``` 52 | cd \path\to\postgres\contrib\ 53 | git clone https://github.com/HighgoSoftware/wal2mongo.git 54 | cd \path\to\postgres\src\tools\msvc\ 55 | build 56 | install \path\to\install\foler\ 57 | ``` 58 | 4. Run regress test (notes, regress test a single extension is not supported by vcregress yet) 59 | ``` 60 | vcregress contribcheck 61 | ``` 62 | * Build against PostgreSQL binaries 63 | 64 | `wal2mongo` can be built in a separate project folder using Visual Studio 2019 by following the steps below: 65 | 1. Create a `new projec`t using `Empty Project` template, and set `Project name` to `wal2mong`, for example. 66 | 2. Right click `wal2mongo` project -> Add -> New items ..., select `C++ File` but name it as `wal2mongo.c` 67 | 3. Paste all the c source code from github to local `wal2mongo.c` file. 68 | 4. Add `PGDLLEXPORT` to function `_PG_init` and `_PG_output_plugin_init`, like below, 69 | ``` 70 | extern PGDLLEXPORT void _PG_init(void); 71 | extern PGDLLEXPORT void _PG_output_plugin_init(OutputPluginCallbacks* cb); 72 | ``` 73 | 5. Right click `wal2mongo` project -> Properties, then change below settings, 74 | General -> Configuration Type `Dynamic Library (.dll)` 75 | 76 | Set C/C++ -> Code Generation -> Enable C++ Exceptions to `NO` 77 | 78 | Set C/C++ -> Advanced -> Compile As to `Compile As C Code` 79 | 80 | Set Linker -> Manifest File -> Generate Manifest to `No` 81 | 82 | Add `postgres.lib` to Linker -> Input -> Additional Dependencies 83 | 84 | Depends on where PostgreSQL binaries are installed, add below pathes in below order to C/C++ -> General -> Additional Include Directories 85 | ``` 86 | C:\Users\Administrator\Downloads\pg12.2\include\server\port\win32_msvc 87 | C:\Users\Administrator\Downloads\pg12.2\include\server\port\win32 88 | C:\Users\Administrator\Downloads\pg12.2\include\server 89 | C:\Users\Administrator\Downloads\pg12.2\include 90 | ``` 91 | Again, depends on where PostgreSQL binaries are installed, add a path like below to Linker -> General -> Additional Library Directories 92 | ``` 93 | C:\Users\Administrator\Downloads\pg12.2\lib 94 | ``` 95 | 6. Right click `wal2mongo1, then click `build`. If everything goes fine, then below message should show up. 96 | ``` 97 | 1>wal2mongo.vcxproj -> C:\Users\Administrator\source\repos\wal2mongo\Debug\wal2mongo.dll 98 | 1>Done building project "wal2mongo.vcxproj". 99 | ========== Rebuild All: 1 succeeded, 0 failed, 0 skipped ========== 100 | ``` 101 | 7. Manually copy `wal2mongo.dll` to the `lib` where PostgreaSQL is installed, then run test following `Examples` section. 102 | 103 | 104 | ### Setup and configuration 105 | Edit PostgreSQl configuration file `postgresql.conf` and make sure `wal_level` is set to `logical`, and `max_replication_slots` is set at least 1 (default settings is 10). 106 | Restart postgres services. 107 | 108 | ### Options 109 | TBD... 110 | 111 | ### Examples 112 | Below are two simple ways to replicate data from PostgreSQL to MongoDB using `wal2mongo`: one use psql console; the other one use pg_recvlogical tools. 113 | 114 | #### using psql 115 | * Create a slot 116 | For example, create a slot nameed 'w2m_slot' using the output plugin `wal2mongo`. 117 | ``` 118 | postgres=# SELECT * FROM pg_create_logical_replication_slot('w2m_slot', 'wal2mongo'); 119 | slot_name | lsn 120 | -----------+------------ 121 | w2m_slot | 1/3CB04148 122 | (1 row) 123 | ``` 124 | 125 | * Check the slot just created 126 | ``` 127 | postgres=# SELECT slot_name, plugin, slot_type, database, active, restart_lsn, confirmed_flush_lsn FROM pg_replication_slots; 128 | slot_name | plugin | slot_type | database | active | restart_lsn | confirmed_flush_lsn 129 | -----------+-----------+-----------+----------+--------+-------------+--------------------- 130 | w2m_slot | wal2mongo | logical | postgres | f | 1/3CB04110 | 1/3CB04148 131 | (1 row) 132 | ``` 133 | 134 | * Create a table and insert data 135 | ``` 136 | postgres=# CREATE TABLE books ( 137 | id SERIAL PRIMARY KEY, 138 | title VARCHAR(100) NOT NULL, 139 | author VARCHAR(100) NULL 140 | ); 141 | 142 | postgres=# insert into books 143 | (id, title, author) 144 | values 145 | (123, 'HG-PGSQL1.1', 'Highgo'); 146 | ``` 147 | 148 | * Peek if any changes 149 | ``` 150 | postgres=# SELECT * FROM pg_logical_slot_peek_changes('w2m_slot', NULL, NULL); 151 | lsn | xid | data 152 | ------------+------+------------------------------------------------------------------------ 153 | 1/3CB247F8 | 1793 | db.books.insertOne( { id:123, title:"HG-PGSQL1.1", author:"Highgo" } ) 154 | (1 row) 155 | ``` 156 | 157 | * Retrieve the changes 158 | ``` 159 | postgres=# SELECT * FROM pg_logical_slot_get_changes('w2m_slot', NULL, NULL); 160 | lsn | xid | data 161 | ------------+------+------------------------------------------------------------------------ 162 | 1/3CB247F8 | 1793 | db.books.insertOne( { id:123, title:"HG-PGSQL1.1", author:"Highgo" } ) 163 | (1 row) 164 | ``` 165 | 166 | * Replicate data within mongo console (option 1) 167 | 168 | log into mongoDB, and copy all the strings from data section, and paste to mongo console 169 | ``` 170 | > db.books.insertOne( { id:123, title:"HG-PGSQL1.1", author:"Highgo" } ) 171 | { 172 | "acknowledged" : true, 173 | "insertedId" : ObjectId("5e5ea92be9684c562aae5b7a") 174 | } 175 | ``` 176 | 177 | * Replicate data using .js file (option 2) 178 | 179 | Copy all the strings from data section, and paste it to a file, e.g. test.js, then import the file using mongo 180 | ``` 181 | $ mongo < test.js 182 | MongoDB shell version v4.0.16 183 | connecting to: mongodb://127.0.0.1:27017/?gssapiServiceName=mongodb 184 | Implicit session: session { "id" : UUID("86ddf177-9704-43f9-9f66-31ac1f9f89e0") } 185 | MongoDB server version: 4.0.16 186 | { 187 | "acknowledged" : true, 188 | "insertedId" : ObjectId("5e5ea8f3bb2265ca8fa4b7ae") 189 | } 190 | bye 191 | ``` 192 | 193 | * check the data replicated 194 | ``` 195 | > db.books.find(); 196 | { "_id" : ObjectId("5e5ea8f3bb2265ca8fa4b7ae"), "id" : 123, "title" : "HG-PGSQL1.1", "author" : "Highgo" } 197 | > 198 | ``` 199 | 200 | * Drop a slot if not used any more 201 | ``` 202 | postgres=# SELECT pg_drop_replication_slot('w2m_slot'); 203 | pg_drop_replication_slot 204 | -------------------------- 205 | 206 | (1 row) 207 | ``` 208 | 209 | #### using pg_recvlogical 210 | * create a slot 211 | ``` 212 | $ pg_recvlogical -d postgres --slot w2m_slot2 --create-slot --plugin=wal2mongo 213 | ``` 214 | 215 | * start logical decoding stream on terminal 1 216 | ``` 217 | $ pg_recvlogical -d postgres --slot w2m_slot2 --start -f - 218 | ``` 219 | Or let pg_recvlogical record all the changes to a file, e.g. 220 | ``` 221 | $ pg_recvlogical -d postgres --slot w2m_slot2 --start -f test2.js 222 | ``` 223 | 224 | * Create a table and insert data from terminal 2 225 | ``` 226 | postgres=# CREATE TABLE books ( 227 | id SERIAL PRIMARY KEY, 228 | title VARCHAR(100) NOT NULL, 229 | author VARCHAR(100) NULL 230 | ); 231 | 232 | postgres=# insert into books 233 | (id, title, author) 234 | values 235 | (124, 'HG-PGSQL1.2', 'Highgo'); 236 | ``` 237 | 238 | * Check the changes by switching back to terminal 1 239 | 240 | One record like below should be showing up either in console or inside file test2.js, 241 | ``` 242 | db.books.insertOne( { id:124, title:"HG-PGSQL1.2", author:"Highgo" } ) 243 | ``` 244 | 245 | * replcate data to mongoDB same as above 246 | 247 | * Drop a slot if not used any more 248 | ``` 249 | $ pg_recvlogical -d postgres --slot w2m_slot2 --drop-slot 250 | ``` 251 | 252 | -------------------------------------------------------------------------------- /expected/actions.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- actions 11 | CREATE TABLE testing (a integer primary key); 12 | INSERT INTO testing (a) VALUES(200); 13 | UPDATE testing SET a = 500 WHERE a = 200; 14 | DELETE FROM testing WHERE a = 500; 15 | TRUNCATE TABLE testing; 16 | -- peek changes according to action configuration 17 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'insert', 'regress', 'true'); 18 | data 19 | -------------------------------------------------------------------------------------- 20 | use mycluster_mydb_regression_slot; 21 | db.testing.insertOne( { a: NumberInt("200") } ); 22 | use mycluster_mydb_regression_slot; 23 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 24 | use mycluster_mydb_regression_slot; 25 | db.testing.deleteOne( { a: NumberInt("500") } ); 26 | (6 rows) 27 | 28 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'update', 'regress', 'true'); 29 | data 30 | -------------------------------------------------------------------------------------- 31 | use mycluster_mydb_regression_slot; 32 | db.testing.insertOne( { a: NumberInt("200") } ); 33 | use mycluster_mydb_regression_slot; 34 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 35 | use mycluster_mydb_regression_slot; 36 | db.testing.deleteOne( { a: NumberInt("500") } ); 37 | (6 rows) 38 | 39 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'delete', 'regress', 'true'); 40 | data 41 | -------------------------------------------------------------------------------------- 42 | use mycluster_mydb_regression_slot; 43 | db.testing.insertOne( { a: NumberInt("200") } ); 44 | use mycluster_mydb_regression_slot; 45 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 46 | use mycluster_mydb_regression_slot; 47 | db.testing.deleteOne( { a: NumberInt("500") } ); 48 | (6 rows) 49 | 50 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'truncate', 'regress', 'true'); 51 | data 52 | -------------------------------------------------------------------------------------- 53 | use mycluster_mydb_regression_slot; 54 | db.testing.insertOne( { a: NumberInt("200") } ); 55 | use mycluster_mydb_regression_slot; 56 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 57 | use mycluster_mydb_regression_slot; 58 | db.testing.deleteOne( { a: NumberInt("500") } ); 59 | (6 rows) 60 | 61 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'insert, update, delete, truncate', 'regress', 'true'); 62 | data 63 | -------------------------------------------------------------------------------------- 64 | use mycluster_mydb_regression_slot; 65 | db.testing.insertOne( { a: NumberInt("200") } ); 66 | use mycluster_mydb_regression_slot; 67 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 68 | use mycluster_mydb_regression_slot; 69 | db.testing.deleteOne( { a: NumberInt("500") } ); 70 | (6 rows) 71 | 72 | -- peek changes with default action configuraiton 73 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 74 | data 75 | -------------------------------------------------------------------------------------- 76 | use mycluster_mydb_regression_slot; 77 | db.testing.insertOne( { a: NumberInt("200") } ); 78 | use mycluster_mydb_regression_slot; 79 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 80 | use mycluster_mydb_regression_slot; 81 | db.testing.deleteOne( { a: NumberInt("500") } ); 82 | (6 rows) 83 | 84 | -- peek changes with several configuration parameter combinations 85 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include_cluster_name', 'true', 'regress', 'true'); 86 | data 87 | -------------------------------------------------------------------------------------- 88 | use mycluster_mydb_regression_slot; 89 | db.testing.insertOne( { a: NumberInt("200") } ); 90 | use mycluster_mydb_regression_slot; 91 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 92 | use mycluster_mydb_regression_slot; 93 | db.testing.deleteOne( { a: NumberInt("500") } ); 94 | (6 rows) 95 | 96 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip_empty_xacts', 'true', 'regress', 'true'); 97 | data 98 | -------------------------------------------------------------------------------------- 99 | use mycluster_mydb_regression_slot; 100 | db.testing.insertOne( { a: NumberInt("200") } ); 101 | use mycluster_mydb_regression_slot; 102 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 103 | use mycluster_mydb_regression_slot; 104 | db.testing.deleteOne( { a: NumberInt("500") } ); 105 | (6 rows) 106 | 107 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'only_local', 'true', 'regress', 'true'); 108 | data 109 | -------------------------------------------------------------------------------------- 110 | use mycluster_mydb_regression_slot; 111 | db.testing.insertOne( { a: NumberInt("200") } ); 112 | use mycluster_mydb_regression_slot; 113 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 114 | use mycluster_mydb_regression_slot; 115 | db.testing.deleteOne( { a: NumberInt("500") } ); 116 | (6 rows) 117 | 118 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'use_transaction', 'true', 'regress', 'true'); 119 | data 120 | -------------------------------------------------------------------------------------- 121 | session0regression_slot = db.getMongo().startSession(); 122 | session0regression_slot.startTransaction(); 123 | session0regression_slot.commitTransaction(); 124 | session0regression_slot.endSession(); 125 | session0regression_slot = db.getMongo().startSession(); 126 | session0regression_slot.startTransaction(); 127 | use mycluster_mydb_regression_slot; 128 | db.testing.insertOne( { a: NumberInt("200") } ); 129 | session0regression_slot.commitTransaction(); 130 | session0regression_slot.endSession(); 131 | session0regression_slot = db.getMongo().startSession(); 132 | session0regression_slot.startTransaction(); 133 | use mycluster_mydb_regression_slot; 134 | db.testing.updateOne( { a: NumberInt("200") }, { $set: { a: NumberInt("500") } } ); 135 | session0regression_slot.commitTransaction(); 136 | session0regression_slot.endSession(); 137 | session0regression_slot = db.getMongo().startSession(); 138 | session0regression_slot.startTransaction(); 139 | use mycluster_mydb_regression_slot; 140 | db.testing.deleteOne( { a: NumberInt("500") } ); 141 | session0regression_slot.commitTransaction(); 142 | session0regression_slot.endSession(); 143 | session0regression_slot = db.getMongo().startSession(); 144 | session0regression_slot.startTransaction(); 145 | session0regression_slot.commitTransaction(); 146 | session0regression_slot.endSession(); 147 | (26 rows) 148 | 149 | -- peek changes with invalid actions 150 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'insert, xxx, delete, xxx', 'regress', 'true'); 151 | ERROR: could not parse value "xxx" for parameter "actions" 152 | DROP TABLE testing; 153 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 154 | ?column? 155 | ---------- 156 | end 157 | (1 row) 158 | 159 | -------------------------------------------------------------------------------- /expected/array.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- create table with different numeric type 11 | CREATE TABLE tbl_array(id serial primary key, a bool[], c char[], d name[], e int2[], f int4[], g text[], h varchar[], i int8[], j float4[], k float8[], l timestamptz[], m numeric[], n uuid[] ); 12 | -- different data types for array 13 | INSERT INTO tbl_array (a, c, d, e, f, g, h, i, j, k, l, m, n) VALUES( 14 | ARRAY[true, false], 15 | ARRAY['c'::char,'h'::char], 16 | ARRAY['student'::name, 'teacher'::name], 17 | ARRAY['123'::int2, '456'::int2], 18 | ARRAY['123456789'::int4,'987654321'::int4], 19 | ARRAY['abc'::text, '123'::text], 20 | ARRAY['ABCD'::varchar, '1234'::varchar], 21 | ARRAY['112233445566778899'::int8, '998877665544332211'::int8], 22 | ARRAY['123.456'::float4, '2222.3333'::float4], 23 | ARRAY['123456.123'::float8, '654321.123'::float8], 24 | ARRAY['2020-03-30 10:18:40.12-07'::timestamptz, '2020-03-30 20:28:40.12-07'::timestamptz], 25 | ARRAY['123456789'::numeric, '987654321'::numeric], 26 | ARRAY['40e6215d-b5c6-4896-987c-f30f3678f608'::uuid, '3f333df6-90a4-4fda-8dd3-9485d27cee36'::uuid] 27 | ); 28 | UPDATE tbl_array SET 29 | a=ARRAY[false, true], 30 | c=ARRAY['h'::char, 'c'::char], 31 | d=ARRAY['teacher'::name, 'student'::name], 32 | e=ARRAY['456'::int2, '123'::int2], 33 | f=ARRAY['987654321'::int4, '123456789'::int4], 34 | g=ARRAY['123'::text, 'abc'::text], 35 | h=ARRAY['1234'::varchar, 'ABCD'::varchar], 36 | i=ARRAY['998877665544332211'::int8, '112233445566778899'::int8], 37 | j=ARRAY['2222.3333'::float4, '123.456'::float4], 38 | k=ARRAY['654321.123'::float8, '123456.123'::float8], 39 | l=ARRAY['2020-03-30 20:28:40.12-07'::timestamptz, '2020-03-30 10:18:40.12-07'::timestamptz], 40 | m=ARRAY['987654321'::numeric, '123456789'::numeric], 41 | n=ARRAY['3f333df6-90a4-4fda-8dd3-9485d27cee36'::uuid, '40e6215d-b5c6-4896-987c-f30f3678f608'::uuid] 42 | WHERE id=1; 43 | DELETE FROM tbl_array WHERE id = 1; 44 | TRUNCATE TABLE tbl_array; 45 | -- peek changes according to action configuration 46 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 47 | data 48 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 49 | use mycluster_mydb_regression_slot; 50 | db.tbl_array.insertOne( { id: NumberInt("1"), a:[true,false], c:["c","h"], d:["student","teacher"], e:[NumberInt(123),NumberInt(456)], f:[NumberInt(123456789),NumberInt(987654321)], g:["abc","123"], h:["ABCD","1234"], i:[NumberLong(112233445566778899),NumberLong(998877665544332211)], j:[123.456,2222.3333], k:[NumberDecimal("123456.123"),NumberDecimal("654321.123")], l:[ISODate("Mon Mar 30 10:18:40.12 2020 PDT"),ISODate("Mon Mar 30 20:28:40.12 2020 PDT")], m:[NumberDecimal("123456789"),NumberDecimal("987654321")], n:[UUID("40e6215d-b5c6-4896-987c-f30f3678f608"),UUID("3f333df6-90a4-4fda-8dd3-9485d27cee36")] } ); 51 | use mycluster_mydb_regression_slot; 52 | db.tbl_array.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:[false,true], c:["h","c"], d:["teacher","student"], e:[NumberInt(456),NumberInt(123)], f:[NumberInt(987654321),NumberInt(123456789)], g:["123","abc"], h:["1234","ABCD"], i:[NumberLong(998877665544332211),NumberLong(112233445566778899)], j:[2222.3333,123.456], k:[NumberDecimal("654321.123"),NumberDecimal("123456.123")], l:[ISODate("Mon Mar 30 20:28:40.12 2020 PDT"),ISODate("Mon Mar 30 10:18:40.12 2020 PDT")], m:[NumberDecimal("987654321"),NumberDecimal("123456789")], n:[UUID("3f333df6-90a4-4fda-8dd3-9485d27cee36"),UUID("40e6215d-b5c6-4896-987c-f30f3678f608")] } } ); 53 | use mycluster_mydb_regression_slot; 54 | db.tbl_array.deleteOne( { id: NumberInt("1") } ); 55 | (6 rows) 56 | 57 | -- get changes according to action configuration 58 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 59 | data 60 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 61 | use mycluster_mydb_regression_slot; 62 | db.tbl_array.insertOne( { id: NumberInt("1"), a:[true,false], c:["c","h"], d:["student","teacher"], e:[NumberInt(123),NumberInt(456)], f:[NumberInt(123456789),NumberInt(987654321)], g:["abc","123"], h:["ABCD","1234"], i:[NumberLong(112233445566778899),NumberLong(998877665544332211)], j:[123.456,2222.3333], k:[NumberDecimal("123456.123"),NumberDecimal("654321.123")], l:[ISODate("Mon Mar 30 10:18:40.12 2020 PDT"),ISODate("Mon Mar 30 20:28:40.12 2020 PDT")], m:[NumberDecimal("123456789"),NumberDecimal("987654321")], n:[UUID("40e6215d-b5c6-4896-987c-f30f3678f608"),UUID("3f333df6-90a4-4fda-8dd3-9485d27cee36")] } ); 63 | use mycluster_mydb_regression_slot; 64 | db.tbl_array.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:[false,true], c:["h","c"], d:["teacher","student"], e:[NumberInt(456),NumberInt(123)], f:[NumberInt(987654321),NumberInt(123456789)], g:["123","abc"], h:["1234","ABCD"], i:[NumberLong(998877665544332211),NumberLong(112233445566778899)], j:[2222.3333,123.456], k:[NumberDecimal("654321.123"),NumberDecimal("123456.123")], l:[ISODate("Mon Mar 30 20:28:40.12 2020 PDT"),ISODate("Mon Mar 30 10:18:40.12 2020 PDT")], m:[NumberDecimal("987654321"),NumberDecimal("123456789")], n:[UUID("3f333df6-90a4-4fda-8dd3-9485d27cee36"),UUID("40e6215d-b5c6-4896-987c-f30f3678f608")] } } ); 65 | use mycluster_mydb_regression_slot; 66 | db.tbl_array.deleteOne( { id: NumberInt("1") } ); 67 | (6 rows) 68 | 69 | -- drop table 70 | DROP TABLE tbl_array; 71 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 72 | ?column? 73 | ---------- 74 | end 75 | (1 row) 76 | 77 | -------------------------------------------------------------------------------- /expected/binData.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- UUID 11 | CREATE TABLE tbl_uuid ( id serial PRIMARY KEY, a_uuid UUID NOT NULL ); 12 | INSERT INTO tbl_uuid (a_uuid) VALUES('47deacb1-3ad0-4a0e-8254-9ad3f589c9f3') ; 13 | UPDATE tbl_uuid SET a_uuid = 'e7d8e462-12cc-49dc-aac2-2b5dccdabeda' WHERE id=1; 14 | DELETE FROM tbl_uuid WHERE id = 1; 15 | TRUNCATE TABLE tbl_uuid; 16 | -- BYTEA 17 | CREATE TABLE tbl_bytea ( id serial PRIMARY KEY, a_bytea bytea NOT NULL ); 18 | ALTER TABLE tbl_bytea REPLICA IDENTITY FULL; 19 | INSERT INTO tbl_bytea(a_bytea) SELECT 'abc \153\154\155 \052\251\124'::bytea RETURNING a_bytea; 20 | a_bytea 21 | -------------------------- 22 | \x616263206b6c6d202aa954 23 | (1 row) 24 | 25 | UPDATE tbl_bytea SET a_bytea = SELECT '\134'::bytea RETURNING a_bytea; 26 | ERROR: syntax error at or near "SELECT" at character 32 27 | DELETE FROM tbl_bytea WHERE id = 1; 28 | TRUNCATE TABLE tbl_bytea; 29 | -- peek changes according to action configuration 30 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 31 | data 32 | ------------------------------------------------------------------------------------------------------------------------------------------- 33 | use mycluster_mydb_regression_slot; 34 | db.tbl_uuid.insertOne( { id: NumberInt("1"), a_uuid: UUID("47deacb1-3ad0-4a0e-8254-9ad3f589c9f3") } ); 35 | use mycluster_mydb_regression_slot; 36 | db.tbl_uuid.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a_uuid: UUID("e7d8e462-12cc-49dc-aac2-2b5dccdabeda") } } ); 37 | use mycluster_mydb_regression_slot; 38 | db.tbl_uuid.deleteOne( { id: NumberInt("1") } ); 39 | use mycluster_mydb_regression_slot; 40 | db.tbl_bytea.insertOne( { id: NumberInt("1"), a_bytea: HexData(0, "616263206b6c6d202aa954") } ); 41 | use mycluster_mydb_regression_slot; 42 | db.tbl_bytea.deleteOne( { id: NumberInt("1") } ); 43 | (10 rows) 44 | 45 | -- get changes according to action configuration 46 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 47 | data 48 | ------------------------------------------------------------------------------------------------------------------------------------------- 49 | use mycluster_mydb_regression_slot; 50 | db.tbl_uuid.insertOne( { id: NumberInt("1"), a_uuid: UUID("47deacb1-3ad0-4a0e-8254-9ad3f589c9f3") } ); 51 | use mycluster_mydb_regression_slot; 52 | db.tbl_uuid.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a_uuid: UUID("e7d8e462-12cc-49dc-aac2-2b5dccdabeda") } } ); 53 | use mycluster_mydb_regression_slot; 54 | db.tbl_uuid.deleteOne( { id: NumberInt("1") } ); 55 | use mycluster_mydb_regression_slot; 56 | db.tbl_bytea.insertOne( { id: NumberInt("1"), a_bytea: HexData(0, "616263206b6c6d202aa954") } ); 57 | use mycluster_mydb_regression_slot; 58 | db.tbl_bytea.deleteOne( { id: NumberInt("1") } ); 59 | (10 rows) 60 | 61 | -- drop tables 62 | DROP TABLE tbl_uuid; 63 | DROP TABLE tbl_bytea; 64 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 65 | ?column? 66 | ---------- 67 | end 68 | (1 row) 69 | 70 | -------------------------------------------------------------------------------- /expected/binary.out: -------------------------------------------------------------------------------- 1 | -- predictability 2 | SET synchronous_commit = on; 3 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 4 | ?column? 5 | ---------- 6 | init 7 | (1 row) 8 | 9 | -- succeeds, textual plugin, textual consumer 10 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'force_binary', '0'); 11 | data 12 | ------ 13 | (0 rows) 14 | 15 | -- fails, binary plugin, textual consumer 16 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'force_binary', '1'); 17 | ERROR: logical decoding output plugin "wal2mongo" produces binary output, but function "pg_logical_slot_get_changes(name,pg_lsn,integer,text[])" expects textual data 18 | -- succeeds, textual plugin, binary consumer 19 | SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot', NULL, NULL, 'force_binary', '0'); 20 | data 21 | ------ 22 | (0 rows) 23 | 24 | -- succeeds, binary plugin, binary consumer 25 | SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot', NULL, NULL, 'force_binary', '1'); 26 | data 27 | ------ 28 | (0 rows) 29 | 30 | SELECT 'init' FROM pg_drop_replication_slot('regression_slot'); 31 | ?column? 32 | ---------- 33 | init 34 | (1 row) 35 | 36 | -------------------------------------------------------------------------------- /expected/boolean.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- create table with boolean type 11 | CREATE TABLE tbl_boolean (a integer primary key, b BOOLEAN); 12 | -- true 13 | INSERT INTO tbl_boolean (a, b) VALUES(1, true); 14 | UPDATE tbl_boolean SET b = false WHERE a = 1; 15 | DELETE FROM tbl_boolean WHERE a = 1; 16 | TRUNCATE TABLE tbl_boolean; 17 | -- 'true' 18 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'true'); 19 | UPDATE tbl_boolean SET b = 'false' WHERE a = 1; 20 | DELETE FROM tbl_boolean WHERE a = 1; 21 | TRUNCATE TABLE tbl_boolean; 22 | -- 't' 23 | INSERT INTO tbl_boolean (a, b) VALUES(1, 't'); 24 | UPDATE tbl_boolean SET b = 'f' WHERE a = 1; 25 | DELETE FROM tbl_boolean WHERE a = 1; 26 | TRUNCATE TABLE tbl_boolean; 27 | -- TRUE 28 | INSERT INTO tbl_boolean (a, b) VALUES(1, TRUE); 29 | UPDATE tbl_boolean SET b = TRUE WHERE a = 1; 30 | DELETE FROM tbl_boolean WHERE a = 1; 31 | TRUNCATE TABLE tbl_boolean; 32 | -- 'TRUE' 33 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'TRUE'); 34 | UPDATE tbl_boolean SET b = 'TRUE' WHERE a = 1; 35 | DELETE FROM tbl_boolean WHERE a = 1; 36 | TRUNCATE TABLE tbl_boolean; 37 | -- 'T' 38 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'T'); 39 | UPDATE tbl_boolean SET b = 'T' WHERE a = 1; 40 | DELETE FROM tbl_boolean WHERE a = 1; 41 | TRUNCATE TABLE tbl_boolean; 42 | -- 'on' 43 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'on'); 44 | UPDATE tbl_boolean SET b = 'off' WHERE a = 1; 45 | DELETE FROM tbl_boolean WHERE a = 1; 46 | TRUNCATE TABLE tbl_boolean; 47 | -- '1' 48 | INSERT INTO tbl_boolean (a, b) VALUES(1, '1'); 49 | UPDATE tbl_boolean SET b = '0' WHERE a = 1; 50 | DELETE FROM tbl_boolean WHERE a = 1; 51 | TRUNCATE TABLE tbl_boolean; 52 | -- false 53 | INSERT INTO tbl_boolean (a, b) VALUES(1, false); 54 | UPDATE tbl_boolean SET b = true WHERE a = 1; 55 | DELETE FROM tbl_boolean WHERE a = 1; 56 | TRUNCATE TABLE tbl_boolean; 57 | -- 'false' 58 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'false'); 59 | UPDATE tbl_boolean SET b = 'true' WHERE a = 1; 60 | DELETE FROM tbl_boolean WHERE a = 1; 61 | TRUNCATE TABLE tbl_boolean; 62 | -- 'f' 63 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'f'); 64 | UPDATE tbl_boolean SET b = 't' WHERE a = 1; 65 | DELETE FROM tbl_boolean WHERE a = 1; 66 | TRUNCATE TABLE tbl_boolean; 67 | -- FALSE 68 | INSERT INTO tbl_boolean (a, b) VALUES(1, FALSE); 69 | UPDATE tbl_boolean SET b = TRUE WHERE a = 1; 70 | DELETE FROM tbl_boolean WHERE a = 1; 71 | TRUNCATE TABLE tbl_boolean; 72 | -- 'FALSE' 73 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'FALSE'); 74 | UPDATE tbl_boolean SET b = 'TRUE' WHERE a = 1; 75 | DELETE FROM tbl_boolean WHERE a = 1; 76 | TRUNCATE TABLE tbl_boolean; 77 | -- 'F' 78 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'F'); 79 | UPDATE tbl_boolean SET b = 'T' WHERE a = 1; 80 | DELETE FROM tbl_boolean WHERE a = 1; 81 | TRUNCATE TABLE tbl_boolean; 82 | -- 'off' 83 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'off'); 84 | UPDATE tbl_boolean SET b = 'on' WHERE a = 1; 85 | DELETE FROM tbl_boolean WHERE a = 1; 86 | TRUNCATE TABLE tbl_boolean; 87 | -- '0' 88 | INSERT INTO tbl_boolean (a, b) VALUES(1, '0'); 89 | UPDATE tbl_boolean SET b = '1' WHERE a = 1; 90 | DELETE FROM tbl_boolean WHERE a = 1; 91 | TRUNCATE TABLE tbl_boolean; 92 | -- 'on' exception 93 | INSERT INTO tbl_boolean (a, b) VALUES(1, on); 94 | ERROR: syntax error at or near "on" at character 42 95 | UPDATE tbl_boolean SET b = off WHERE a = 1; 96 | ERROR: column "off" does not exist at character 28 97 | DELETE FROM tbl_boolean WHERE a = 1; 98 | TRUNCATE TABLE tbl_boolean; 99 | -- 'enable' exception 100 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'enable'); 101 | ERROR: invalid input syntax for type boolean: "enable" at character 42 102 | UPDATE tbl_boolean SET b = 'disable' WHERE a = 1; 103 | ERROR: invalid input syntax for type boolean: "disable" at character 28 104 | DELETE FROM tbl_boolean WHERE a = 1; 105 | TRUNCATE TABLE tbl_boolean; 106 | -- 'disable' exception 107 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'disable'); 108 | ERROR: invalid input syntax for type boolean: "disable" at character 42 109 | UPDATE tbl_boolean SET b = 'enable' WHERE a = 1; 110 | ERROR: invalid input syntax for type boolean: "enable" at character 28 111 | DELETE FROM tbl_boolean WHERE a = 1; 112 | TRUNCATE TABLE tbl_boolean; 113 | -- peek changes according to action configuration 114 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 115 | data 116 | ----------------------------------------------------------------------------------------------- 117 | use mycluster_mydb_regression_slot; 118 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 119 | use mycluster_mydb_regression_slot; 120 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 121 | use mycluster_mydb_regression_slot; 122 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 123 | use mycluster_mydb_regression_slot; 124 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 125 | use mycluster_mydb_regression_slot; 126 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 127 | use mycluster_mydb_regression_slot; 128 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 129 | use mycluster_mydb_regression_slot; 130 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 131 | use mycluster_mydb_regression_slot; 132 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 133 | use mycluster_mydb_regression_slot; 134 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 135 | use mycluster_mydb_regression_slot; 136 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 137 | use mycluster_mydb_regression_slot; 138 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 139 | use mycluster_mydb_regression_slot; 140 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 141 | use mycluster_mydb_regression_slot; 142 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 143 | use mycluster_mydb_regression_slot; 144 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 145 | use mycluster_mydb_regression_slot; 146 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 147 | use mycluster_mydb_regression_slot; 148 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 149 | use mycluster_mydb_regression_slot; 150 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 151 | use mycluster_mydb_regression_slot; 152 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 153 | use mycluster_mydb_regression_slot; 154 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 155 | use mycluster_mydb_regression_slot; 156 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 157 | use mycluster_mydb_regression_slot; 158 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 159 | use mycluster_mydb_regression_slot; 160 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 161 | use mycluster_mydb_regression_slot; 162 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 163 | use mycluster_mydb_regression_slot; 164 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 165 | use mycluster_mydb_regression_slot; 166 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 167 | use mycluster_mydb_regression_slot; 168 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 169 | use mycluster_mydb_regression_slot; 170 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 171 | use mycluster_mydb_regression_slot; 172 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 173 | use mycluster_mydb_regression_slot; 174 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 175 | use mycluster_mydb_regression_slot; 176 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 177 | use mycluster_mydb_regression_slot; 178 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 179 | use mycluster_mydb_regression_slot; 180 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 181 | use mycluster_mydb_regression_slot; 182 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 183 | use mycluster_mydb_regression_slot; 184 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 185 | use mycluster_mydb_regression_slot; 186 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 187 | use mycluster_mydb_regression_slot; 188 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 189 | use mycluster_mydb_regression_slot; 190 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 191 | use mycluster_mydb_regression_slot; 192 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 193 | use mycluster_mydb_regression_slot; 194 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 195 | use mycluster_mydb_regression_slot; 196 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 197 | use mycluster_mydb_regression_slot; 198 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 199 | use mycluster_mydb_regression_slot; 200 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 201 | use mycluster_mydb_regression_slot; 202 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 203 | use mycluster_mydb_regression_slot; 204 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 205 | use mycluster_mydb_regression_slot; 206 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 207 | use mycluster_mydb_regression_slot; 208 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 209 | use mycluster_mydb_regression_slot; 210 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 211 | use mycluster_mydb_regression_slot; 212 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 213 | (96 rows) 214 | 215 | -- get changes according to action configuration 216 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 217 | data 218 | ----------------------------------------------------------------------------------------------- 219 | use mycluster_mydb_regression_slot; 220 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 221 | use mycluster_mydb_regression_slot; 222 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 223 | use mycluster_mydb_regression_slot; 224 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 225 | use mycluster_mydb_regression_slot; 226 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 227 | use mycluster_mydb_regression_slot; 228 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 229 | use mycluster_mydb_regression_slot; 230 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 231 | use mycluster_mydb_regression_slot; 232 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 233 | use mycluster_mydb_regression_slot; 234 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 235 | use mycluster_mydb_regression_slot; 236 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 237 | use mycluster_mydb_regression_slot; 238 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 239 | use mycluster_mydb_regression_slot; 240 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 241 | use mycluster_mydb_regression_slot; 242 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 243 | use mycluster_mydb_regression_slot; 244 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 245 | use mycluster_mydb_regression_slot; 246 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 247 | use mycluster_mydb_regression_slot; 248 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 249 | use mycluster_mydb_regression_slot; 250 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 251 | use mycluster_mydb_regression_slot; 252 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 253 | use mycluster_mydb_regression_slot; 254 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 255 | use mycluster_mydb_regression_slot; 256 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 257 | use mycluster_mydb_regression_slot; 258 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 259 | use mycluster_mydb_regression_slot; 260 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 261 | use mycluster_mydb_regression_slot; 262 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:true } ); 263 | use mycluster_mydb_regression_slot; 264 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:false } } ); 265 | use mycluster_mydb_regression_slot; 266 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 267 | use mycluster_mydb_regression_slot; 268 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 269 | use mycluster_mydb_regression_slot; 270 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 271 | use mycluster_mydb_regression_slot; 272 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 273 | use mycluster_mydb_regression_slot; 274 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 275 | use mycluster_mydb_regression_slot; 276 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 277 | use mycluster_mydb_regression_slot; 278 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 279 | use mycluster_mydb_regression_slot; 280 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 281 | use mycluster_mydb_regression_slot; 282 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 283 | use mycluster_mydb_regression_slot; 284 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 285 | use mycluster_mydb_regression_slot; 286 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 287 | use mycluster_mydb_regression_slot; 288 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 289 | use mycluster_mydb_regression_slot; 290 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 291 | use mycluster_mydb_regression_slot; 292 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 293 | use mycluster_mydb_regression_slot; 294 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 295 | use mycluster_mydb_regression_slot; 296 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 297 | use mycluster_mydb_regression_slot; 298 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 299 | use mycluster_mydb_regression_slot; 300 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 301 | use mycluster_mydb_regression_slot; 302 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 303 | use mycluster_mydb_regression_slot; 304 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 305 | use mycluster_mydb_regression_slot; 306 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 307 | use mycluster_mydb_regression_slot; 308 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 309 | use mycluster_mydb_regression_slot; 310 | db.tbl_boolean.insertOne( { a: NumberInt("1"), b:false } ); 311 | use mycluster_mydb_regression_slot; 312 | db.tbl_boolean.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b:true } } ); 313 | use mycluster_mydb_regression_slot; 314 | db.tbl_boolean.deleteOne( { a: NumberInt("1") } ); 315 | (96 rows) 316 | 317 | -- peek changes with invalid actions 318 | DROP TABLE tbl_boolean; 319 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 320 | ?column? 321 | ---------- 322 | end 323 | (1 row) 324 | 325 | -------------------------------------------------------------------------------- /expected/identity.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- Replica identity nothing 11 | CREATE TABLE testing (a varchar(30) primary key, b INT, c INT); 12 | ALTER TABLE testing REPLICA IDENTITY NOTHING; 13 | BEGIN; 14 | INSERT INTO testing VALUES('aaa', 123, 456); 15 | UPDATE testing set b = 789 where a = 'aaa'; 16 | UPDATE testing set a = 'bbb'; 17 | DELETE from testing where a = 'bbb'; 18 | COMMIT; 19 | -- Replica identity default 20 | BEGIN; 21 | ALTER TABLE testing REPLICA IDENTITY DEFAULT; 22 | INSERT INTO testing VALUES('aaa', 123, 456); 23 | UPDATE testing set b = 789 where a = 'aaa'; 24 | UPDATE testing set a = 'bbb'; 25 | DELETE from testing where a = 'bbb'; 26 | COMMIT; 27 | -- Replica identity full 28 | BEGIN; 29 | ALTER TABLE testing REPLICA IDENTITY FULL; 30 | INSERT INTO testing VALUES('aaa', 123, 456); 31 | UPDATE testing set b = 789 where a = 'aaa'; 32 | UPDATE testing set a = 'bbb'; 33 | DELETE from testing where a = 'bbb'; 34 | COMMIT; 35 | -- Replica identity index 36 | CREATE INDEX idx_test ON testing(b); 37 | ALTER TABLE testing REPLICA IDENTITY INDEX; 38 | ERROR: syntax error at or near "INDEX" at character 38 39 | INSERT INTO testing VALUES('aaa', 123, 456); 40 | UPDATE testing set b = 789 where a = 'aaa'; 41 | UPDATE testing set a = 'bbb'; 42 | DELETE from testing where a = 'bbb'; 43 | COMMIT; 44 | WARNING: there is no transaction in progress 45 | -- get changes 46 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 47 | data 48 | -------------------------------------------------------------------------------------------------------- 49 | use mycluster_mydb_regression_slot; 50 | db.testing.insertOne( { a:"aaa", b: NumberInt("123"), c: NumberInt("456") } ); 51 | use mycluster_mydb_regression_slot; 52 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"aaa", b: NumberInt("789"), c: NumberInt("456") } } ); 53 | use mycluster_mydb_regression_slot; 54 | db.testing.updateOne( { a:"bbb" }, { $set: { a:"bbb", b: NumberInt("789"), c: NumberInt("456") } } ); 55 | use mycluster_mydb_regression_slot; 56 | db.testing.deleteOne( (no-tuple-data) ); 57 | use mycluster_mydb_regression_slot; 58 | db.testing.insertOne( { a:"aaa", b: NumberInt("123"), c: NumberInt("456") } ); 59 | use mycluster_mydb_regression_slot; 60 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"aaa", b: NumberInt("789"), c: NumberInt("456") } } ); 61 | use mycluster_mydb_regression_slot; 62 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"bbb", b: NumberInt("789"), c: NumberInt("456") } } ); 63 | use mycluster_mydb_regression_slot; 64 | db.testing.deleteOne( { a:"bbb" } ); 65 | use mycluster_mydb_regression_slot; 66 | db.testing.insertOne( { a:"aaa", b: NumberInt("123"), c: NumberInt("456") } ); 67 | use mycluster_mydb_regression_slot; 68 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"aaa", b: NumberInt("789"), c: NumberInt("456") } } ); 69 | use mycluster_mydb_regression_slot; 70 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"bbb", b: NumberInt("789"), c: NumberInt("456") } } ); 71 | use mycluster_mydb_regression_slot; 72 | db.testing.deleteOne( { a:"bbb" } ); 73 | use mycluster_mydb_regression_slot; 74 | db.testing.insertOne( { a:"aaa", b: NumberInt("123"), c: NumberInt("456") } ); 75 | use mycluster_mydb_regression_slot; 76 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"aaa", b: NumberInt("789"), c: NumberInt("456") } } ); 77 | use mycluster_mydb_regression_slot; 78 | db.testing.updateOne( { a:"aaa" }, { $set: { a:"bbb", b: NumberInt("789"), c: NumberInt("456") } } ); 79 | use mycluster_mydb_regression_slot; 80 | db.testing.deleteOne( { a:"bbb" } ); 81 | (32 rows) 82 | 83 | DROP TABLE testing; 84 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 85 | ?column? 86 | ---------- 87 | end 88 | (1 row) 89 | 90 | -------------------------------------------------------------------------------- /expected/json.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | CREATE TABLE testing (a integer primary key, 11 | j1 json, 12 | j2 jsonb, 13 | j3 jsonpath, 14 | j4 json[], 15 | j5 jsonb[]); 16 | -- JSON data type 17 | INSERT INTO testing (a, j1) VALUES(1, '{"customer": "John", "items":{"product":"beer","qty":20}}'); 18 | INSERT INTO testing (a, j1) VALUES(2, '{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10},{"product":"cooler","qty":10}]}'); 19 | INSERT INTO testing (a, j1) VALUES(3, '{"customer": "Joe", "items":{"product":"wine","qty":20}}'); 20 | UPDATE testing set j1='{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10}]}' where a = 3; 21 | DELETE FROM testing where a = 1; 22 | -- JSONB data type 23 | INSERT INTO testing (a, j2) VALUES(4, '{"customer": "John", "items":{"product":"beer","qty":20}}'); 24 | INSERT INTO testing (a, j2) VALUES(5, '{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10},{"product":"cooler","qty":10}]}'); 25 | INSERT INTO testing (a, j2) VALUES(6, '{"customer": "Joe", "items":{"product":"wine","qty":20}}'); 26 | UPDATE testing set j2='{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10}]}' where a = 3; 27 | DELETE FROM testing where a = 1; 28 | -- JSONPATH data type 29 | INSERT INTO testing (a, j3) VALUES(7, '$.equipment.rings[*] ? (@.track.segments > 1)'); 30 | UPDATE testing set j3 = '$.equipment.rings[*]' where a = 7; 31 | DELETE from testing where a = 7; 32 | -- JSON[] data type 33 | INSERT into testing (a, j4) VALUES 34 | ( 35 | 8, 36 | array[ 37 | '{"customer": "Cary", "items":{"product":"beer","qty":1000}}', 38 | '{"customer": "John", "items":{"product":"beer","qty":2000}}', 39 | '{"customer": "David", "items":{"product":"beer","qty":3000}}' 40 | ]::json[] 41 | ); 42 | UPDATE testing set j4 = 43 | array[ 44 | '{"customer": "Cary"}', 45 | '{"customer": "John"}', 46 | '{"customer": "David"}' 47 | ]::json[] where a = 8; 48 | DELETE FROM testing where a = 8; 49 | -- JSONB[] data type 50 | INSERT into testing (a, j5) VALUES 51 | ( 52 | 9, 53 | array[ 54 | '{"customer": "Cary", "items":{"product":"beer","qty":1000}}', 55 | '{"customer": "John", "items":{"product":"beer","qty":2000}}', 56 | '{"customer": "David", "items":{"product":"beer","qty":3000}}' 57 | ]::json[] 58 | ); 59 | UPDATE testing set j5 = 60 | array[ 61 | '{"customer": "Cary"}', 62 | '{"customer": "John"}', 63 | '{"customer": "David"}' 64 | ]::json[] where a = 9; 65 | DELETE FROM testing where a = 9; 66 | -- get the changes 67 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'use_transaction', 'false', 'regress', 'true'); 68 | data 69 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 70 | use mycluster_mydb_regression_slot; 71 | db.testing.insertOne( { a: NumberInt("1"), j1:{"customer": "John", "items":{"product":"beer","qty":20}} } ); 72 | use mycluster_mydb_regression_slot; 73 | db.testing.insertOne( { a: NumberInt("2"), j1:{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10},{"product":"cooler","qty":10}]} } ); 74 | use mycluster_mydb_regression_slot; 75 | db.testing.insertOne( { a: NumberInt("3"), j1:{"customer": "Joe", "items":{"product":"wine","qty":20}} } ); 76 | use mycluster_mydb_regression_slot; 77 | db.testing.updateOne( { a: NumberInt("3") }, { $set: { a: NumberInt("3"), j1:{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10}]}, j2:null, j3:null, j4:null, j5:null } } ); 78 | use mycluster_mydb_regression_slot; 79 | db.testing.deleteOne( { a: NumberInt("1") } ); 80 | use mycluster_mydb_regression_slot; 81 | db.testing.insertOne( { a: NumberInt("4"), j2:{"items": {"qty": 20, "product": "beer"}, "customer": "John"} } ); 82 | use mycluster_mydb_regression_slot; 83 | db.testing.insertOne( { a: NumberInt("5"), j2:{"items": [{"qty": 5, "product": "vodka"}, {"qty": 10, "product": "whiskey"}, {"qty": 10, "product": "cooler"}], "customer": "Michael"} } ); 84 | use mycluster_mydb_regression_slot; 85 | db.testing.insertOne( { a: NumberInt("6"), j2:{"items": {"qty": 20, "product": "wine"}, "customer": "Joe"} } ); 86 | use mycluster_mydb_regression_slot; 87 | db.testing.updateOne( { a: NumberInt("3") }, { $set: { a: NumberInt("3"), j1:{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10}]}, j2:{"items": [{"qty": 5, "product": "vodka"}, {"qty": 10, "product": "whiskey"}], "customer": "Michael"}, j3:null, j4:null, j5:null } } ); 88 | use mycluster_mydb_regression_slot; 89 | db.testing.insertOne( { a: NumberInt("7"), j3:"$.equipment.rings[*]?(@.track.segments > 1)" } ); 90 | use mycluster_mydb_regression_slot; 91 | db.testing.updateOne( { a: NumberInt("7") }, { $set: { a: NumberInt("7"), j1:null, j2:null, j3:"$.equipment.rings[*]", j4:null, j5:null } } ); 92 | use mycluster_mydb_regression_slot; 93 | db.testing.deleteOne( { a: NumberInt("7") } ); 94 | use mycluster_mydb_regression_slot; 95 | db.testing.insertOne( { a: NumberInt("8"), j4:[{"customer": "Cary", "items":{"product":"beer","qty":1000}},{"customer": "John", "items":{"product":"beer","qty":2000}},{"customer": "David", "items":{"product":"beer","qty":3000}}] } ); 96 | use mycluster_mydb_regression_slot; 97 | db.testing.updateOne( { a: NumberInt("8") }, { $set: { a: NumberInt("8"), j1:null, j2:null, j3:null, j4:[{"customer": "Cary"},{"customer": "John"},{"customer": "David"}], j5:null } } ); 98 | use mycluster_mydb_regression_slot; 99 | db.testing.deleteOne( { a: NumberInt("8") } ); 100 | use mycluster_mydb_regression_slot; 101 | db.testing.insertOne( { a: NumberInt("9"), j5:[{"items": {"qty": 1000, "product": "beer"}, "customer": "Cary"},{"items": {"qty": 2000, "product": "beer"}, "customer": "John"},{"items": {"qty": 3000, "product": "beer"}, "customer": "David"}] } ); 102 | use mycluster_mydb_regression_slot; 103 | db.testing.updateOne( { a: NumberInt("9") }, { $set: { a: NumberInt("9"), j1:null, j2:null, j3:null, j4:null, j5:[{"customer": "Cary"},{"customer": "John"},{"customer": "David"}] } } ); 104 | use mycluster_mydb_regression_slot; 105 | db.testing.deleteOne( { a: NumberInt("9") } ); 106 | (36 rows) 107 | 108 | DROP TABLE testing; 109 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 110 | ?column? 111 | ---------- 112 | end 113 | (1 row) 114 | 115 | -------------------------------------------------------------------------------- /expected/numeric.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- create table with different numeric type 11 | CREATE TABLE tbl_int(id serial primary key, a smallint, b integer, c serial, d real, e bigint, f bigserial, g double precision, h decimal, i numeric); 12 | -- minumum values 13 | INSERT INTO tbl_int values(1, -32768, -2147483648, 1, -0.123456, -9223372036854775808, 1, -1.123456789123456, -1234567890.1234567891, -9876543210.0987654321); 14 | UPDATE tbl_int SET a=a+1, b=b+1, c=c+1, d=d+1, e=e+1, f=f+1, g=g+1, h=h+1, i=i+1 WHERE id=1; 15 | DELETE FROM tbl_int WHERE id = 1; 16 | TRUNCATE TABLE tbl_int; 17 | -- maximum values 18 | INSERT INTO tbl_int values(1, 32767, 2147483647, 2147483647, 0.123456, 9223372036854775807, 9223372036854775807, 1.123456789123456, 1234567890.1234567891, 9876543210.0987654321); 19 | UPDATE tbl_int SET a=a-1, b=b-1, c=c-1, d=d-1, e=e-1, f=f-1, g=g-1, h=h-1, i=i-1 WHERE id=1; 20 | DELETE FROM tbl_int WHERE id = 1; 21 | TRUNCATE TABLE tbl_int; 22 | -- peek changes according to action configuration 23 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 24 | data 25 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 26 | use mycluster_mydb_regression_slot; 27 | db.tbl_int.insertOne( { id: NumberInt("1"), a: NumberInt("-32768"), b: NumberInt("-2147483648"), c: NumberInt("1"), d:-0.123456, e: NumberLong("-9223372036854775808"), f: NumberLong("1"), g: NumberDecimal("-1.123456789123456"), h: NumberDecimal("-1234567890.1234567891"), i: NumberDecimal("-9876543210.0987654321") } ); 28 | use mycluster_mydb_regression_slot; 29 | db.tbl_int.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a: NumberInt("-32767"), b: NumberInt("-2147483647"), c: NumberInt("2"), d:0.876544, e: NumberLong("-9223372036854775807"), f: NumberLong("2"), g: NumberDecimal("-0.12345678912345592"), h: NumberDecimal("-1234567889.1234567891"), i: NumberDecimal("-9876543209.0987654321") } } ); 30 | use mycluster_mydb_regression_slot; 31 | db.tbl_int.deleteOne( { id: NumberInt("1") } ); 32 | use mycluster_mydb_regression_slot; 33 | db.tbl_int.insertOne( { id: NumberInt("1"), a: NumberInt("32767"), b: NumberInt("2147483647"), c: NumberInt("2147483647"), d:0.123456, e: NumberLong("9223372036854775807"), f: NumberLong("9223372036854775807"), g: NumberDecimal("1.123456789123456"), h: NumberDecimal("1234567890.1234567891"), i: NumberDecimal("9876543210.0987654321") } ); 34 | use mycluster_mydb_regression_slot; 35 | db.tbl_int.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a: NumberInt("32766"), b: NumberInt("2147483646"), c: NumberInt("2147483646"), d:-0.876544, e: NumberLong("9223372036854775806"), f: NumberLong("9223372036854775806"), g: NumberDecimal("0.12345678912345592"), h: NumberDecimal("1234567889.1234567891"), i: NumberDecimal("9876543209.0987654321") } } ); 36 | use mycluster_mydb_regression_slot; 37 | db.tbl_int.deleteOne( { id: NumberInt("1") } ); 38 | (12 rows) 39 | 40 | -- get changes according to action configuration 41 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 42 | data 43 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 44 | use mycluster_mydb_regression_slot; 45 | db.tbl_int.insertOne( { id: NumberInt("1"), a: NumberInt("-32768"), b: NumberInt("-2147483648"), c: NumberInt("1"), d:-0.123456, e: NumberLong("-9223372036854775808"), f: NumberLong("1"), g: NumberDecimal("-1.123456789123456"), h: NumberDecimal("-1234567890.1234567891"), i: NumberDecimal("-9876543210.0987654321") } ); 46 | use mycluster_mydb_regression_slot; 47 | db.tbl_int.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a: NumberInt("-32767"), b: NumberInt("-2147483647"), c: NumberInt("2"), d:0.876544, e: NumberLong("-9223372036854775807"), f: NumberLong("2"), g: NumberDecimal("-0.12345678912345592"), h: NumberDecimal("-1234567889.1234567891"), i: NumberDecimal("-9876543209.0987654321") } } ); 48 | use mycluster_mydb_regression_slot; 49 | db.tbl_int.deleteOne( { id: NumberInt("1") } ); 50 | use mycluster_mydb_regression_slot; 51 | db.tbl_int.insertOne( { id: NumberInt("1"), a: NumberInt("32767"), b: NumberInt("2147483647"), c: NumberInt("2147483647"), d:0.123456, e: NumberLong("9223372036854775807"), f: NumberLong("9223372036854775807"), g: NumberDecimal("1.123456789123456"), h: NumberDecimal("1234567890.1234567891"), i: NumberDecimal("9876543210.0987654321") } ); 52 | use mycluster_mydb_regression_slot; 53 | db.tbl_int.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a: NumberInt("32766"), b: NumberInt("2147483646"), c: NumberInt("2147483646"), d:-0.876544, e: NumberLong("9223372036854775806"), f: NumberLong("9223372036854775806"), g: NumberDecimal("0.12345678912345592"), h: NumberDecimal("1234567889.1234567891"), i: NumberDecimal("9876543209.0987654321") } } ); 54 | use mycluster_mydb_regression_slot; 55 | db.tbl_int.deleteOne( { id: NumberInt("1") } ); 56 | (12 rows) 57 | 58 | -- drop table 59 | DROP TABLE tbl_int; 60 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 61 | ?column? 62 | ---------- 63 | end 64 | (1 row) 65 | 66 | -------------------------------------------------------------------------------- /expected/pkey.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | CREATE TABLE testing_one_pkey ( 11 | a smallserial PRIMARY KEY, 12 | b smallint, 13 | c int, 14 | d bigint, 15 | e int, 16 | f real, 17 | g double precision, 18 | h char(20), 19 | i varchar(30), 20 | j text 21 | ); 22 | CREATE TABLE testing_multi_pkey ( 23 | a smallserial, 24 | b smallint, 25 | c int, 26 | d bigint, 27 | e int, 28 | f real, 29 | g double precision, 30 | h char(20), 31 | i varchar(30), 32 | j text, 33 | PRIMARY KEY(a, b, c) 34 | ); 35 | CREATE TABLE testing_no_pkey ( 36 | a smallserial, 37 | b smallint, 38 | c int, 39 | d bigint, 40 | e int, 41 | f real, 42 | g double precision, 43 | h char(20), 44 | i varchar(30), 45 | j text 46 | ); 47 | CREATE TABLE testing_unique ( 48 | a smallserial, 49 | b smallint, 50 | c int, 51 | d bigint, 52 | e int, 53 | f real, 54 | g double precision, 55 | h char(20), 56 | i varchar(30), 57 | j text, 58 | UNIQUE(f, g) 59 | ); 60 | INSERT INTO testing_one_pkey (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 61 | INSERT INTO testing_multi_pkey (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 62 | INSERT INTO testing_no_pkey (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 63 | INSERT INTO testing_unique (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 64 | -- non pkey change 65 | UPDATE testing_one_pkey SET f = 777.888888888 WHERE b = 1; 66 | -- pkey change 67 | UPDATE testing_one_pkey SET a = 14 WHERE b = 1; 68 | -- non pkey change 69 | UPDATE testing_multi_pkey SET f = 777.888888888 WHERE b = 1; 70 | -- some pkeys change 71 | UPDATE testing_multi_pkey SET a = 14, c = 14 WHERE b = 1; 72 | -- all pkeys change 73 | UPDATE testing_multi_pkey SET a = 15, b = 15, c = 15 WHERE b = 1; 74 | -- no pkey 75 | UPDATE testing_no_pkey SET h = 'changed' WHERE b = 1; 76 | -- non unique change 77 | UPDATE testing_unique SET h = 'changed' WHERE b = 1; 78 | -- one unique val change 79 | UPDATE testing_unique SET f = 888.888888888 WHERE b = 1; 80 | -- all unique vals change 81 | UPDATE testing_unique SET f = 888.888888888, g = 6.66 WHERE b = 1; 82 | -- get changes 83 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 84 | data 85 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 86 | use mycluster_mydb_regression_slot; 87 | db.testing_one_pkey.insertOne( { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:666.7778, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } ); 88 | use mycluster_mydb_regression_slot; 89 | db.testing_multi_pkey.insertOne( { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:666.7778, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } ); 90 | use mycluster_mydb_regression_slot; 91 | db.testing_no_pkey.insertOne( { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:666.7778, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } ); 92 | use mycluster_mydb_regression_slot; 93 | db.testing_unique.insertOne( { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:666.7778, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } ); 94 | use mycluster_mydb_regression_slot; 95 | db.testing_one_pkey.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:777.8889, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } } ); 96 | use mycluster_mydb_regression_slot; 97 | db.testing_one_pkey.updateOne( { a: NumberInt("1") }, { $set: { a: NumberInt("14"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:777.8889, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } } ); 98 | use mycluster_mydb_regression_slot; 99 | db.testing_multi_pkey.updateOne( { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2") }, { $set: { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:777.8889, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } } ); 100 | use mycluster_mydb_regression_slot; 101 | db.testing_multi_pkey.updateOne( { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2") }, { $set: { a: NumberInt("14"), b: NumberInt("1"), c: NumberInt("14"), d: NumberLong("33"), e: NumberInt("555"), f:777.8889, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } } ); 102 | use mycluster_mydb_regression_slot; 103 | db.testing_multi_pkey.updateOne( { a: NumberInt("14"), b: NumberInt("1"), c: NumberInt("14") }, { $set: { a: NumberInt("15"), b: NumberInt("15"), c: NumberInt("15"), d: NumberLong("33"), e: NumberInt("555"), f:777.8889, g: NumberDecimal("3.33"), h:"testval1 ", i:"testval2", j:"testval3" } } ); 104 | use mycluster_mydb_regression_slot; 105 | db.testing_no_pkey.updateOne({ selector: "null" }, { $set: { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:666.7778, g: NumberDecimal("3.33"), h:"changed ", i:"testval2", j:"testval3" } } ); 106 | use mycluster_mydb_regression_slot; 107 | db.testing_unique.updateOne({ selector: "null" }, { $set: { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:666.7778, g: NumberDecimal("3.33"), h:"changed ", i:"testval2", j:"testval3" } } ); 108 | use mycluster_mydb_regression_slot; 109 | db.testing_unique.updateOne({ selector: "null" }, { $set: { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:888.8889, g: NumberDecimal("3.33"), h:"changed ", i:"testval2", j:"testval3" } } ); 110 | use mycluster_mydb_regression_slot; 111 | db.testing_unique.updateOne({ selector: "null" }, { $set: { a: NumberInt("1"), b: NumberInt("1"), c: NumberInt("2"), d: NumberLong("33"), e: NumberInt("555"), f:888.8889, g: NumberDecimal("6.66"), h:"changed ", i:"testval2", j:"testval3" } } ); 112 | (26 rows) 113 | 114 | DROP TABLE testing_unique; 115 | DROP TABLE testing_no_pkey; 116 | DROP TABLE testing_multi_pkey; 117 | DROP TABLE testing_one_pkey; 118 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 119 | ?column? 120 | ---------- 121 | end 122 | (1 row) 123 | 124 | -------------------------------------------------------------------------------- /expected/specval.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | CREATE TABLE testing ( 11 | a serial PRIMARY KEY, 12 | b varchar(80), 13 | c bool, 14 | d real 15 | ); 16 | BEGIN; 17 | INSERT INTO testing (b, c, d) VALUES( E'valid: '' " \\ / \d \f \r \t \u447F \u967F invalid: \\g \\k end', FALSE, 123.456); 18 | INSERT INTO testing (b, c, d) VALUES('aaa', 't', '+inf'); 19 | INSERT INTO testing (b, c, d) VALUES('aaa', 'f', 'nan'); 20 | INSERT INTO testing (b, c, d) VALUES('null', NULL, '-inf'); 21 | COMMIT; 22 | -- get changes 23 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 24 | data 25 | ------------------------------------------------------------------------------------------------------------------------------------- 26 | use mycluster_mydb_regression_slot; 27 | db.testing.insertOne( { a: NumberInt("1"), b:"valid: '' " \ / d \x0C \r 䑿 陿 invalid: \g \k end", c:false, d:123.456 } ); 28 | use mycluster_mydb_regression_slot; 29 | db.testing.insertOne( { a: NumberInt("2"), b:"aaa", c:true, d:Infinity } ); 30 | use mycluster_mydb_regression_slot; 31 | db.testing.insertOne( { a: NumberInt("3"), b:"aaa", c:false, d:NaN } ); 32 | use mycluster_mydb_regression_slot; 33 | db.testing.insertOne( { a: NumberInt("4"), b:"null", d:-Infinity } ); 34 | (8 rows) 35 | 36 | DROP TABLE testing; 37 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 38 | ?column? 39 | ---------- 40 | end 41 | (1 row) 42 | 43 | -------------------------------------------------------------------------------- /expected/specval_1.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | CREATE TABLE testing ( 11 | a serial PRIMARY KEY, 12 | b varchar(80), 13 | c bool, 14 | d real 15 | ); 16 | BEGIN; 17 | INSERT INTO testing (b, c, d) VALUES( E'valid: '' " \\ / \d \f \r \t \u447F \u967F invalid: \\g \\k end', FALSE, 123.456); 18 | ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'valid: '' " \\ / \d \f \r \t \u447F" at character 39 19 | INSERT INTO testing (b, c, d) VALUES('aaa', 't', '+inf'); 20 | ERROR: current transaction is aborted, commands ignored until end of transaction block 21 | INSERT INTO testing (b, c, d) VALUES('aaa', 'f', 'nan'); 22 | ERROR: current transaction is aborted, commands ignored until end of transaction block 23 | INSERT INTO testing (b, c, d) VALUES('null', NULL, '-inf'); 24 | ERROR: current transaction is aborted, commands ignored until end of transaction block 25 | COMMIT; 26 | -- get changes 27 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 28 | data 29 | ------ 30 | (0 rows) 31 | 32 | DROP TABLE testing; 33 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 34 | ?column? 35 | ---------- 36 | end 37 | (1 row) 38 | 39 | -------------------------------------------------------------------------------- /expected/string.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- network address 11 | CREATE TABLE tbl_net (id serial PRIMARY KEY, a inet, b cidr, c macaddr, d macaddr8); 12 | -- ipv4 in different format 13 | INSERT INTO tbl_net(a,b,c,d) VALUES('192.168.100.128/25'::inet, '192.168.100.128/25'::cidr, '08:00:2b:01:02:03'::macaddr, '08:00:2b:01:02:03:04:05'::macaddr8); 14 | INSERT INTO tbl_net(a,b,c,d) VALUES('192.168.0.1/24'::inet, '192.168/24'::cidr, '08-00-2b-01-02-03'::macaddr, '08-00-2b-01-02-03-04-05'::macaddr8); 15 | INSERT INTO tbl_net(a,b,c,d) VALUES('192.168.0.0/16'::inet, '192.168'::cidr, '08002b:010203'::macaddr, '08002b:0102030405'::macaddr8); 16 | INSERT INTO tbl_net(a,b,c,d) VALUES('10.0.0.1/8'::inet, '10'::cidr, '08002b-010203'::macaddr, '08002b-0102030405'::macaddr8); 17 | INSERT INTO tbl_net(a,b,c,d) VALUES('10.1.2.3/32'::inet, '10.1.2.3/32'::cidr, '0800.2b01.0203'::macaddr, '0800.2b01.0203.0405'::macaddr8); 18 | INSERT INTO tbl_net(a,b,c,d) VALUES('2001:4f8:3:ba::/64'::inet, '2001:4f8:3:ba::/64'::cidr, '0800-2b01-0203'::macaddr, '0800-2b01-0203-0405'::macaddr8); 19 | -- ipv6 and mac in lower case 20 | INSERT INTO tbl_net(a,b,c,d) VALUES('2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128'::inet, '2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128'::cidr, '08002b010203'::macaddr, '08002b01:02030405'::macaddr8); 21 | -- ipv6 and mac in upper case 22 | INSERT INTO tbl_net(a,b,c,d) VALUES('2001:4F8:3:BA:2E0:81FF:FE22:D1F1/128'::inet, '2001:4F8:3:BA:2E0:81FF:FE22:D1F1/128'::cidr, '08002B010203'::macaddr, '08002B0102030405'::macaddr8); 23 | UPDATE tbl_net SET a='10.1.2.3/32'::inet, b='10.1.2.3/32'::cidr, c='0800-2b01-0203'::macaddr, d='0800-2b01-0203-0405'::macaddr8 WHERE id=1; 24 | DELETE FROM tbl_net WHERE id = 1; 25 | TRUNCATE TABLE tbl_net; 26 | -- peek changes 27 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 28 | data 29 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 30 | use mycluster_mydb_regression_slot; 31 | db.tbl_net.insertOne( { id: NumberInt("1"), a:"192.168.100.128/25", b:"192.168.100.128/25", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 32 | use mycluster_mydb_regression_slot; 33 | db.tbl_net.insertOne( { id: NumberInt("2"), a:"192.168.0.1/24", b:"192.168.0.0/24", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 34 | use mycluster_mydb_regression_slot; 35 | db.tbl_net.insertOne( { id: NumberInt("3"), a:"192.168.0.0/16", b:"192.168.0.0/24", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 36 | use mycluster_mydb_regression_slot; 37 | db.tbl_net.insertOne( { id: NumberInt("4"), a:"10.0.0.1/8", b:"10.0.0.0/8", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 38 | use mycluster_mydb_regression_slot; 39 | db.tbl_net.insertOne( { id: NumberInt("5"), a:"10.1.2.3", b:"10.1.2.3/32", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 40 | use mycluster_mydb_regression_slot; 41 | db.tbl_net.insertOne( { id: NumberInt("6"), a:"2001:4f8:3:ba::/64", b:"2001:4f8:3:ba::/64", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 42 | use mycluster_mydb_regression_slot; 43 | db.tbl_net.insertOne( { id: NumberInt("7"), a:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1", b:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 44 | use mycluster_mydb_regression_slot; 45 | db.tbl_net.insertOne( { id: NumberInt("8"), a:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1", b:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 46 | use mycluster_mydb_regression_slot; 47 | db.tbl_net.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:"10.1.2.3", b:"10.1.2.3/32", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } } ); 48 | use mycluster_mydb_regression_slot; 49 | db.tbl_net.deleteOne( { id: NumberInt("1") } ); 50 | (20 rows) 51 | 52 | -- get changes 53 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 54 | data 55 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 56 | use mycluster_mydb_regression_slot; 57 | db.tbl_net.insertOne( { id: NumberInt("1"), a:"192.168.100.128/25", b:"192.168.100.128/25", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 58 | use mycluster_mydb_regression_slot; 59 | db.tbl_net.insertOne( { id: NumberInt("2"), a:"192.168.0.1/24", b:"192.168.0.0/24", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 60 | use mycluster_mydb_regression_slot; 61 | db.tbl_net.insertOne( { id: NumberInt("3"), a:"192.168.0.0/16", b:"192.168.0.0/24", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 62 | use mycluster_mydb_regression_slot; 63 | db.tbl_net.insertOne( { id: NumberInt("4"), a:"10.0.0.1/8", b:"10.0.0.0/8", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 64 | use mycluster_mydb_regression_slot; 65 | db.tbl_net.insertOne( { id: NumberInt("5"), a:"10.1.2.3", b:"10.1.2.3/32", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 66 | use mycluster_mydb_regression_slot; 67 | db.tbl_net.insertOne( { id: NumberInt("6"), a:"2001:4f8:3:ba::/64", b:"2001:4f8:3:ba::/64", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 68 | use mycluster_mydb_regression_slot; 69 | db.tbl_net.insertOne( { id: NumberInt("7"), a:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1", b:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 70 | use mycluster_mydb_regression_slot; 71 | db.tbl_net.insertOne( { id: NumberInt("8"), a:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1", b:"2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } ); 72 | use mycluster_mydb_regression_slot; 73 | db.tbl_net.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:"10.1.2.3", b:"10.1.2.3/32", c:"08:00:2b:01:02:03", d:"08:00:2b:01:02:03:04:05" } } ); 74 | use mycluster_mydb_regression_slot; 75 | db.tbl_net.deleteOne( { id: NumberInt("1") } ); 76 | (20 rows) 77 | 78 | -- drop table 79 | DROP TABLE tbl_net; 80 | -- geo data 81 | CREATE TABLE tbl_geo (id serial PRIMARY KEY, a point, b line, c lseg, d box, e path, f path, g polygon, h circle); 82 | ALTER TABLE tbl_geo REPLICA IDENTITY FULL; 83 | -- geo different data types 84 | INSERT INTO tbl_geo(a,b,c,d,e,f,g,h) VALUES('(1,1)'::point, '[(1,1),(2,2)]'::line, '[(3,3), (4,4)]'::lseg, '((1,1),(2,2))'::box, '[(1,1),(2,2),(3,3)]'::path, '((1,1),(2,2),(3,3))'::path, '((1,1),(2,2),(3,3),(4,4))'::polygon, '<(1,1),5>'::circle); 85 | UPDATE tbl_geo SET a='11,11'::point, b='((11,11),(12,12))'::line, c='((13,13), (14,14))'::lseg, d='(11,11),(12,12)'::box, e='(11,11),(12,12),(13,13)'::path, f='(11,11,12,12,13,13)'::path, g='(11,11),(12,12),(13,13),(14,14)'::polygon, h='((11,11),15)'::circle WHERE id=1; 86 | INSERT INTO tbl_geo(a,b,c,d,e,f,g,h) VALUES('(1,1)'::point, '(1,1),(2,2)'::line, '(3,3), (4,4)'::lseg, '1,1,2,2'::box, '1,1,2,2,3,3'::path, '1,1,2,2,3,3'::path, '(1,1,2,2,3,3,4,4)'::polygon, '1,1,5'::circle); 87 | UPDATE tbl_geo SET a='21,21'::point, b='21,21,22,22'::line, c='23,23, 24,24'::lseg, d='21,21,22,22'::box, e='(21,21,22,22,23,23)'::path, f='21,21,22,22,23,23'::path, g='21,21,22,22,23,23,24,24'::polygon, h='21,21,25'::circle WHERE id=2; 88 | DELETE FROM tbl_geo WHERE id = 1; 89 | DELETE FROM tbl_geo WHERE id = 2; 90 | TRUNCATE TABLE tbl_geo; 91 | -- peek changes 92 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 93 | data 94 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 95 | use mycluster_mydb_regression_slot; 96 | db.tbl_geo.insertOne( { id: NumberInt("1"), a:"(1,1)", b:"{1,-1,0}", c:"[(3,3),(4,4)]", d:"(2,2),(1,1)", e:"[(1,1),(2,2),(3,3)]", f:"((1,1),(2,2),(3,3))", g:"((1,1),(2,2),(3,3),(4,4))", h:"<(1,1),5>" } ); 97 | use mycluster_mydb_regression_slot; 98 | db.tbl_geo.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:"(11,11)", b:"{1,-1,0}", c:"[(13,13),(14,14)]", d:"(12,12),(11,11)", e:"((11,11),(12,12),(13,13))", f:"((11,11),(12,12),(13,13))", g:"((11,11),(12,12),(13,13),(14,14))", h:"<(11,11),15>" } } ); 99 | use mycluster_mydb_regression_slot; 100 | db.tbl_geo.insertOne( { id: NumberInt("2"), a:"(1,1)", b:"{1,-1,0}", c:"[(3,3),(4,4)]", d:"(2,2),(1,1)", e:"((1,1),(2,2),(3,3))", f:"((1,1),(2,2),(3,3))", g:"((1,1),(2,2),(3,3),(4,4))", h:"<(1,1),5>" } ); 101 | use mycluster_mydb_regression_slot; 102 | db.tbl_geo.updateOne( { id: NumberInt("2") }, { $set: { id: NumberInt("2"), a:"(21,21)", b:"{1,-1,0}", c:"[(23,23),(24,24)]", d:"(22,22),(21,21)", e:"((21,21),(22,22),(23,23))", f:"((21,21),(22,22),(23,23))", g:"((21,21),(22,22),(23,23),(24,24))", h:"<(21,21),25>" } } ); 103 | use mycluster_mydb_regression_slot; 104 | db.tbl_geo.deleteOne( { id: NumberInt("1") } ); 105 | use mycluster_mydb_regression_slot; 106 | db.tbl_geo.deleteOne( { id: NumberInt("2") } ); 107 | (12 rows) 108 | 109 | -- get changes 110 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 111 | data 112 | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 113 | use mycluster_mydb_regression_slot; 114 | db.tbl_geo.insertOne( { id: NumberInt("1"), a:"(1,1)", b:"{1,-1,0}", c:"[(3,3),(4,4)]", d:"(2,2),(1,1)", e:"[(1,1),(2,2),(3,3)]", f:"((1,1),(2,2),(3,3))", g:"((1,1),(2,2),(3,3),(4,4))", h:"<(1,1),5>" } ); 115 | use mycluster_mydb_regression_slot; 116 | db.tbl_geo.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:"(11,11)", b:"{1,-1,0}", c:"[(13,13),(14,14)]", d:"(12,12),(11,11)", e:"((11,11),(12,12),(13,13))", f:"((11,11),(12,12),(13,13))", g:"((11,11),(12,12),(13,13),(14,14))", h:"<(11,11),15>" } } ); 117 | use mycluster_mydb_regression_slot; 118 | db.tbl_geo.insertOne( { id: NumberInt("2"), a:"(1,1)", b:"{1,-1,0}", c:"[(3,3),(4,4)]", d:"(2,2),(1,1)", e:"((1,1),(2,2),(3,3))", f:"((1,1),(2,2),(3,3))", g:"((1,1),(2,2),(3,3),(4,4))", h:"<(1,1),5>" } ); 119 | use mycluster_mydb_regression_slot; 120 | db.tbl_geo.updateOne( { id: NumberInt("2") }, { $set: { id: NumberInt("2"), a:"(21,21)", b:"{1,-1,0}", c:"[(23,23),(24,24)]", d:"(22,22),(21,21)", e:"((21,21),(22,22),(23,23))", f:"((21,21),(22,22),(23,23))", g:"((21,21),(22,22),(23,23),(24,24))", h:"<(21,21),25>" } } ); 121 | use mycluster_mydb_regression_slot; 122 | db.tbl_geo.deleteOne( { id: NumberInt("1") } ); 123 | use mycluster_mydb_regression_slot; 124 | db.tbl_geo.deleteOne( { id: NumberInt("2") } ); 125 | (12 rows) 126 | 127 | -- drop table 128 | DROP TABLE tbl_geo; 129 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 130 | ?column? 131 | ---------- 132 | end 133 | (1 row) 134 | 135 | -------------------------------------------------------------------------------- /expected/timestamptz.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- timestamptz 11 | CREATE TABLE tbl_tstz ( id serial PRIMARY KEY, tstz TIMESTAMPTZ NOT NULL ); 12 | INSERT INTO tbl_tstz (tstz) VALUES('2020-03-24 16:00:30-07') ; 13 | UPDATE tbl_tstz SET tstz = '2020-03-24 16:30:00-07' WHERE id=1; 14 | DELETE FROM tbl_tstz WHERE id = 1; 15 | TRUNCATE TABLE tbl_tstz; 16 | -- peek changes according to action configuration 17 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 18 | data 19 | --------------------------------------------------------------------------------------------------------------------------------- 20 | use mycluster_mydb_regression_slot; 21 | db.tbl_tstz.insertOne( { id: NumberInt("1"), tstz: ISODate("2020-03-24T16:00:30-07:00") } ); 22 | use mycluster_mydb_regression_slot; 23 | db.tbl_tstz.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), tstz: ISODate("2020-03-24T16:30:00-07:00") } } ); 24 | use mycluster_mydb_regression_slot; 25 | db.tbl_tstz.deleteOne( { id: NumberInt("1") } ); 26 | (6 rows) 27 | 28 | -- get changes according to action configuration 29 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 30 | data 31 | --------------------------------------------------------------------------------------------------------------------------------- 32 | use mycluster_mydb_regression_slot; 33 | db.tbl_tstz.insertOne( { id: NumberInt("1"), tstz: ISODate("2020-03-24T16:00:30-07:00") } ); 34 | use mycluster_mydb_regression_slot; 35 | db.tbl_tstz.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), tstz: ISODate("2020-03-24T16:30:00-07:00") } } ); 36 | use mycluster_mydb_regression_slot; 37 | db.tbl_tstz.deleteOne( { id: NumberInt("1") } ); 38 | (6 rows) 39 | 40 | -- drop tables 41 | DROP TABLE tbl_tstz; 42 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 43 | ?column? 44 | ---------- 45 | end 46 | (1 row) 47 | 48 | -------------------------------------------------------------------------------- /expected/transaction.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- actions 11 | CREATE TABLE testing (a varchar(30) primary key); 12 | INSERT INTO testing (a) VALUES('cary'); 13 | INSERT INTO testing (a) VALUES('john'); 14 | INSERT INTO testing (a) VALUES('peter'); 15 | -- define a transaction containing sub transactions 16 | BEGIN; 17 | INSERT INTO testing (a) VALUES('david'); 18 | SAVEPOINT p1; 19 | INSERT INTO testing (a) VALUES('grant'); 20 | SAVEPOINT p2; 21 | INSERT INTO testing (a) VALUES('mike'); 22 | SAVEPOINT p3; 23 | INSERT INTO testing (a) VALUES('allen'); 24 | SAVEPOINT p4; 25 | INSERT INTO testing (a) VALUES('dan'); 26 | ROLLBACK TO SAVEPOINT p3; 27 | RELEASE SAVEPOINT p1; 28 | INSERT INTO testing (a) VALUES('cheese'); 29 | COMMIT; 30 | -- peek and get changes with and without transaction mode 31 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'use_transaction', 'true', 'regress', 'true'); 32 | data 33 | --------------------------------------------------------- 34 | session0regression_slot = db.getMongo().startSession(); 35 | session0regression_slot.startTransaction(); 36 | session0regression_slot.commitTransaction(); 37 | session0regression_slot.endSession(); 38 | session0regression_slot = db.getMongo().startSession(); 39 | session0regression_slot.startTransaction(); 40 | use mycluster_mydb_regression_slot; 41 | db.testing.insertOne( { a:"cary" } ); 42 | session0regression_slot.commitTransaction(); 43 | session0regression_slot.endSession(); 44 | session0regression_slot = db.getMongo().startSession(); 45 | session0regression_slot.startTransaction(); 46 | use mycluster_mydb_regression_slot; 47 | db.testing.insertOne( { a:"john" } ); 48 | session0regression_slot.commitTransaction(); 49 | session0regression_slot.endSession(); 50 | session0regression_slot = db.getMongo().startSession(); 51 | session0regression_slot.startTransaction(); 52 | use mycluster_mydb_regression_slot; 53 | db.testing.insertOne( { a:"peter" } ); 54 | session0regression_slot.commitTransaction(); 55 | session0regression_slot.endSession(); 56 | session0regression_slot = db.getMongo().startSession(); 57 | session0regression_slot.startTransaction(); 58 | use mycluster_mydb_regression_slot; 59 | db.testing.insertOne( { a:"david" } ); 60 | use mycluster_mydb_regression_slot; 61 | db.testing.insertOne( { a:"grant" } ); 62 | use mycluster_mydb_regression_slot; 63 | db.testing.insertOne( { a:"mike" } ); 64 | use mycluster_mydb_regression_slot; 65 | db.testing.insertOne( { a:"cheese" } ); 66 | session0regression_slot.commitTransaction(); 67 | session0regression_slot.endSession(); 68 | (34 rows) 69 | 70 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'use_transaction', 'false', 'regress', 'true', 'skip_empty_xacts', 'false'); 71 | data 72 | ----------------------------------------- 73 | use mycluster_mydb_regression_slot; 74 | db.testing.insertOne( { a:"cary" } ); 75 | use mycluster_mydb_regression_slot; 76 | db.testing.insertOne( { a:"john" } ); 77 | use mycluster_mydb_regression_slot; 78 | db.testing.insertOne( { a:"peter" } ); 79 | use mycluster_mydb_regression_slot; 80 | db.testing.insertOne( { a:"david" } ); 81 | use mycluster_mydb_regression_slot; 82 | db.testing.insertOne( { a:"grant" } ); 83 | use mycluster_mydb_regression_slot; 84 | db.testing.insertOne( { a:"mike" } ); 85 | use mycluster_mydb_regression_slot; 86 | db.testing.insertOne( { a:"cheese" } ); 87 | (14 rows) 88 | 89 | DROP TABLE testing; 90 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 91 | ?column? 92 | ---------- 93 | end 94 | (1 row) 95 | 96 | -------------------------------------------------------------------------------- /expected/xml.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- xml data 11 | CREATE TABLE tbl_xml (id serial primary key, a xml); 12 | -- insert data simple SQL way 13 | INSERT INTO tbl_xml(a) VALUES(' xml-data '::xml); 14 | -- using XMLPARSE and DOCUMENT syntaxes 15 | INSERT INTO tbl_xml(a) VALUES(XMLPARSE (DOCUMENT' xml-data ')); 16 | -- xpath example simple SQL way 17 | INSERT INTO tbl_xml(a) VALUES(' 18 | 20 | IT 21 | 22 | 23 | John Smith 24 | 24 25 | 26 | 27 | Michael Black 28 | 28 29 | 30 | 31 | '::xml); 32 | -- xpath example in minify way 33 | INSERT INTO tbl_xml(a) VALUES('ITJohn Smith24Michael Black28'::xml); 34 | -- xpath example using XMLPARSE and DOCUMENT 35 | INSERT INTO tbl_xml(a) VALUES(XMLPARSE (DOCUMENT' 36 | 38 | IT 39 | 40 | 41 | John Smith 42 | 24 43 | 44 | 45 | Michael Black 46 | 28 47 | 48 | 49 | ')); 50 | -- update using simple SQL way 51 | UPDATE tbl_xml SET a = ' 52 | 54 | IT 55 | 56 | 57 | John Smith 58 | 24 59 | 60 | 61 | Michael Black 62 | 28 63 | 64 | 65 | '::xml WHERE id=1; 66 | -- update using XMLPARSE and DOCUMENT syntaxes 67 | UPDATE tbl_xml SET a = 68 | XMLPARSE (DOCUMENT' 69 | 71 | IT 72 | 73 | 74 | John Smith 75 | 24 76 | 77 | 78 | Michael Black 79 | 28 80 | 81 | 82 | ') WHERE id=2; 83 | -- delete 84 | DELETE FROM tbl_xml WHERE id = 1; 85 | TRUNCATE TABLE tbl_xml; 86 | -- peek changes 87 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 88 | data 89 | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 90 | use mycluster_mydb_regression_slot; 91 | db.tbl_xml.insertOne( { id: NumberInt("1"), a:" xml-data " } ); 92 | use mycluster_mydb_regression_slot; 93 | db.tbl_xml.insertOne( { id: NumberInt("2"), a:" xml-data " } ); 94 | use mycluster_mydb_regression_slot; 95 | db.tbl_xml.insertOne( { id: NumberInt("3"), a:" IT John Smith 24 Michael Black 28 " } ); 96 | use mycluster_mydb_regression_slot; 97 | db.tbl_xml.insertOne( { id: NumberInt("4"), a:"ITJohn Smith24Michael Black28" } ); 98 | use mycluster_mydb_regression_slot; 99 | db.tbl_xml.insertOne( { id: NumberInt("5"), a:" IT John Smith 24 Michael Black 28 " } ); 100 | use mycluster_mydb_regression_slot; 101 | db.tbl_xml.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:" IT John Smith 24 Michael Black 28 " } } ); 102 | use mycluster_mydb_regression_slot; 103 | db.tbl_xml.updateOne( { id: NumberInt("2") }, { $set: { id: NumberInt("2"), a:" IT John Smith 24 Michael Black 28 " } } ); 104 | use mycluster_mydb_regression_slot; 105 | db.tbl_xml.deleteOne( { id: NumberInt("1") } ); 106 | (16 rows) 107 | 108 | -- get changes 109 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 110 | data 111 | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 112 | use mycluster_mydb_regression_slot; 113 | db.tbl_xml.insertOne( { id: NumberInt("1"), a:" xml-data " } ); 114 | use mycluster_mydb_regression_slot; 115 | db.tbl_xml.insertOne( { id: NumberInt("2"), a:" xml-data " } ); 116 | use mycluster_mydb_regression_slot; 117 | db.tbl_xml.insertOne( { id: NumberInt("3"), a:" IT John Smith 24 Michael Black 28 " } ); 118 | use mycluster_mydb_regression_slot; 119 | db.tbl_xml.insertOne( { id: NumberInt("4"), a:"ITJohn Smith24Michael Black28" } ); 120 | use mycluster_mydb_regression_slot; 121 | db.tbl_xml.insertOne( { id: NumberInt("5"), a:" IT John Smith 24 Michael Black 28 " } ); 122 | use mycluster_mydb_regression_slot; 123 | db.tbl_xml.updateOne( { id: NumberInt("1") }, { $set: { id: NumberInt("1"), a:" IT John Smith 24 Michael Black 28 " } } ); 124 | use mycluster_mydb_regression_slot; 125 | db.tbl_xml.updateOne( { id: NumberInt("2") }, { $set: { id: NumberInt("2"), a:" IT John Smith 24 Michael Black 28 " } } ); 126 | use mycluster_mydb_regression_slot; 127 | db.tbl_xml.deleteOne( { id: NumberInt("1") } ); 128 | (16 rows) 129 | 130 | -- drop table 131 | DROP TABLE tbl_xml; 132 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 133 | ?column? 134 | ---------- 135 | end 136 | (1 row) 137 | 138 | -------------------------------------------------------------------------------- /expected/xml_1.out: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | -- predictability 3 | SET synchronous_commit = on; 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | ?column? 6 | ---------- 7 | init 8 | (1 row) 9 | 10 | -- xml data 11 | CREATE TABLE tbl_xml (id serial primary key, a xml); 12 | -- insert data simple SQL way 13 | INSERT INTO tbl_xml(a) VALUES(' xml-data '::xml); 14 | ERROR: unsupported XML feature at character 31 15 | -- using XMLPARSE and DOCUMENT syntaxes 16 | INSERT INTO tbl_xml(a) VALUES(XMLPARSE (DOCUMENT' xml-data ')); 17 | ERROR: unsupported XML feature 18 | -- xpath example simple SQL way 19 | INSERT INTO tbl_xml(a) VALUES(' 20 | 22 | IT 23 | 24 | 25 | John Smith 26 | 24 27 | 28 | 29 | Michael Black 30 | 28 31 | 32 | 33 | '::xml); 34 | ERROR: unsupported XML feature at character 31 35 | -- xpath example in minify way 36 | INSERT INTO tbl_xml(a) VALUES('ITJohn Smith24Michael Black28'::xml); 37 | ERROR: unsupported XML feature at character 31 38 | -- xpath example using XMLPARSE and DOCUMENT 39 | INSERT INTO tbl_xml(a) VALUES(XMLPARSE (DOCUMENT' 40 | 42 | IT 43 | 44 | 45 | John Smith 46 | 24 47 | 48 | 49 | Michael Black 50 | 28 51 | 52 | 53 | ')); 54 | ERROR: unsupported XML feature 55 | -- update using simple SQL way 56 | UPDATE tbl_xml SET a = ' 57 | 59 | IT 60 | 61 | 62 | John Smith 63 | 24 64 | 65 | 66 | Michael Black 67 | 28 68 | 69 | 70 | '::xml WHERE id=1; 71 | ERROR: unsupported XML feature at character 24 72 | -- update using XMLPARSE and DOCUMENT syntaxes 73 | UPDATE tbl_xml SET a = 74 | XMLPARSE (DOCUMENT' 75 | 77 | IT 78 | 79 | 80 | John Smith 81 | 24 82 | 83 | 84 | Michael Black 85 | 28 86 | 87 | 88 | ') WHERE id=2; 89 | -- delete 90 | DELETE FROM tbl_xml WHERE id = 1; 91 | TRUNCATE TABLE tbl_xml; 92 | -- peek changes 93 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 94 | data 95 | ------ 96 | (0 rows) 97 | 98 | -- get changes 99 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 100 | data 101 | ------ 102 | (0 rows) 103 | 104 | -- drop table 105 | DROP TABLE tbl_xml; 106 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 107 | ?column? 108 | ---------- 109 | end 110 | (1 row) 111 | 112 | -------------------------------------------------------------------------------- /logical.conf: -------------------------------------------------------------------------------- 1 | wal_level = logical 2 | max_replication_slots = 4 3 | -------------------------------------------------------------------------------- /sql/actions.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- actions 9 | CREATE TABLE testing (a integer primary key); 10 | INSERT INTO testing (a) VALUES(200); 11 | UPDATE testing SET a = 500 WHERE a = 200; 12 | DELETE FROM testing WHERE a = 500; 13 | TRUNCATE TABLE testing; 14 | 15 | -- peek changes according to action configuration 16 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'insert', 'regress', 'true'); 17 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'update', 'regress', 'true'); 18 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'delete', 'regress', 'true'); 19 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'truncate', 'regress', 'true'); 20 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'insert, update, delete, truncate', 'regress', 'true'); 21 | 22 | -- peek changes with default action configuraiton 23 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 24 | 25 | -- peek changes with several configuration parameter combinations 26 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include_cluster_name', 'true', 'regress', 'true'); 27 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'skip_empty_xacts', 'true', 'regress', 'true'); 28 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'only_local', 'true', 'regress', 'true'); 29 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'use_transaction', 'true', 'regress', 'true'); 30 | 31 | -- peek changes with invalid actions 32 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'actions', 'insert, xxx, delete, xxx', 'regress', 'true'); 33 | 34 | DROP TABLE testing; 35 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 36 | -------------------------------------------------------------------------------- /sql/array.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- create table with different numeric type 9 | CREATE TABLE tbl_array(id serial primary key, a bool[], c char[], d name[], e int2[], f int4[], g text[], h varchar[], i int8[], j float4[], k float8[], l timestamptz[], m numeric[], n uuid[] ); 10 | 11 | -- different data types for array 12 | INSERT INTO tbl_array (a, c, d, e, f, g, h, i, j, k, l, m, n) VALUES( 13 | ARRAY[true, false], 14 | ARRAY['c'::char,'h'::char], 15 | ARRAY['student'::name, 'teacher'::name], 16 | ARRAY['123'::int2, '456'::int2], 17 | ARRAY['123456789'::int4,'987654321'::int4], 18 | ARRAY['abc'::text, '123'::text], 19 | ARRAY['ABCD'::varchar, '1234'::varchar], 20 | ARRAY['112233445566778899'::int8, '998877665544332211'::int8], 21 | ARRAY['123.456'::float4, '2222.3333'::float4], 22 | ARRAY['123456.123'::float8, '654321.123'::float8], 23 | ARRAY['2020-03-30 10:18:40.12-07'::timestamptz, '2020-03-30 20:28:40.12-07'::timestamptz], 24 | ARRAY['123456789'::numeric, '987654321'::numeric], 25 | ARRAY['40e6215d-b5c6-4896-987c-f30f3678f608'::uuid, '3f333df6-90a4-4fda-8dd3-9485d27cee36'::uuid] 26 | ); 27 | UPDATE tbl_array SET 28 | a=ARRAY[false, true], 29 | c=ARRAY['h'::char, 'c'::char], 30 | d=ARRAY['teacher'::name, 'student'::name], 31 | e=ARRAY['456'::int2, '123'::int2], 32 | f=ARRAY['987654321'::int4, '123456789'::int4], 33 | g=ARRAY['123'::text, 'abc'::text], 34 | h=ARRAY['1234'::varchar, 'ABCD'::varchar], 35 | i=ARRAY['998877665544332211'::int8, '112233445566778899'::int8], 36 | j=ARRAY['2222.3333'::float4, '123.456'::float4], 37 | k=ARRAY['654321.123'::float8, '123456.123'::float8], 38 | l=ARRAY['2020-03-30 20:28:40.12-07'::timestamptz, '2020-03-30 10:18:40.12-07'::timestamptz], 39 | m=ARRAY['987654321'::numeric, '123456789'::numeric], 40 | n=ARRAY['3f333df6-90a4-4fda-8dd3-9485d27cee36'::uuid, '40e6215d-b5c6-4896-987c-f30f3678f608'::uuid] 41 | WHERE id=1; 42 | DELETE FROM tbl_array WHERE id = 1; 43 | TRUNCATE TABLE tbl_array; 44 | 45 | 46 | -- peek changes according to action configuration 47 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 48 | 49 | -- get changes according to action configuration 50 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 51 | 52 | -- drop table 53 | DROP TABLE tbl_array; 54 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 55 | -------------------------------------------------------------------------------- /sql/binData.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- UUID 9 | CREATE TABLE tbl_uuid ( id serial PRIMARY KEY, a_uuid UUID NOT NULL ); 10 | INSERT INTO tbl_uuid (a_uuid) VALUES('47deacb1-3ad0-4a0e-8254-9ad3f589c9f3') ; 11 | UPDATE tbl_uuid SET a_uuid = 'e7d8e462-12cc-49dc-aac2-2b5dccdabeda' WHERE id=1; 12 | DELETE FROM tbl_uuid WHERE id = 1; 13 | TRUNCATE TABLE tbl_uuid; 14 | 15 | -- BYTEA 16 | CREATE TABLE tbl_bytea ( id serial PRIMARY KEY, a_bytea bytea NOT NULL ); 17 | ALTER TABLE tbl_bytea REPLICA IDENTITY FULL; 18 | INSERT INTO tbl_bytea(a_bytea) SELECT 'abc \153\154\155 \052\251\124'::bytea RETURNING a_bytea; 19 | UPDATE tbl_bytea SET a_bytea = SELECT '\134'::bytea RETURNING a_bytea; 20 | DELETE FROM tbl_bytea WHERE id = 1; 21 | TRUNCATE TABLE tbl_bytea; 22 | 23 | -- peek changes according to action configuration 24 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 25 | 26 | -- get changes according to action configuration 27 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 28 | 29 | 30 | -- drop tables 31 | DROP TABLE tbl_uuid; 32 | DROP TABLE tbl_bytea; 33 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 34 | -------------------------------------------------------------------------------- /sql/binary.sql: -------------------------------------------------------------------------------- 1 | -- predictability 2 | SET synchronous_commit = on; 3 | 4 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 5 | -- succeeds, textual plugin, textual consumer 6 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'force_binary', '0'); 7 | -- fails, binary plugin, textual consumer 8 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'force_binary', '1'); 9 | -- succeeds, textual plugin, binary consumer 10 | SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot', NULL, NULL, 'force_binary', '0'); 11 | -- succeeds, binary plugin, binary consumer 12 | SELECT data FROM pg_logical_slot_get_binary_changes('regression_slot', NULL, NULL, 'force_binary', '1'); 13 | 14 | SELECT 'init' FROM pg_drop_replication_slot('regression_slot'); 15 | -------------------------------------------------------------------------------- /sql/boolean.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- create table with boolean type 9 | CREATE TABLE tbl_boolean (a integer primary key, b BOOLEAN); 10 | 11 | -- true 12 | INSERT INTO tbl_boolean (a, b) VALUES(1, true); 13 | UPDATE tbl_boolean SET b = false WHERE a = 1; 14 | DELETE FROM tbl_boolean WHERE a = 1; 15 | TRUNCATE TABLE tbl_boolean; 16 | 17 | -- 'true' 18 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'true'); 19 | UPDATE tbl_boolean SET b = 'false' WHERE a = 1; 20 | DELETE FROM tbl_boolean WHERE a = 1; 21 | TRUNCATE TABLE tbl_boolean; 22 | 23 | -- 't' 24 | INSERT INTO tbl_boolean (a, b) VALUES(1, 't'); 25 | UPDATE tbl_boolean SET b = 'f' WHERE a = 1; 26 | DELETE FROM tbl_boolean WHERE a = 1; 27 | TRUNCATE TABLE tbl_boolean; 28 | 29 | -- TRUE 30 | INSERT INTO tbl_boolean (a, b) VALUES(1, TRUE); 31 | UPDATE tbl_boolean SET b = TRUE WHERE a = 1; 32 | DELETE FROM tbl_boolean WHERE a = 1; 33 | TRUNCATE TABLE tbl_boolean; 34 | 35 | -- 'TRUE' 36 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'TRUE'); 37 | UPDATE tbl_boolean SET b = 'TRUE' WHERE a = 1; 38 | DELETE FROM tbl_boolean WHERE a = 1; 39 | TRUNCATE TABLE tbl_boolean; 40 | 41 | -- 'T' 42 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'T'); 43 | UPDATE tbl_boolean SET b = 'T' WHERE a = 1; 44 | DELETE FROM tbl_boolean WHERE a = 1; 45 | TRUNCATE TABLE tbl_boolean; 46 | 47 | -- 'on' 48 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'on'); 49 | UPDATE tbl_boolean SET b = 'off' WHERE a = 1; 50 | DELETE FROM tbl_boolean WHERE a = 1; 51 | TRUNCATE TABLE tbl_boolean; 52 | 53 | -- '1' 54 | INSERT INTO tbl_boolean (a, b) VALUES(1, '1'); 55 | UPDATE tbl_boolean SET b = '0' WHERE a = 1; 56 | DELETE FROM tbl_boolean WHERE a = 1; 57 | TRUNCATE TABLE tbl_boolean; 58 | 59 | -- false 60 | INSERT INTO tbl_boolean (a, b) VALUES(1, false); 61 | UPDATE tbl_boolean SET b = true WHERE a = 1; 62 | DELETE FROM tbl_boolean WHERE a = 1; 63 | TRUNCATE TABLE tbl_boolean; 64 | 65 | -- 'false' 66 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'false'); 67 | UPDATE tbl_boolean SET b = 'true' WHERE a = 1; 68 | DELETE FROM tbl_boolean WHERE a = 1; 69 | TRUNCATE TABLE tbl_boolean; 70 | 71 | -- 'f' 72 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'f'); 73 | UPDATE tbl_boolean SET b = 't' WHERE a = 1; 74 | DELETE FROM tbl_boolean WHERE a = 1; 75 | TRUNCATE TABLE tbl_boolean; 76 | 77 | -- FALSE 78 | INSERT INTO tbl_boolean (a, b) VALUES(1, FALSE); 79 | UPDATE tbl_boolean SET b = TRUE WHERE a = 1; 80 | DELETE FROM tbl_boolean WHERE a = 1; 81 | TRUNCATE TABLE tbl_boolean; 82 | 83 | -- 'FALSE' 84 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'FALSE'); 85 | UPDATE tbl_boolean SET b = 'TRUE' WHERE a = 1; 86 | DELETE FROM tbl_boolean WHERE a = 1; 87 | TRUNCATE TABLE tbl_boolean; 88 | 89 | -- 'F' 90 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'F'); 91 | UPDATE tbl_boolean SET b = 'T' WHERE a = 1; 92 | DELETE FROM tbl_boolean WHERE a = 1; 93 | TRUNCATE TABLE tbl_boolean; 94 | 95 | -- 'off' 96 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'off'); 97 | UPDATE tbl_boolean SET b = 'on' WHERE a = 1; 98 | DELETE FROM tbl_boolean WHERE a = 1; 99 | TRUNCATE TABLE tbl_boolean; 100 | 101 | -- '0' 102 | INSERT INTO tbl_boolean (a, b) VALUES(1, '0'); 103 | UPDATE tbl_boolean SET b = '1' WHERE a = 1; 104 | DELETE FROM tbl_boolean WHERE a = 1; 105 | TRUNCATE TABLE tbl_boolean; 106 | 107 | 108 | -- 'on' exception 109 | INSERT INTO tbl_boolean (a, b) VALUES(1, on); 110 | UPDATE tbl_boolean SET b = off WHERE a = 1; 111 | DELETE FROM tbl_boolean WHERE a = 1; 112 | TRUNCATE TABLE tbl_boolean; 113 | 114 | -- 'enable' exception 115 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'enable'); 116 | UPDATE tbl_boolean SET b = 'disable' WHERE a = 1; 117 | DELETE FROM tbl_boolean WHERE a = 1; 118 | TRUNCATE TABLE tbl_boolean; 119 | 120 | -- 'disable' exception 121 | INSERT INTO tbl_boolean (a, b) VALUES(1, 'disable'); 122 | UPDATE tbl_boolean SET b = 'enable' WHERE a = 1; 123 | DELETE FROM tbl_boolean WHERE a = 1; 124 | TRUNCATE TABLE tbl_boolean; 125 | 126 | 127 | -- peek changes according to action configuration 128 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 129 | 130 | -- get changes according to action configuration 131 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 132 | 133 | -- peek changes with invalid actions 134 | 135 | DROP TABLE tbl_boolean; 136 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 137 | -------------------------------------------------------------------------------- /sql/identity.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- Replica identity nothing 9 | CREATE TABLE testing (a varchar(30) primary key, b INT, c INT); 10 | ALTER TABLE testing REPLICA IDENTITY NOTHING; 11 | BEGIN; 12 | INSERT INTO testing VALUES('aaa', 123, 456); 13 | UPDATE testing set b = 789 where a = 'aaa'; 14 | UPDATE testing set a = 'bbb'; 15 | DELETE from testing where a = 'bbb'; 16 | COMMIT; 17 | 18 | -- Replica identity default 19 | BEGIN; 20 | ALTER TABLE testing REPLICA IDENTITY DEFAULT; 21 | INSERT INTO testing VALUES('aaa', 123, 456); 22 | UPDATE testing set b = 789 where a = 'aaa'; 23 | UPDATE testing set a = 'bbb'; 24 | DELETE from testing where a = 'bbb'; 25 | COMMIT; 26 | 27 | -- Replica identity full 28 | BEGIN; 29 | ALTER TABLE testing REPLICA IDENTITY FULL; 30 | INSERT INTO testing VALUES('aaa', 123, 456); 31 | UPDATE testing set b = 789 where a = 'aaa'; 32 | UPDATE testing set a = 'bbb'; 33 | DELETE from testing where a = 'bbb'; 34 | COMMIT; 35 | 36 | -- Replica identity index 37 | CREATE INDEX idx_test ON testing(b); 38 | ALTER TABLE testing REPLICA IDENTITY INDEX; 39 | INSERT INTO testing VALUES('aaa', 123, 456); 40 | UPDATE testing set b = 789 where a = 'aaa'; 41 | UPDATE testing set a = 'bbb'; 42 | DELETE from testing where a = 'bbb'; 43 | COMMIT; 44 | 45 | -- get changes 46 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 47 | 48 | DROP TABLE testing; 49 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 50 | -------------------------------------------------------------------------------- /sql/json.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | CREATE TABLE testing (a integer primary key, 9 | j1 json, 10 | j2 jsonb, 11 | j3 jsonpath, 12 | j4 json[], 13 | j5 jsonb[]); 14 | 15 | -- JSON data type 16 | INSERT INTO testing (a, j1) VALUES(1, '{"customer": "John", "items":{"product":"beer","qty":20}}'); 17 | INSERT INTO testing (a, j1) VALUES(2, '{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10},{"product":"cooler","qty":10}]}'); 18 | INSERT INTO testing (a, j1) VALUES(3, '{"customer": "Joe", "items":{"product":"wine","qty":20}}'); 19 | UPDATE testing set j1='{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10}]}' where a = 3; 20 | DELETE FROM testing where a = 1; 21 | 22 | -- JSONB data type 23 | INSERT INTO testing (a, j2) VALUES(4, '{"customer": "John", "items":{"product":"beer","qty":20}}'); 24 | INSERT INTO testing (a, j2) VALUES(5, '{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10},{"product":"cooler","qty":10}]}'); 25 | INSERT INTO testing (a, j2) VALUES(6, '{"customer": "Joe", "items":{"product":"wine","qty":20}}'); 26 | UPDATE testing set j2='{"customer": "Michael", "items":[{"product":"vodka","qty":5},{"product":"whiskey","qty":10}]}' where a = 3; 27 | DELETE FROM testing where a = 1; 28 | 29 | -- JSONPATH data type 30 | INSERT INTO testing (a, j3) VALUES(7, '$.equipment.rings[*] ? (@.track.segments > 1)'); 31 | UPDATE testing set j3 = '$.equipment.rings[*]' where a = 7; 32 | DELETE from testing where a = 7; 33 | 34 | -- JSON[] data type 35 | INSERT into testing (a, j4) VALUES 36 | ( 37 | 8, 38 | array[ 39 | '{"customer": "Cary", "items":{"product":"beer","qty":1000}}', 40 | '{"customer": "John", "items":{"product":"beer","qty":2000}}', 41 | '{"customer": "David", "items":{"product":"beer","qty":3000}}' 42 | ]::json[] 43 | ); 44 | UPDATE testing set j4 = 45 | array[ 46 | '{"customer": "Cary"}', 47 | '{"customer": "John"}', 48 | '{"customer": "David"}' 49 | ]::json[] where a = 8; 50 | DELETE FROM testing where a = 8; 51 | 52 | -- JSONB[] data type 53 | INSERT into testing (a, j5) VALUES 54 | ( 55 | 9, 56 | array[ 57 | '{"customer": "Cary", "items":{"product":"beer","qty":1000}}', 58 | '{"customer": "John", "items":{"product":"beer","qty":2000}}', 59 | '{"customer": "David", "items":{"product":"beer","qty":3000}}' 60 | ]::json[] 61 | ); 62 | UPDATE testing set j5 = 63 | array[ 64 | '{"customer": "Cary"}', 65 | '{"customer": "John"}', 66 | '{"customer": "David"}' 67 | ]::json[] where a = 9; 68 | DELETE FROM testing where a = 9; 69 | 70 | -- get the changes 71 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'use_transaction', 'false', 'regress', 'true'); 72 | 73 | DROP TABLE testing; 74 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 75 | -------------------------------------------------------------------------------- /sql/numeric.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- create table with different numeric type 9 | CREATE TABLE tbl_int(id serial primary key, a smallint, b integer, c serial, d real, e bigint, f bigserial, g double precision, h decimal, i numeric); 10 | 11 | -- minumum values 12 | INSERT INTO tbl_int values(1, -32768, -2147483648, 1, -0.123456, -9223372036854775808, 1, -1.123456789123456, -1234567890.1234567891, -9876543210.0987654321); 13 | UPDATE tbl_int SET a=a+1, b=b+1, c=c+1, d=d+1, e=e+1, f=f+1, g=g+1, h=h+1, i=i+1 WHERE id=1; 14 | DELETE FROM tbl_int WHERE id = 1; 15 | TRUNCATE TABLE tbl_int; 16 | 17 | -- maximum values 18 | INSERT INTO tbl_int values(1, 32767, 2147483647, 2147483647, 0.123456, 9223372036854775807, 9223372036854775807, 1.123456789123456, 1234567890.1234567891, 9876543210.0987654321); 19 | UPDATE tbl_int SET a=a-1, b=b-1, c=c-1, d=d-1, e=e-1, f=f-1, g=g-1, h=h-1, i=i-1 WHERE id=1; 20 | DELETE FROM tbl_int WHERE id = 1; 21 | TRUNCATE TABLE tbl_int; 22 | 23 | 24 | -- peek changes according to action configuration 25 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 26 | 27 | -- get changes according to action configuration 28 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 29 | 30 | -- drop table 31 | DROP TABLE tbl_int; 32 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 33 | -------------------------------------------------------------------------------- /sql/pkey.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | 9 | CREATE TABLE testing_one_pkey ( 10 | a smallserial PRIMARY KEY, 11 | b smallint, 12 | c int, 13 | d bigint, 14 | e int, 15 | f real, 16 | g double precision, 17 | h char(20), 18 | i varchar(30), 19 | j text 20 | ); 21 | 22 | CREATE TABLE testing_multi_pkey ( 23 | a smallserial, 24 | b smallint, 25 | c int, 26 | d bigint, 27 | e int, 28 | f real, 29 | g double precision, 30 | h char(20), 31 | i varchar(30), 32 | j text, 33 | PRIMARY KEY(a, b, c) 34 | ); 35 | 36 | CREATE TABLE testing_no_pkey ( 37 | a smallserial, 38 | b smallint, 39 | c int, 40 | d bigint, 41 | e int, 42 | f real, 43 | g double precision, 44 | h char(20), 45 | i varchar(30), 46 | j text 47 | ); 48 | 49 | CREATE TABLE testing_unique ( 50 | a smallserial, 51 | b smallint, 52 | c int, 53 | d bigint, 54 | e int, 55 | f real, 56 | g double precision, 57 | h char(20), 58 | i varchar(30), 59 | j text, 60 | UNIQUE(f, g) 61 | ); 62 | 63 | INSERT INTO testing_one_pkey (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 64 | INSERT INTO testing_multi_pkey (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 65 | INSERT INTO testing_no_pkey (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 66 | INSERT INTO testing_unique (b, c, d, e, f, g, h, i, j) VALUES (1, 2, 33, 555, 666.777777777, 3.33, 'testval1', 'testval2', 'testval3'); 67 | 68 | -- non pkey change 69 | UPDATE testing_one_pkey SET f = 777.888888888 WHERE b = 1; 70 | 71 | -- pkey change 72 | UPDATE testing_one_pkey SET a = 14 WHERE b = 1; 73 | 74 | -- non pkey change 75 | UPDATE testing_multi_pkey SET f = 777.888888888 WHERE b = 1; 76 | 77 | -- some pkeys change 78 | UPDATE testing_multi_pkey SET a = 14, c = 14 WHERE b = 1; 79 | 80 | -- all pkeys change 81 | UPDATE testing_multi_pkey SET a = 15, b = 15, c = 15 WHERE b = 1; 82 | 83 | -- no pkey 84 | UPDATE testing_no_pkey SET h = 'changed' WHERE b = 1; 85 | 86 | -- non unique change 87 | UPDATE testing_unique SET h = 'changed' WHERE b = 1; 88 | 89 | -- one unique val change 90 | UPDATE testing_unique SET f = 888.888888888 WHERE b = 1; 91 | 92 | -- all unique vals change 93 | UPDATE testing_unique SET f = 888.888888888, g = 6.66 WHERE b = 1; 94 | 95 | -- get changes 96 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 97 | 98 | DROP TABLE testing_unique; 99 | DROP TABLE testing_no_pkey; 100 | DROP TABLE testing_multi_pkey; 101 | DROP TABLE testing_one_pkey; 102 | 103 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 104 | -------------------------------------------------------------------------------- /sql/specval.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | CREATE TABLE testing ( 9 | a serial PRIMARY KEY, 10 | b varchar(80), 11 | c bool, 12 | d real 13 | ); 14 | 15 | BEGIN; 16 | INSERT INTO testing (b, c, d) VALUES( E'valid: '' " \\ / \d \f \r \t \u447F \u967F invalid: \\g \\k end', FALSE, 123.456); 17 | INSERT INTO testing (b, c, d) VALUES('aaa', 't', '+inf'); 18 | INSERT INTO testing (b, c, d) VALUES('aaa', 'f', 'nan'); 19 | INSERT INTO testing (b, c, d) VALUES('null', NULL, '-inf'); 20 | COMMIT; 21 | 22 | -- get changes 23 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true', 'skip_empty_xacts', 'true'); 24 | 25 | DROP TABLE testing; 26 | 27 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 28 | -------------------------------------------------------------------------------- /sql/string.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- network address 9 | CREATE TABLE tbl_net (id serial PRIMARY KEY, a inet, b cidr, c macaddr, d macaddr8); 10 | 11 | -- ipv4 in different format 12 | INSERT INTO tbl_net(a,b,c,d) VALUES('192.168.100.128/25'::inet, '192.168.100.128/25'::cidr, '08:00:2b:01:02:03'::macaddr, '08:00:2b:01:02:03:04:05'::macaddr8); 13 | INSERT INTO tbl_net(a,b,c,d) VALUES('192.168.0.1/24'::inet, '192.168/24'::cidr, '08-00-2b-01-02-03'::macaddr, '08-00-2b-01-02-03-04-05'::macaddr8); 14 | INSERT INTO tbl_net(a,b,c,d) VALUES('192.168.0.0/16'::inet, '192.168'::cidr, '08002b:010203'::macaddr, '08002b:0102030405'::macaddr8); 15 | INSERT INTO tbl_net(a,b,c,d) VALUES('10.0.0.1/8'::inet, '10'::cidr, '08002b-010203'::macaddr, '08002b-0102030405'::macaddr8); 16 | INSERT INTO tbl_net(a,b,c,d) VALUES('10.1.2.3/32'::inet, '10.1.2.3/32'::cidr, '0800.2b01.0203'::macaddr, '0800.2b01.0203.0405'::macaddr8); 17 | INSERT INTO tbl_net(a,b,c,d) VALUES('2001:4f8:3:ba::/64'::inet, '2001:4f8:3:ba::/64'::cidr, '0800-2b01-0203'::macaddr, '0800-2b01-0203-0405'::macaddr8); 18 | 19 | -- ipv6 and mac in lower case 20 | INSERT INTO tbl_net(a,b,c,d) VALUES('2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128'::inet, '2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128'::cidr, '08002b010203'::macaddr, '08002b01:02030405'::macaddr8); 21 | -- ipv6 and mac in upper case 22 | INSERT INTO tbl_net(a,b,c,d) VALUES('2001:4F8:3:BA:2E0:81FF:FE22:D1F1/128'::inet, '2001:4F8:3:BA:2E0:81FF:FE22:D1F1/128'::cidr, '08002B010203'::macaddr, '08002B0102030405'::macaddr8); 23 | 24 | UPDATE tbl_net SET a='10.1.2.3/32'::inet, b='10.1.2.3/32'::cidr, c='0800-2b01-0203'::macaddr, d='0800-2b01-0203-0405'::macaddr8 WHERE id=1; 25 | DELETE FROM tbl_net WHERE id = 1; 26 | TRUNCATE TABLE tbl_net; 27 | 28 | -- peek changes 29 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 30 | 31 | -- get changes 32 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 33 | 34 | -- drop table 35 | DROP TABLE tbl_net; 36 | 37 | 38 | -- geo data 39 | CREATE TABLE tbl_geo (id serial PRIMARY KEY, a point, b line, c lseg, d box, e path, f path, g polygon, h circle); 40 | ALTER TABLE tbl_geo REPLICA IDENTITY FULL; 41 | 42 | -- geo different data types 43 | INSERT INTO tbl_geo(a,b,c,d,e,f,g,h) VALUES('(1,1)'::point, '[(1,1),(2,2)]'::line, '[(3,3), (4,4)]'::lseg, '((1,1),(2,2))'::box, '[(1,1),(2,2),(3,3)]'::path, '((1,1),(2,2),(3,3))'::path, '((1,1),(2,2),(3,3),(4,4))'::polygon, '<(1,1),5>'::circle); 44 | 45 | UPDATE tbl_geo SET a='11,11'::point, b='((11,11),(12,12))'::line, c='((13,13), (14,14))'::lseg, d='(11,11),(12,12)'::box, e='(11,11),(12,12),(13,13)'::path, f='(11,11,12,12,13,13)'::path, g='(11,11),(12,12),(13,13),(14,14)'::polygon, h='((11,11),15)'::circle WHERE id=1; 46 | 47 | INSERT INTO tbl_geo(a,b,c,d,e,f,g,h) VALUES('(1,1)'::point, '(1,1),(2,2)'::line, '(3,3), (4,4)'::lseg, '1,1,2,2'::box, '1,1,2,2,3,3'::path, '1,1,2,2,3,3'::path, '(1,1,2,2,3,3,4,4)'::polygon, '1,1,5'::circle); 48 | 49 | UPDATE tbl_geo SET a='21,21'::point, b='21,21,22,22'::line, c='23,23, 24,24'::lseg, d='21,21,22,22'::box, e='(21,21,22,22,23,23)'::path, f='21,21,22,22,23,23'::path, g='21,21,22,22,23,23,24,24'::polygon, h='21,21,25'::circle WHERE id=2; 50 | 51 | DELETE FROM tbl_geo WHERE id = 1; 52 | DELETE FROM tbl_geo WHERE id = 2; 53 | TRUNCATE TABLE tbl_geo; 54 | 55 | -- peek changes 56 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 57 | 58 | -- get changes 59 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 60 | 61 | -- drop table 62 | DROP TABLE tbl_geo; 63 | 64 | 65 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 66 | -------------------------------------------------------------------------------- /sql/timestamptz.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- timestamptz 9 | CREATE TABLE tbl_tstz ( id serial PRIMARY KEY, tstz TIMESTAMPTZ NOT NULL ); 10 | INSERT INTO tbl_tstz (tstz) VALUES('2020-03-24 16:00:30-07') ; 11 | UPDATE tbl_tstz SET tstz = '2020-03-24 16:30:00-07' WHERE id=1; 12 | DELETE FROM tbl_tstz WHERE id = 1; 13 | TRUNCATE TABLE tbl_tstz; 14 | 15 | -- peek changes according to action configuration 16 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 17 | 18 | -- get changes according to action configuration 19 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 20 | 21 | 22 | -- drop tables 23 | DROP TABLE tbl_tstz; 24 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 25 | -------------------------------------------------------------------------------- /sql/transaction.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- actions 9 | CREATE TABLE testing (a varchar(30) primary key); 10 | INSERT INTO testing (a) VALUES('cary'); 11 | INSERT INTO testing (a) VALUES('john'); 12 | INSERT INTO testing (a) VALUES('peter'); 13 | 14 | -- define a transaction containing sub transactions 15 | BEGIN; 16 | INSERT INTO testing (a) VALUES('david'); 17 | SAVEPOINT p1; 18 | INSERT INTO testing (a) VALUES('grant'); 19 | SAVEPOINT p2; 20 | INSERT INTO testing (a) VALUES('mike'); 21 | SAVEPOINT p3; 22 | INSERT INTO testing (a) VALUES('allen'); 23 | SAVEPOINT p4; 24 | INSERT INTO testing (a) VALUES('dan'); 25 | ROLLBACK TO SAVEPOINT p3; 26 | RELEASE SAVEPOINT p1; 27 | INSERT INTO testing (a) VALUES('cheese'); 28 | COMMIT; 29 | 30 | 31 | -- peek and get changes with and without transaction mode 32 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'use_transaction', 'true', 'regress', 'true'); 33 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'use_transaction', 'false', 'regress', 'true', 'skip_empty_xacts', 'false'); 34 | 35 | DROP TABLE testing; 36 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 37 | -------------------------------------------------------------------------------- /sql/xml.sql: -------------------------------------------------------------------------------- 1 | \set VERBOSITY terse 2 | 3 | -- predictability 4 | SET synchronous_commit = on; 5 | 6 | SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2mongo'); 7 | 8 | -- xml data 9 | CREATE TABLE tbl_xml (id serial primary key, a xml); 10 | 11 | -- insert data simple SQL way 12 | INSERT INTO tbl_xml(a) VALUES(' xml-data '::xml); 13 | -- using XMLPARSE and DOCUMENT syntaxes 14 | INSERT INTO tbl_xml(a) VALUES(XMLPARSE (DOCUMENT' xml-data ')); 15 | -- xpath example simple SQL way 16 | INSERT INTO tbl_xml(a) VALUES(' 17 | 19 | IT 20 | 21 | 22 | John Smith 23 | 24 24 | 25 | 26 | Michael Black 27 | 28 28 | 29 | 30 | '::xml); 31 | -- xpath example in minify way 32 | INSERT INTO tbl_xml(a) VALUES('ITJohn Smith24Michael Black28'::xml); 33 | -- xpath example using XMLPARSE and DOCUMENT 34 | INSERT INTO tbl_xml(a) VALUES(XMLPARSE (DOCUMENT' 35 | 37 | IT 38 | 39 | 40 | John Smith 41 | 24 42 | 43 | 44 | Michael Black 45 | 28 46 | 47 | 48 | ')); 49 | 50 | -- update using simple SQL way 51 | UPDATE tbl_xml SET a = ' 52 | 54 | IT 55 | 56 | 57 | John Smith 58 | 24 59 | 60 | 61 | Michael Black 62 | 28 63 | 64 | 65 | '::xml WHERE id=1; 66 | 67 | -- update using XMLPARSE and DOCUMENT syntaxes 68 | UPDATE tbl_xml SET a = 69 | XMLPARSE (DOCUMENT' 70 | 72 | IT 73 | 74 | 75 | John Smith 76 | 24 77 | 78 | 79 | Michael Black 80 | 28 81 | 82 | 83 | ') WHERE id=2; 84 | 85 | -- delete 86 | DELETE FROM tbl_xml WHERE id = 1; 87 | TRUNCATE TABLE tbl_xml; 88 | 89 | -- peek changes 90 | SELECT data FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'regress', 'true'); 91 | 92 | -- get changes 93 | SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'regress', 'true'); 94 | 95 | -- drop table 96 | DROP TABLE tbl_xml; 97 | 98 | SELECT 'end' FROM pg_drop_replication_slot('regression_slot'); 99 | -------------------------------------------------------------------------------- /wal2mongo.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * wal2mongo.c 4 | * logical decoding output plugin for MongoDB 5 | * 6 | * Copyright (c) 2019-2020, Highgo Global Development Center 7 | * 8 | * IDENTIFICATION 9 | * when build under PostgreSQL source code tree, 10 | * contrib/wal2mongo/wal2mongo.c 11 | * 12 | *------------------------------------------------------------------------- 13 | */ 14 | #include "postgres.h" 15 | 16 | #include "catalog/pg_type.h" 17 | 18 | #include "replication/logical.h" 19 | #include "replication/origin.h" 20 | #include "commands/dbcommands.h" 21 | #include "utils/builtins.h" 22 | #include "utils/lsyscache.h" 23 | #include "utils/memutils.h" 24 | #include "utils/rel.h" 25 | #include "utils/guc.h" 26 | #include "common/jsonapi.h" 27 | #include "utils/datetime.h" 28 | #include "utils/json.h" 29 | #include "miscadmin.h" 30 | 31 | PG_MODULE_MAGIC; 32 | 33 | /* These must be available to pg_dlsym() */ 34 | extern void _PG_init(void); 35 | extern void _PG_output_plugin_init(OutputPluginCallbacks *cb); 36 | 37 | typedef struct 38 | { 39 | bool insert; 40 | bool update; 41 | bool delete; 42 | bool truncate; 43 | } Wal2MongoAction; 44 | 45 | typedef struct 46 | { 47 | MemoryContext context; 48 | bool skip_empty_xacts; 49 | bool xact_wrote_changes; 50 | bool only_local; 51 | bool use_transaction; 52 | bool include_cluster_name; 53 | bool regress; 54 | Wal2MongoAction actions; 55 | } Wal2MongoData; 56 | 57 | static void pg_w2m_decode_startup(LogicalDecodingContext *ctx, 58 | OutputPluginOptions *opt, 59 | bool is_init); 60 | 61 | static void pg_w2m_decode_shutdown(LogicalDecodingContext *ctx); 62 | 63 | static void pg_w2m_decode_begin_txn(LogicalDecodingContext *ctx, 64 | ReorderBufferTXN *txn); 65 | 66 | static void pg_w2m_decode_begin(LogicalDecodingContext *ctx, 67 | Wal2MongoData *data, 68 | ReorderBufferTXN *txn); 69 | 70 | static void pg_w2m_decode_commit_txn(LogicalDecodingContext *ctx, 71 | ReorderBufferTXN *txn, XLogRecPtr commit_lsn); 72 | 73 | static void pg_w2m_decode_change(LogicalDecodingContext *ctx, 74 | ReorderBufferTXN *txn, Relation rel, 75 | ReorderBufferChange *change); 76 | 77 | static void pg_w2m_decode_truncate(LogicalDecodingContext *ctx, 78 | ReorderBufferTXN *txn, 79 | int nrelations, Relation relations[], 80 | ReorderBufferChange *change); 81 | 82 | static bool pg_w2m_decode_filter(LogicalDecodingContext *ctx, 83 | RepOriginId origin_id); 84 | 85 | static void pg_w2m_decode_message(LogicalDecodingContext *ctx, 86 | ReorderBufferTXN *txn, XLogRecPtr message_lsn, 87 | bool transactional, const char *prefix, 88 | Size sz, const char *message); 89 | 90 | static bool split_string_to_list(char *rawstring, char separator, List **sl); 91 | 92 | 93 | /* Will be called immediately after loaded */ 94 | void 95 | _PG_init(void) 96 | { 97 | 98 | } 99 | 100 | /* Initialize callback functions */ 101 | void 102 | _PG_output_plugin_init(OutputPluginCallbacks *cb) 103 | { 104 | AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit); 105 | 106 | cb->startup_cb = pg_w2m_decode_startup; 107 | cb->begin_cb = pg_w2m_decode_begin_txn; 108 | cb->change_cb = pg_w2m_decode_change; 109 | cb->truncate_cb = pg_w2m_decode_truncate; 110 | cb->commit_cb = pg_w2m_decode_commit_txn; 111 | cb->filter_by_origin_cb = pg_w2m_decode_filter; 112 | cb->shutdown_cb = pg_w2m_decode_shutdown; 113 | cb->message_cb = pg_w2m_decode_message; 114 | } 115 | 116 | 117 | /* Initialize this plugin's resources */ 118 | static void 119 | pg_w2m_decode_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, 120 | bool is_init) 121 | { 122 | ListCell *option; 123 | Wal2MongoData *data; 124 | 125 | data = palloc0(sizeof(Wal2MongoData)); 126 | data->context = AllocSetContextCreate(ctx->context, 127 | "wal2mongo context", 128 | ALLOCSET_DEFAULT_SIZES); 129 | data->skip_empty_xacts = false; 130 | data->only_local = false; 131 | data->use_transaction = false; 132 | data->include_cluster_name = true; 133 | data->regress = false; 134 | 135 | data->actions.delete = true; 136 | data->actions.insert = true; 137 | data->actions.update = true; 138 | data->actions.truncate = true; 139 | 140 | ctx->output_plugin_private = data; 141 | 142 | opt->output_type = OUTPUT_PLUGIN_TEXTUAL_OUTPUT; 143 | opt->receive_rewrites = false; 144 | 145 | foreach(option, ctx->output_plugin_options) 146 | { 147 | DefElem *elem = lfirst(option); 148 | Assert(elem->arg == NULL || IsA(elem->arg, String)); 149 | if (strcmp(elem->defname, "skip_empty_xacts") == 0) 150 | { 151 | /* if option value is NULL then assume that value is false */ 152 | if (elem->arg == NULL) 153 | data->skip_empty_xacts = false; 154 | else if (!parse_bool(strVal(elem->arg), &data->skip_empty_xacts)) 155 | ereport(ERROR, 156 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 157 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 158 | strVal(elem->arg), elem->defname))); 159 | } 160 | else if (strcmp(elem->defname, "only_local") == 0) 161 | { 162 | /* if option value is NULL then assume that value is false */ 163 | if (elem->arg == NULL) 164 | data->only_local = false; 165 | else if (!parse_bool(strVal(elem->arg), &data->only_local)) 166 | ereport(ERROR, 167 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 168 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 169 | strVal(elem->arg), elem->defname))); 170 | } 171 | else if (strcmp(elem->defname, "use_transaction") == 0) 172 | { 173 | /* if option value is NULL then assume that value is false */ 174 | if (elem->arg == NULL) 175 | data->use_transaction = false; 176 | else if (!parse_bool(strVal(elem->arg), &data->use_transaction)) 177 | ereport(ERROR, 178 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 179 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 180 | strVal(elem->arg), elem->defname))); 181 | } 182 | else if (strcmp(elem->defname, "force_binary") == 0) 183 | { 184 | bool force_binary; 185 | 186 | if (elem->arg == NULL) 187 | continue; 188 | else if (!parse_bool(strVal(elem->arg), &force_binary)) 189 | ereport(ERROR, 190 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 191 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 192 | strVal(elem->arg), elem->defname))); 193 | 194 | if (force_binary) 195 | opt->output_type = OUTPUT_PLUGIN_BINARY_OUTPUT; 196 | } 197 | else if (strcmp(elem->defname, "include_cluster_name") == 0) 198 | { 199 | /* if option value is NULL then assume that value is false */ 200 | if (elem->arg == NULL) 201 | data->include_cluster_name = false; 202 | else if (!parse_bool(strVal(elem->arg), &data->include_cluster_name)) 203 | ereport(ERROR, 204 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 205 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 206 | strVal(elem->arg), elem->defname))); 207 | } 208 | else if (strcmp(elem->defname, "regress") == 0) 209 | { 210 | /* if option value is NULL then assume that value is false */ 211 | if (elem->arg == NULL) 212 | data->regress = false; 213 | else if (!parse_bool(strVal(elem->arg), &data->regress)) 214 | ereport(ERROR, 215 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 216 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 217 | strVal(elem->arg), elem->defname))); 218 | } 219 | else if (strcmp(elem->defname, "actions") == 0) 220 | { 221 | char *rawstr; 222 | 223 | if (elem->arg == NULL) 224 | { 225 | elog(DEBUG1, "actions argument is null"); 226 | /* argument null means default; nothing to do here */ 227 | } 228 | else 229 | { 230 | List *selected_actions = NIL; 231 | ListCell *lc; 232 | 233 | rawstr = pstrdup(strVal(elem->arg)); 234 | if (!split_string_to_list(rawstr, ',', &selected_actions)) 235 | { 236 | pfree(rawstr); 237 | ereport(ERROR, 238 | (errcode(ERRCODE_INVALID_NAME), 239 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 240 | strVal(elem->arg), elem->defname))); 241 | } 242 | 243 | data->actions.insert = false; 244 | data->actions.update = false; 245 | data->actions.delete = false; 246 | data->actions.truncate = false; 247 | 248 | foreach(lc, selected_actions) 249 | { 250 | char *p = lfirst(lc); 251 | 252 | if (strcmp(p, "insert") == 0) 253 | data->actions.insert = true; 254 | else if (strcmp(p, "update") == 0) 255 | data->actions.update = true; 256 | else if (strcmp(p, "delete") == 0) 257 | data->actions.delete = true; 258 | else if (strcmp(p, "truncate") == 0) 259 | data->actions.truncate = true; 260 | else 261 | ereport(ERROR, 262 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 263 | errmsg("could not parse value \"%s\" for parameter \"%s\"", 264 | p, elem->defname))); 265 | } 266 | 267 | pfree(rawstr); 268 | list_free_deep(selected_actions); 269 | } 270 | } 271 | else 272 | { 273 | ereport(ERROR, 274 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 275 | errmsg("option \"%s\" = \"%s\" is unknown", 276 | elem->defname, 277 | elem->arg ? strVal(elem->arg) : "(null)"))); 278 | } 279 | } 280 | } 281 | 282 | /* Cleanup this plugin's resources */ 283 | static void 284 | pg_w2m_decode_shutdown(LogicalDecodingContext *ctx) 285 | { 286 | Wal2MongoData *data = ctx->output_plugin_private; 287 | 288 | /* cleanup our own resources via memory context reset */ 289 | MemoryContextDelete(data->context); 290 | } 291 | 292 | /* BEGIN callback */ 293 | static void 294 | pg_w2m_decode_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn) 295 | { 296 | Wal2MongoData *data = ctx->output_plugin_private; 297 | 298 | data->xact_wrote_changes = false; 299 | if (data->skip_empty_xacts) 300 | return; 301 | 302 | pg_w2m_decode_begin(ctx, data, txn); 303 | } 304 | 305 | static void pg_w2m_decode_begin(LogicalDecodingContext *ctx, 306 | Wal2MongoData *data, 307 | ReorderBufferTXN *txn) 308 | { 309 | 310 | /* Skip this callback if transaction mode is not enabled */ 311 | if(!data->use_transaction) 312 | return; 313 | 314 | /* first write the session variable for Mongo */ 315 | OutputPluginPrepareWrite(ctx, false); 316 | appendStringInfo(ctx->out, "session%u%s = db.getMongo().startSession();", 317 | data->regress == true? 0 : txn->xid, ctx->slot->data.name.data); 318 | OutputPluginWrite(ctx, false); 319 | 320 | /* then write transaction start */ 321 | OutputPluginPrepareWrite(ctx, true); 322 | appendStringInfo(ctx->out, "session%u%s.startTransaction();", 323 | data->regress == true? 0 : txn->xid, ctx->slot->data.name.data); 324 | OutputPluginWrite(ctx, true); 325 | } 326 | 327 | /* COMMIT callback */ 328 | static void 329 | pg_w2m_decode_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, 330 | XLogRecPtr commit_lsn) 331 | { 332 | Wal2MongoData *data = ctx->output_plugin_private; 333 | 334 | /* Skip this callback if it is an empty transaction */ 335 | if (data->skip_empty_xacts && !data->xact_wrote_changes) 336 | return; 337 | 338 | /* Skip this callback if transaction mode is not enabled */ 339 | if(!data->use_transaction) 340 | return; 341 | 342 | /* first write the commit transaction cmd */ 343 | OutputPluginPrepareWrite(ctx, false); 344 | appendStringInfo(ctx->out, "session%u%s.commitTransaction();", 345 | data->regress == true? 0 : txn->xid, ctx->slot->data.name.data); 346 | OutputPluginWrite(ctx, false); 347 | 348 | /* then write session termination */ 349 | OutputPluginPrepareWrite(ctx, true); 350 | appendStringInfo(ctx->out, "session%u%s.endSession();", 351 | data->regress == true? 0 : txn->xid, ctx->slot->data.name.data); 352 | OutputPluginWrite(ctx, true); 353 | } 354 | 355 | /* Generic message callback */ 356 | static bool 357 | pg_w2m_decode_filter(LogicalDecodingContext *ctx, 358 | RepOriginId origin_id) 359 | { 360 | Wal2MongoData *data = ctx->output_plugin_private; 361 | 362 | if (data->only_local && origin_id != InvalidRepOriginId) 363 | return true; 364 | return false; 365 | } 366 | 367 | /* Postgres data types to MongoDB data types conversion 368 | * if input data has already been quoted, then use false to skip quotation 369 | * if input data hasn't been quoted yet, then use true to request a quotation 370 | */ 371 | static void 372 | print_w2m_data_type(StringInfo s, char *outputstr, const char *type, bool quotation) 373 | { 374 | const char *valptr; 375 | 376 | for (valptr = outputstr; *valptr; valptr++) 377 | { 378 | char ch = *valptr; 379 | if(ch == '{') 380 | { 381 | appendStringInfo(s, "[%s(", type); 382 | if (quotation) 383 | appendStringInfo(s, "\""); 384 | } 385 | else if(ch == ',') 386 | { 387 | if (quotation) 388 | appendStringInfo(s, "\""); 389 | appendStringInfo(s, "),%s(", type); 390 | if (quotation) 391 | appendStringInfo(s, "\""); 392 | } 393 | else if(ch == '}') 394 | { 395 | if (quotation) 396 | appendStringInfo(s, "\""); 397 | appendStringInfo(s, ")]"); 398 | } 399 | else 400 | appendStringInfoChar(s, ch); 401 | } 402 | } 403 | 404 | /* PG to MG data conversion */ 405 | static void 406 | print_w2m_literal(StringInfo s, Oid typid, char *outputstr) 407 | { 408 | const char *valptr; 409 | 410 | switch (typid) 411 | { 412 | case INT2OID: 413 | case INT4OID: 414 | appendStringInfo(s, " NumberInt(\"%s\")", outputstr); 415 | break; 416 | 417 | case FLOAT4OID: 418 | appendStringInfoString(s, outputstr); 419 | break; 420 | 421 | case INT8OID: 422 | case OIDOID: 423 | appendStringInfo(s, " NumberLong(\"%s\")", outputstr); 424 | break; 425 | 426 | case FLOAT8OID: 427 | case NUMERICOID: 428 | appendStringInfo(s, " NumberDecimal(\"%s\")", outputstr); 429 | break; 430 | 431 | case BITOID: 432 | case VARBITOID: 433 | appendStringInfo(s, "B'%s'", outputstr); 434 | break; 435 | 436 | case BOOLOID: 437 | if (strcmp(outputstr, "t") == 0) 438 | appendStringInfoString(s, "true"); 439 | else 440 | appendStringInfoString(s, "false"); 441 | break; 442 | 443 | case TIMESTAMPTZOID: 444 | appendStringInfo(s, " ISODate(\"%s\")", outputstr); 445 | break; 446 | 447 | case UUIDOID: 448 | appendStringInfo(s, " UUID(\"%s\")", outputstr); 449 | break; 450 | 451 | case BYTEAOID: 452 | appendStringInfo(s, " HexData(0, \""); 453 | for (valptr = outputstr+2; *valptr; valptr++) 454 | { 455 | char ch = *valptr; 456 | appendStringInfoChar(s, ch); 457 | } 458 | appendStringInfo(s, "\")"); 459 | break; 460 | 461 | /* Array data type */ 462 | case BOOLARRAYOID: 463 | for (valptr = outputstr; *valptr; valptr++) 464 | { 465 | char ch = *valptr; 466 | if(ch == '{') 467 | appendStringInfoChar(s, '['); 468 | else if(ch == ',') 469 | appendStringInfoChar(s, ','); 470 | else if(ch == '}') 471 | appendStringInfoChar(s, ']'); 472 | else 473 | { 474 | if (ch == 't') 475 | appendStringInfoString(s, "true"); 476 | else 477 | appendStringInfoString(s, "false"); 478 | } 479 | } 480 | break; 481 | 482 | case INT2ARRAYOID: 483 | case INT4ARRAYOID: 484 | print_w2m_data_type(s, outputstr, "NumberInt", false); 485 | break; 486 | 487 | case FLOAT4ARRAYOID: 488 | for (valptr = outputstr; *valptr; valptr++) 489 | { 490 | char ch = *valptr; 491 | if(ch == '{') 492 | appendStringInfoChar(s, '['); 493 | else if(ch == ',') 494 | appendStringInfoChar(s, ','); 495 | else if(ch == '}') 496 | appendStringInfoChar(s, ']'); 497 | else 498 | { 499 | if (SQL_STR_DOUBLE(ch, false)) 500 | appendStringInfoChar(s, ch); 501 | appendStringInfoChar(s, ch); 502 | } 503 | } 504 | break; 505 | 506 | case INT8ARRAYOID: 507 | print_w2m_data_type(s, outputstr, "NumberLong", false); 508 | break; 509 | 510 | case TIMESTAMPTZARRAYOID: 511 | print_w2m_data_type(s, outputstr, "ISODate", false); 512 | break; 513 | 514 | case FLOAT8ARRAYOID: 515 | case NUMERICARRAYOID: 516 | print_w2m_data_type(s, outputstr, "NumberDecimal", true); 517 | break; 518 | 519 | case UUIDARRAYOID: 520 | print_w2m_data_type(s, outputstr, "UUID", true); 521 | break; 522 | 523 | case CHARARRAYOID: 524 | case NAMEARRAYOID: 525 | case TEXTARRAYOID: 526 | case BPCHARARRAYOID: 527 | case VARCHARARRAYOID: 528 | for (valptr = outputstr; *valptr; valptr++) 529 | { 530 | char ch = *valptr; 531 | if(ch == '{') 532 | if(*(valptr+1) == '{' || *(valptr+1) == '"') 533 | appendStringInfoChar(s, '['); 534 | else 535 | appendStringInfo(s, "[\""); 536 | else if(ch == ',') 537 | if(*(valptr+1) == '{' || *(valptr+1) == '"') 538 | appendStringInfoChar(s, ','); 539 | else 540 | appendStringInfo(s, "\",\""); 541 | else if(ch == '}') 542 | if(*(valptr-1) != '}' && *(valptr-1) != '"') 543 | appendStringInfo(s, "\"]"); 544 | else 545 | appendStringInfoChar(s, ']'); 546 | else 547 | { 548 | if (SQL_STR_DOUBLE(ch, false)) 549 | appendStringInfoChar(s, ch); 550 | appendStringInfoChar(s, ch); 551 | } 552 | } 553 | break; 554 | /* Array data type */ 555 | case JSONOID: 556 | case JSONBOID: 557 | appendStringInfo(s, "%s", outputstr); 558 | break; 559 | case JSONPATHOID: 560 | appendStringInfoChar(s, '\"'); 561 | for (valptr = outputstr; *valptr; valptr++) 562 | { 563 | char ch = *valptr; 564 | 565 | if(ch != '"') 566 | appendStringInfoChar(s, ch); 567 | } 568 | appendStringInfoChar(s, '\"'); 569 | break; 570 | case JSONARRAYOID: 571 | case JSONBARRAYOID: 572 | /* For JSON arrays, we need to strip first and last curly brackets and 573 | * replace them with square brackets in order to be accepted by Mongodb 574 | */ 575 | outputstr = outputstr + 1; 576 | outputstr[strlen(outputstr) - 1] = '\0'; 577 | appendStringInfoChar(s, '['); 578 | for (valptr = outputstr; *valptr; valptr++) 579 | { 580 | char ch = *valptr; 581 | if(ch == '\\' && (valptr+1) != NULL && *(valptr+1) == '"' ) 582 | { 583 | /* replace all \" with " */ 584 | appendStringInfoChar(s, '"'); 585 | } 586 | else if(ch == '"') 587 | { 588 | /* ignore all the double quote without backslash escape */ 589 | } 590 | else 591 | { 592 | appendStringInfoChar(s, ch); 593 | } 594 | } 595 | appendStringInfoChar(s, ']'); 596 | break; 597 | 598 | case XMLOID: 599 | appendStringInfoChar(s, '\"'); 600 | for (valptr = outputstr; *valptr; valptr++) 601 | { 602 | char ch = *valptr; 603 | 604 | if (ch == '\n') 605 | continue; 606 | if ((ch == '"') && (*(valptr+1) != '\0')) 607 | appendStringInfoChar(s, '\\'); 608 | 609 | if (SQL_STR_DOUBLE(ch, false)) 610 | appendStringInfoChar(s, ch); 611 | appendStringInfoChar(s, ch); 612 | } 613 | appendStringInfoChar(s, '\"'); 614 | break; 615 | 616 | default: 617 | appendStringInfoChar(s, '\"'); 618 | for (valptr = outputstr; *valptr; valptr++) 619 | { 620 | char ch = *valptr; 621 | 622 | if (SQL_STR_DOUBLE(ch, false)) 623 | appendStringInfoChar(s, ch); 624 | appendStringInfoChar(s, ch); 625 | } 626 | appendStringInfoChar(s, '\"'); 627 | break; 628 | } 629 | } 630 | 631 | /* print the tuple 'tuple' into the StringInfo s */ 632 | static void 633 | tuple_to_stringinfo(StringInfo s, TupleDesc tupdesc, HeapTuple tuple, bool skip_nulls, 634 | Bitmapset * pkAttrs) 635 | { 636 | int natt; 637 | 638 | appendStringInfoString(s, " \{ "); 639 | /* print all columns individually */ 640 | for (natt = 0; natt < tupdesc->natts; natt++) 641 | { 642 | Form_pg_attribute attr; /* the attribute itself */ 643 | Oid typid; /* type of current attribute */ 644 | Oid typoutput; /* output function */ 645 | bool typisvarlena; 646 | Datum origval; /* possibly toasted Datum */ 647 | bool isnull; /* column is null? */ 648 | 649 | attr = TupleDescAttr(tupdesc, natt); 650 | 651 | /* 652 | * don't print dropped columns, we can't be sure everything is 653 | * available for them 654 | */ 655 | if (attr->attisdropped) 656 | continue; 657 | 658 | /* 659 | * Don't print system columns, oid will already have been printed if 660 | * present. 661 | */ 662 | if (attr->attnum < 0) 663 | continue; 664 | 665 | typid = attr->atttypid; 666 | 667 | /* 668 | * if pkAttrs is valid and not empty. Check if the current attribute is 669 | * a member of the pkAttrs. If not a member, continue to next attribute. 670 | */ 671 | if(pkAttrs && !bms_is_empty(pkAttrs) && 672 | !bms_is_member(natt + 1 - FirstLowInvalidHeapAttributeNumber, 673 | pkAttrs)) 674 | continue; 675 | 676 | 677 | /* get Datum from tuple */ 678 | origval = heap_getattr(tuple, natt + 1, tupdesc, &isnull); 679 | 680 | if (isnull && skip_nulls) 681 | continue; 682 | 683 | /* print attribute name */ 684 | if ( natt != 0 ) 685 | appendStringInfoString(s, ", "); 686 | 687 | appendStringInfoString(s, quote_identifier(NameStr(attr->attname))); 688 | 689 | /* print separator */ 690 | appendStringInfoChar(s, ':'); 691 | 692 | /* query output function */ 693 | getTypeOutputInfo(typid, 694 | &typoutput, &typisvarlena); 695 | 696 | /* print data */ 697 | if (isnull) 698 | appendStringInfoString(s, "null"); 699 | else if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) 700 | appendStringInfoString(s, "unchanged-toast-datum"); 701 | else if (!typisvarlena) 702 | { 703 | if (typid == TIMESTAMPTZOID || typid == TIMESTAMPTZARRAYOID) 704 | { 705 | char buf[MAXDATELEN + 1]; 706 | JsonEncodeDateTime(buf, origval, TIMESTAMPTZOID, NULL); 707 | print_w2m_literal(s, typid, buf); 708 | } 709 | else 710 | print_w2m_literal(s, typid, 711 | OidOutputFunctionCall(typoutput, origval)); 712 | } 713 | else 714 | { 715 | Datum val; /* definitely detoasted Datum */ 716 | char *result; 717 | char *rp; 718 | 719 | val = PointerGetDatum(PG_DETOAST_DATUM(origval)); 720 | if (typid == BYTEAOID ) 721 | { 722 | /* Print hex format */ 723 | rp = result = palloc(VARSIZE_ANY_EXHDR(val) * 2 + 2 + 1); 724 | *rp++ = '\\'; 725 | *rp++ = 'x'; 726 | rp += hex_encode(VARDATA_ANY(val), VARSIZE_ANY_EXHDR(val), rp); 727 | *rp = '\0'; 728 | print_w2m_literal(s, typid, result); 729 | } 730 | else 731 | print_w2m_literal(s, typid, OidOutputFunctionCall(typoutput, val)); 732 | } 733 | 734 | } 735 | appendStringInfoString(s, " }"); 736 | } 737 | 738 | /* 739 | * callback for individual changed tuples 740 | */ 741 | static void 742 | pg_w2m_decode_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, 743 | Relation relation, ReorderBufferChange *change) 744 | { 745 | Wal2MongoData *data; 746 | Form_pg_class class_form; 747 | TupleDesc tupdesc; 748 | MemoryContext old; 749 | Bitmapset *pkAttrs; 750 | 751 | data = ctx->output_plugin_private; 752 | 753 | /* output BEGIN if we haven't yet */ 754 | if (data->skip_empty_xacts && !data->xact_wrote_changes) 755 | { 756 | pg_w2m_decode_begin(ctx, data, txn); 757 | } 758 | data->xact_wrote_changes = true; 759 | 760 | class_form = RelationGetForm(relation); 761 | tupdesc = RelationGetDescr(relation); 762 | 763 | /* Avoid leaking memory by using and resetting our own context */ 764 | old = MemoryContextSwitchTo(data->context); 765 | 766 | /* write the db switch command */ 767 | OutputPluginPrepareWrite(ctx, false); 768 | 769 | /* Here we are concatenating ClusterName, DatabaseName, and SlotName to form 770 | * a unified database name in MongoDB's perspective, so Mongo knows the changes 771 | * are streamed from which cluster, which database and via which slot 772 | * 773 | * TODO: we will introduce more configurable options to fine-tune this output style 774 | * behaviors. 775 | */ 776 | if(data->include_cluster_name) 777 | { 778 | char *cluster_name = GetConfigOptionByName("cluster_name", NULL, true); 779 | 780 | appendStringInfo(ctx->out, "use %s_%s_%s;", 781 | data->regress == true ? "mycluster" : (cluster_name[0] == '\0' ? "mycluster" : cluster_name), 782 | data->regress == true ? "mydb" : get_database_name(MyDatabaseId), 783 | ctx->slot->data.name.data[0] == '\0' ? "myslot" : 784 | ctx->slot->data.name.data); 785 | } 786 | else 787 | { 788 | appendStringInfo(ctx->out, "use %s_%s;", 789 | data->regress == true ? "mydb" : get_database_name(MyDatabaseId), 790 | ctx->slot->data.name.data[0] == '\0' ? "myslot" : 791 | ctx->slot->data.name.data); 792 | } 793 | OutputPluginWrite(ctx, false); 794 | 795 | OutputPluginPrepareWrite(ctx, true); 796 | appendStringInfoString(ctx->out, 797 | quote_qualified_identifier("db", class_form->relrewrite ? 798 | get_rel_name(class_form->relrewrite) : 799 | NameStr(class_form->relname))); 800 | 801 | switch (change->action) 802 | { 803 | case REORDER_BUFFER_CHANGE_INSERT: 804 | appendStringInfoString(ctx->out, ".insertOne("); 805 | if (change->data.tp.newtuple == NULL) 806 | appendStringInfoString(ctx->out, " (no-tuple-data)"); 807 | else 808 | tuple_to_stringinfo(ctx->out, tupdesc, 809 | &change->data.tp.newtuple->tuple, 810 | true, NULL); 811 | appendStringInfoString(ctx->out, " );"); 812 | break; 813 | case REORDER_BUFFER_CHANGE_UPDATE: 814 | appendStringInfoString(ctx->out, ".updateOne("); 815 | if (change->data.tp.oldtuple != NULL) 816 | { 817 | /* 818 | * the old tuple will contain the old value of primary key if it has been changed under DEFAULT replica identity 819 | * the old tuple will contain all old values regardless if they have bee changed under FULL replica identity 820 | * either way, find the primary key columns and print them only. 821 | */ 822 | pkAttrs = RelationGetIndexAttrBitmap(relation, 823 | INDEX_ATTR_BITMAP_PRIMARY_KEY); 824 | tuple_to_stringinfo(ctx->out, tupdesc, 825 | &change->data.tp.oldtuple->tuple, 826 | true, pkAttrs); 827 | bms_free(pkAttrs); 828 | } 829 | else 830 | { 831 | /* 832 | * the old tuple is NULL case. This means primary key has not been changed and the replica identity is not set to FULL. 833 | * we need to figure out the primary key column from new tuple 834 | */ 835 | if (change->data.tp.newtuple != NULL) 836 | { 837 | pkAttrs = RelationGetIndexAttrBitmap(relation, 838 | INDEX_ATTR_BITMAP_PRIMARY_KEY); 839 | if (!pkAttrs) 840 | { 841 | /* old tuple is NULL and no primary key is present in newtuple: Write null */ 842 | appendStringInfoString(ctx->out, "{ selector: \"null\" }"); 843 | } 844 | else 845 | { 846 | tuple_to_stringinfo(ctx->out, tupdesc, 847 | &change->data.tp.newtuple->tuple, 848 | true, pkAttrs); 849 | bms_free(pkAttrs); 850 | } 851 | } 852 | } 853 | 854 | if (change->data.tp.newtuple != NULL) 855 | { 856 | appendStringInfoString(ctx->out, ", \{ $set: "); 857 | tuple_to_stringinfo(ctx->out, tupdesc, 858 | &change->data.tp.newtuple->tuple, 859 | false, NULL); 860 | appendStringInfoString(ctx->out, " }"); 861 | } 862 | appendStringInfoString(ctx->out, " );"); 863 | break; 864 | case REORDER_BUFFER_CHANGE_DELETE: 865 | appendStringInfoString(ctx->out, ".deleteOne("); 866 | /* if there was no PK, we only know that a delete happened */ 867 | if (change->data.tp.oldtuple == NULL) 868 | appendStringInfoString(ctx->out, " (no-tuple-data)"); 869 | /* In DELETE, only the replica identity is present; display that */ 870 | else 871 | { 872 | /* For delete, we looked up the primary key attribute bitmap for use 873 | * in the subsequent tuple_to_stringinfo() call so the output will only 874 | * contain primary key columns in the oldtuple instead of all columns. 875 | */ 876 | pkAttrs = RelationGetIndexAttrBitmap(relation, 877 | INDEX_ATTR_BITMAP_PRIMARY_KEY); 878 | tuple_to_stringinfo(ctx->out, tupdesc, 879 | &change->data.tp.oldtuple->tuple, 880 | true, pkAttrs); 881 | bms_free(pkAttrs); 882 | } 883 | appendStringInfoString(ctx->out, " );"); 884 | break; 885 | default: 886 | Assert(false); 887 | } 888 | 889 | MemoryContextSwitchTo(old); 890 | MemoryContextReset(data->context); 891 | 892 | OutputPluginWrite(ctx, true); 893 | } 894 | 895 | static void 896 | pg_w2m_decode_truncate(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, 897 | int nrelations, Relation relations[], ReorderBufferChange *change) 898 | { 899 | /* TODO: to be supported in future version */ 900 | elog(DEBUG1, "TRUNCATE replication is not supported\n"); 901 | } 902 | 903 | static void 904 | pg_w2m_decode_message(LogicalDecodingContext *ctx, 905 | ReorderBufferTXN *txn, XLogRecPtr lsn, bool transactional, 906 | const char *prefix, Size sz, const char *message) 907 | { 908 | /* message decoding not supported */ 909 | elog(DEBUG1, "message decoding is not supported\n"); 910 | } 911 | 912 | static bool 913 | split_string_to_list(char *rawstring, char separator, List **sl) 914 | { 915 | char *nextp; 916 | bool done = false; 917 | 918 | nextp = rawstring; 919 | 920 | while (isspace(*nextp)) 921 | nextp++; /* skip leading whitespace */ 922 | 923 | if (*nextp == '\0') 924 | return true; /* allow empty string */ 925 | 926 | /* At the top of the loop, we are at start of a new identifier. */ 927 | do 928 | { 929 | char *curname; 930 | char *endp; 931 | char *pname; 932 | 933 | curname = nextp; 934 | while (*nextp && *nextp != separator && !isspace(*nextp)) 935 | { 936 | if (*nextp == '\\') 937 | nextp++; /* ignore next character because of escape */ 938 | nextp++; 939 | } 940 | endp = nextp; 941 | if (curname == nextp) 942 | return false; /* empty unquoted name not allowed */ 943 | 944 | while (isspace(*nextp)) 945 | nextp++; /* skip trailing whitespace */ 946 | 947 | if (*nextp == separator) 948 | { 949 | nextp++; 950 | while (isspace(*nextp)) 951 | nextp++; /* skip leading whitespace for next */ 952 | /* we expect another name, so done remains false */ 953 | } 954 | else if (*nextp == '\0') 955 | done = true; 956 | else 957 | return false; /* invalid syntax */ 958 | 959 | /* Now safe to overwrite separator with a null */ 960 | *endp = '\0'; 961 | 962 | /* 963 | * Finished isolating current name --- add it to list 964 | */ 965 | pname = pstrdup(curname); 966 | *sl = lappend(*sl, pname); 967 | 968 | /* Loop back if we didn't reach end of string */ 969 | } while (!done); 970 | 971 | return true; 972 | } 973 | --------------------------------------------------------------------------------