├── .dockerignore ├── .editorconfig ├── .gitattributes ├── .github └── workflows │ ├── docker-publish.yml │ └── run-tests.yml ├── .gitignore ├── .readthedocs.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.debian ├── Dockerfile.debian-qa ├── GIT-VERSION-GEN ├── LICENSE ├── Makefile ├── README.md ├── ci ├── Dockerfile.docs.template ├── banned.h.sh └── check-docs.mk ├── debian ├── .editorconfig ├── changelog ├── clean ├── control ├── copyright ├── rules ├── source │ └── format ├── tests │ ├── control │ └── copydb └── watch ├── docs ├── .gitignore ├── Makefile ├── concurrency.rst ├── conf.py ├── features.rst ├── include │ ├── clone.rst │ ├── compare-data.rst │ ├── compare-schema.rst │ ├── compare.rst │ ├── copy-blobs.rst │ ├── copy-constraints.rst │ ├── copy-data.rst │ ├── copy-db.rst │ ├── copy-extensions.rst │ ├── copy-indexes.rst │ ├── copy-roles.rst │ ├── copy-schema.rst │ ├── copy-sequences.rst │ ├── copy-table-data.rst │ ├── copy.rst │ ├── dump-roles.rst │ ├── dump-schema.rst │ ├── dump.rst │ ├── follow.rst │ ├── help.rst │ ├── list-collations.rst │ ├── list-databases.rst │ ├── list-depends.rst │ ├── list-extensions.rst │ ├── list-indexes.rst │ ├── list-progress.rst │ ├── list-schema.rst │ ├── list-sequences.rst │ ├── list-table-parts.rst │ ├── list-tables.rst │ ├── list.rst │ ├── pgcopydb.rst │ ├── ping.rst │ ├── restore-parse-list.rst │ ├── restore-post-data.rst │ ├── restore-pre-data.rst │ ├── restore-roles.rst │ ├── restore-schema.rst │ ├── restore.rst │ ├── snapshot.rst │ ├── stream-apply.rst │ ├── stream-catchup.rst │ ├── stream-cleanup.rst │ ├── stream-prefetch.rst │ ├── stream-receive.rst │ ├── stream-replay.rst │ ├── stream-sentinel-get.rst │ ├── stream-sentinel-set-apply.rst │ ├── stream-sentinel-set-endpos.rst │ ├── stream-sentinel-set-prefetch.rst │ ├── stream-sentinel-set-startpos.rst │ ├── stream-sentinel-set.rst │ ├── stream-sentinel-setup.rst │ ├── stream-sentinel.rst │ ├── stream-setup.rst │ ├── stream-transform.rst │ └── stream.rst ├── index.rst ├── install.rst ├── intro.rst ├── make.bat ├── ref │ ├── pgcopydb.rst │ ├── pgcopydb_clone.rst │ ├── pgcopydb_compare.rst │ ├── pgcopydb_config.rst │ ├── pgcopydb_copy.rst │ ├── pgcopydb_dump.rst │ ├── pgcopydb_follow.rst │ ├── pgcopydb_list.rst │ ├── pgcopydb_restore.rst │ ├── pgcopydb_snapshot.rst │ └── pgcopydb_stream.rst ├── requirements.txt ├── resume.rst ├── tutorial.rst └── update-help-messages.sh ├── src └── bin │ ├── Makefile │ ├── lib │ ├── .editorconfig │ ├── README.md │ ├── jenkins │ │ ├── Makefile │ │ ├── README.md │ │ ├── lookup3.c │ │ └── lookup3.h │ ├── libs │ │ ├── docs │ │ │ └── ini.md │ │ └── ini.h │ ├── log │ │ ├── LICENSE │ │ ├── README.md │ │ └── src │ │ │ ├── log.c │ │ │ └── log.h │ ├── parson │ │ ├── .gitignore │ │ ├── LICENSE │ │ ├── Makefile │ │ ├── parson.c │ │ ├── parson.h │ │ ├── tests.c │ │ └── tests │ │ │ ├── test_1_1.txt │ │ │ ├── test_1_2.txt │ │ │ ├── test_1_3.txt │ │ │ ├── test_2.txt │ │ │ ├── test_2_comments.txt │ │ │ ├── test_2_pretty.txt │ │ │ ├── test_2_serialized.txt │ │ │ ├── test_2_serialized_pretty.txt │ │ │ └── test_5.txt │ ├── pg │ │ ├── README.md │ │ ├── dumputils.c │ │ ├── dumputils.h │ │ ├── snprintf.c │ │ ├── snprintf.h │ │ ├── strerror.c │ │ ├── string_utils.c │ │ └── string_utils.h │ ├── sqlite │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── shell.c │ │ ├── sqlite3.c │ │ ├── sqlite3.h │ │ └── sqlite3ext.h │ ├── subcommands.c │ │ ├── commandline.c │ │ ├── commandline.h │ │ └── runprogram.h │ └── uthash │ │ └── uthash.h │ └── pgcopydb │ ├── .gitignore │ ├── Makefile │ ├── blobs.c │ ├── catalog.c │ ├── catalog.h │ ├── cli_clone_follow.c │ ├── cli_common.c │ ├── cli_common.h │ ├── cli_compare.c │ ├── cli_copy.c │ ├── cli_dump.c │ ├── cli_list.c │ ├── cli_list.h │ ├── cli_ping.c │ ├── cli_restore.c │ ├── cli_root.c │ ├── cli_root.h │ ├── cli_sentinel.c │ ├── cli_snapshot.c │ ├── cli_stream.c │ ├── compare.c │ ├── copydb.c │ ├── copydb.h │ ├── copydb_paths.h │ ├── copydb_schema.c │ ├── defaults.h │ ├── dump_restore.c │ ├── env_utils.c │ ├── env_utils.h │ ├── extensions.c │ ├── file_utils.c │ ├── file_utils.h │ ├── filtering.c │ ├── filtering.h │ ├── follow.c │ ├── indexes.c │ ├── ini_implementation.c │ ├── ld_apply.c │ ├── ld_replay.c │ ├── ld_stream.c │ ├── ld_stream.h │ ├── ld_test_decoding.c │ ├── ld_transform.c │ ├── ld_wal2json.c │ ├── lock_utils.c │ ├── lock_utils.h │ ├── main.c │ ├── parsing_utils.c │ ├── parsing_utils.h │ ├── pg_depend_sql.h │ ├── pg_utils.c │ ├── pg_utils.h │ ├── pgcmd.c │ ├── pgcmd.h │ ├── pgsql.c │ ├── pgsql.h │ ├── pgsql_timeline.c │ ├── pgsql_timeline.h │ ├── pgsql_utils.h │ ├── pidfile.c │ ├── pidfile.h │ ├── progress.c │ ├── progress.h │ ├── queue_utils.c │ ├── queue_utils.h │ ├── schema.c │ ├── schema.h │ ├── sentinel.c │ ├── sequences.c │ ├── signals.c │ ├── signals.h │ ├── snapshot.c │ ├── string_utils.c │ ├── string_utils.h │ ├── summary.c │ ├── summary.h │ ├── table-data.c │ └── vacuum.c └── tests ├── .psqlrc ├── .sqliterc ├── Dockerfile.pagila ├── Makefile ├── blobs ├── Dockerfile ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── imgs │ ├── aj-robbie-BuQ1RZckYW4-unsplash.jpg │ ├── bisakha-datta--6SmukZ_w6s-unsplash.jpg │ ├── geran-de-klerk-AX9sJ-mPoL4-unsplash.jpg │ ├── nam-anh-QJbyG6O0ick-unsplash.jpg │ ├── redcharlie-Y--zr3CPaPs-unsplash.jpg │ └── richard-jacobs-8oenpCXktqQ-unsplash.jpg └── import.sql ├── cdc-endpos-between-transaction ├── 000000010000000000000002.json ├── 000000010000000000000002.sql ├── Dockerfile ├── Dockerfile.pg ├── Makefile ├── compose.yaml ├── copydb.sh ├── ddl.sql ├── dml.sql ├── dml1.sql └── dml2.sql ├── cdc-low-level ├── 000000010000000000000002.json ├── 000000010000000000000002.sql ├── Dockerfile ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── ddl.sql ├── dml.sql └── pipeline-deadlock.sql ├── cdc-test-decoding ├── 000000010000000000000002.json ├── 000000010000000000000002.sql ├── Dockerfile ├── Makefile ├── README.md ├── compose.yaml ├── continued-txn.json ├── continued-txn.sql ├── copydb.sh ├── ddl.sql └── dml.sql ├── cdc-wal2json ├── 000000010000000000000002.json ├── 000000010000000000000002.sql ├── Dockerfile ├── Dockerfile.pg ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── ddl.sql ├── dml.sql ├── special-ddl.sql └── special-dml.sql ├── cleanup.mk ├── endpos-in-multi-wal-txn ├── Dockerfile ├── Dockerfile.pg ├── Makefile ├── compose.yaml ├── copydb.sh ├── ddl.sql └── multi-wal-txn.sql ├── extensions ├── .editorconfig ├── Dockerfile ├── Dockerfile.pg ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── countries.sql └── ddl.sql ├── filtering ├── Dockerfile ├── Makefile ├── compose.yaml ├── copydb.sh ├── exclude.ini ├── exclude │ ├── expected │ │ ├── 1-ignore-matview.out │ │ └── 2-sequences.out │ └── sql │ │ ├── 1-ignore-matview.sql │ │ └── 2-sequences.sql ├── extra.sql ├── include.ini └── include │ ├── expected │ ├── 1-empty-rental.out │ ├── 2-payment-does-not-exists.out │ ├── 3-schema-bar-does-not-exists.out │ └── 4-exclude-index.out │ └── sql │ ├── 1-empty-rental.sql │ ├── 2-payment-does-not-exists.sql │ ├── 3-schema-bar-does-not-exists.sql │ └── 4-exclude-index.sql ├── follow-9.6 ├── .dockerignore ├── Dockerfile ├── Dockerfile.inject ├── Dockerfile.pg ├── Makefile ├── README.md ├── allow-logical-rep.sh ├── compose.yaml ├── copydb.sh ├── ddl.sql ├── dml.sql └── inject.sh ├── follow-data-only ├── .dockerignore ├── Dockerfile ├── Dockerfile.inject ├── Dockerfile.pg ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── ddl.sql ├── dml-bufsize.sql ├── dml.sql ├── inject.sh ├── multi-wal-txn.sql └── run-background-traffic.sh ├── follow-wal2json ├── .dockerignore ├── Dockerfile ├── Dockerfile.inject ├── Dockerfile.pg ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── ddl.sql ├── dml.sql └── inject.sh ├── pagila-multi-steps ├── Dockerfile ├── Makefile ├── compose.yaml └── copydb.sh ├── pagila-standby ├── 00_init_replication_slot.sql ├── Dockerfile ├── Makefile ├── compose.yaml └── copydb.sh ├── pagila ├── Dockerfile ├── Makefile ├── compose.yaml └── copydb.sh ├── paths.env ├── postgres.env ├── timescaledb ├── Dockerfile ├── Makefile ├── compose.yml ├── copydb.sh ├── fares.sql ├── nyc_data_rides.10k.csv └── rides.sql ├── unit ├── Dockerfile ├── Makefile ├── README.md ├── compose.yaml ├── copydb.sh ├── expected │ ├── 1-exclusion-constraints.out │ ├── 2-inherit.out │ ├── 3-collations.out │ ├── 3-string-escape.out │ ├── 4-list-table-split.out │ ├── 4-parent-data.out │ ├── 5-attgenerated.out │ └── 6-sequences.out ├── script │ ├── 3-collations.sh │ └── 4-list-table-split.sh ├── setup │ ├── 1-exclusion-constraints.sql │ ├── 10-search-path-index-enum.sql │ ├── 11-check-constraint.sql │ ├── 12-generated-column.sql │ ├── 13-parent-data.sql │ ├── 14-uuid.sql │ ├── 15-attgenerated.sql │ ├── 16-sequences.sql │ ├── 2-inherit.sql │ ├── 3-collations.sql │ ├── 4-list-table-split.sql │ ├── 5-long-index-def.sql │ ├── 6-multiline-table-name.sql │ ├── 7-identity.sql │ ├── 8-string-escape.sql │ ├── 9-table-with-zero-cols.sql │ └── setup.sql └── sql │ ├── 1-exclusion-constraints.sql │ ├── 2-inherit.sql │ ├── 3-string-escape.sql │ ├── 4-parent-data.sql │ ├── 5-attgenerated.sql │ └── 6-sequences.sql └── uris.env /.dockerignore: -------------------------------------------------------------------------------- 1 | .github 2 | 3 | docs/_build 4 | 5 | **/*.o 6 | **/*.bc 7 | **/*.so* 8 | **/.deps/ 9 | 10 | /Debug/ 11 | 12 | # ignore the git version number in release builds 13 | src/bin/pgcopydb/git-version.h 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # rules for all files 5 | # we use tabs with indent size 4 6 | [*] 7 | indent_style = tab 8 | indent_size = 4 9 | tab_width = 4 10 | end_of_line = lf 11 | insert_final_newline = true 12 | charset = utf-8 13 | trim_trailing_whitespace = true 14 | 15 | # Don't change test output files, pngs, test data files and documentation 16 | [*.{out,png,data,md,rst,txt}] 17 | indent_style = unset 18 | indent_size = unset 19 | insert_final_newline = unset 20 | trim_trailing_whitespace = unset 21 | 22 | [*.{py,tex}] 23 | indent_style = space 24 | indent_size = 4 25 | tab_width = 4 26 | 27 | [*.{sql,sh}] 28 | indent_size = unset 29 | indent_style = unset 30 | 31 | [*{.yml,.yaml,Dockerfile*}] 32 | indent_style = space 33 | indent_size = 2 34 | tab_width = 2 35 | 36 | [*.bat] 37 | end_of_line = crlf 38 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * whitespace=space-before-tab,trailing-space 2 | *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4 3 | *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent 4 | *.patch -whitespace 5 | *.pl whitespace=space-before-tab,trailing-space,tabwidth=4 6 | *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof 7 | *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol 8 | *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent 9 | 10 | # Avoid confusing ASCII underlines with leftover merge conflict markers 11 | README conflict-marker-size=32 12 | README.* conflict-marker-size=32 13 | 14 | 15 | # These files are maintained or generated elsewhere. We take them as is. 16 | configure -whitespace 17 | 18 | # all C files (implementation and header) use our style... 19 | *.[ch] citus-style 20 | 21 | # except these exceptions... 22 | src/bin/lib/parson/** -citus-style 23 | src/bin/lib/log/** -citus-style 24 | src/bin/lib/libs/** -citus-style 25 | src/bin/lib/pg/** -citus-style 26 | src/bin/lib/subcommands.c/** -citus-style 27 | src/bin/lib/uthash/** -citus-style 28 | src/bin/lib/jenkins/** -citus-style 29 | src/bin/lib/sqlite/* -citus-style 30 | src/bin/pgcopydb/pg_utils.c -citus-style 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Global excludes across all subdirectories 2 | *.o 3 | *.bc 4 | *.so 5 | *.so.[0-9] 6 | *.so.[0-9].[0-9] 7 | *.sl 8 | *.sl.[0-9] 9 | *.sl.[0-9].[0-9] 10 | *.dylib 11 | *.dll 12 | *.a 13 | *.mo 14 | *.pot 15 | objfiles.txt 16 | .deps/ 17 | *.gcno 18 | *.gcda 19 | *.gcov 20 | *.gcov.out 21 | lcov.info 22 | coverage/ 23 | *.vcproj 24 | *.vcxproj 25 | win32ver.rc 26 | *.exe 27 | lib*dll.def 28 | lib*.pc 29 | 30 | # Local excludes in root directory 31 | /config.log 32 | /config.status 33 | /pgsql.sln 34 | /pgsql.sln.cache 35 | /Debug/ 36 | /Release/ 37 | /autom4te.cache 38 | /Makefile.global 39 | /src/Makefile.custom 40 | /tests/__pycache__/ 41 | /env/ 42 | /GIT-VERSION-FILE 43 | /version 44 | ci/Dockerfile.docs 45 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | python: 21 | install: 22 | - requirements: docs/requirements.txt 23 | -------------------------------------------------------------------------------- /Dockerfile.debian: -------------------------------------------------------------------------------- 1 | FROM debian:sid 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends \ 5 | build-essential \ 6 | devscripts \ 7 | debhelper \ 8 | autotools-dev \ 9 | libedit-dev \ 10 | libgc-dev \ 11 | libpam0g-dev \ 12 | libreadline-dev \ 13 | libselinux1-dev \ 14 | libxslt1-dev \ 15 | libssl-dev \ 16 | libkrb5-dev \ 17 | zlib1g-dev \ 18 | liblz4-dev \ 19 | libpq5 \ 20 | libpq-dev \ 21 | libzstd-dev \ 22 | postgresql-server-dev-all \ 23 | postgresql-common \ 24 | postgresql \ 25 | python3-sphinx \ 26 | lintian \ 27 | curl \ 28 | && rm -rf /var/lib/apt/lists/* 29 | 30 | WORKDIR /usr/src/pgcopydb 31 | 32 | COPY ./Makefile ./ 33 | COPY ./GIT-VERSION-GEN ./ 34 | COPY ./README.md ./ 35 | COPY ./CHANGELOG.md ./ 36 | COPY ./LICENSE ./ 37 | COPY ./src ./src/ 38 | COPY ./docs ./docs/ 39 | 40 | WORKDIR /usr/src 41 | RUN tar czf pgcopydb_0.17.orig.tar.gz pgcopydb 42 | 43 | WORKDIR /usr/src/pgcopydb 44 | 45 | COPY ./debian/ ./debian/ 46 | RUN dpkg-buildpackage --no-sign 47 | 48 | WORKDIR /usr/src 49 | 50 | #RUN lintian --suppress-tags bad-whatis-entry *.changes 51 | -------------------------------------------------------------------------------- /Dockerfile.debian-qa: -------------------------------------------------------------------------------- 1 | FROM debian:sid 2 | 3 | ENV TAR v0.17.tar.gz 4 | ENV ORIG pgcopydb_0.17.orig.tar.gz 5 | ENV WORKDIR /usr/src/pgcopydb-0.17 6 | ENV ARCHIVE https://github.com/dimitri/pgcopydb/archive/refs/tags/ 7 | ENV RELEASE ${ARCHIVE}${TAR} 8 | 9 | RUN apt-get update \ 10 | && apt-get install -y --no-install-recommends \ 11 | build-essential \ 12 | devscripts \ 13 | debhelper \ 14 | autotools-dev \ 15 | libgc-dev \ 16 | libedit-dev \ 17 | libpam0g-dev \ 18 | libreadline-dev \ 19 | libselinux1-dev \ 20 | libxslt1-dev \ 21 | libssl-dev \ 22 | libkrb5-dev \ 23 | zlib1g-dev \ 24 | liblz4-dev \ 25 | libpq5 \ 26 | libpq-dev \ 27 | libzstd-dev \ 28 | postgresql-server-dev-all \ 29 | postgresql-common \ 30 | postgresql \ 31 | python3-sphinx \ 32 | lintian \ 33 | curl \ 34 | && rm -rf /var/lib/apt/lists/* 35 | 36 | WORKDIR /usr/src 37 | 38 | RUN curl -L -o ${TAR} ${RELEASE} 39 | RUN tar xf ${TAR} 40 | RUN mv ${TAR} ${ORIG} 41 | 42 | WORKDIR ${WORKDIR} 43 | COPY ./debian/ ./debian/ 44 | 45 | RUN dpkg-buildpackage --no-sign 46 | 47 | WORKDIR /usr/src 48 | 49 | #RUN lintian --suppress-tags bad-whatis-entry *.changes 50 | -------------------------------------------------------------------------------- /GIT-VERSION-GEN: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | GVF=GIT-VERSION-FILE 4 | DEF_VER=0.17 5 | 6 | LF=' 7 | ' 8 | 9 | # First see if there is a version file (included in release tarballs), 10 | # then try git-describe, then default. 11 | if test -f version 12 | then 13 | VN=$(cat version) || VN="$DEF_VER" 14 | elif test -n "${VERSION}" 15 | then 16 | VN="${VERSION}" || VN="$DEF_VER" 17 | elif test -d ${GIT_DIR:-.git} -o -f .git && 18 | VN=$(git describe --match "v[0-9]*" HEAD 2>/dev/null) && 19 | case "$VN" in 20 | *$LF*) (exit 1) ;; 21 | v[0-9]*) 22 | git update-index -q --refresh 23 | test -z "$(git diff-index --name-only HEAD --)" || 24 | VN="$VN-dirty" ;; 25 | esac 26 | then 27 | VN=$(echo "$VN" | sed -e 's/-/./g'); 28 | else 29 | VN="$DEF_VER" 30 | fi 31 | 32 | VN=$(expr "$VN" : v*'\(.*\)') 33 | 34 | if test -r $GVF 35 | then 36 | VC=$(sed -e 's/^GIT_VERSION = //' <$GVF) 37 | else 38 | VC=unset 39 | fi 40 | test "$VN" = "$VC" || { 41 | echo >&2 "GIT_VERSION = $VN" 42 | echo "GIT_VERSION = $VN" >$GVF 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | 3 | PostgreSQL License 4 | 5 | PostgreSQL is released under the PostgreSQL License, a liberal Open Source 6 | license, similar to the BSD or MIT licenses. 7 | 8 | PostgreSQL Database Management System 9 | (formerly known as Postgres, then as Postgres95) 10 | 11 | Portions Copyright © 1996-2022, The PostgreSQL Global Development Group 12 | 13 | Portions Copyright © 1994, The Regents of the University of California 14 | 15 | Permission to use, copy, modify, and distribute this software and its 16 | documentation for any purpose, without fee, and without a written agreement 17 | is hereby granted, provided that the above copyright notice and this 18 | paragraph and the following two paragraphs appear in all copies. 19 | 20 | IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR 21 | DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING 22 | LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, 23 | EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF 24 | SUCH DAMAGE. 25 | 26 | THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 27 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 28 | FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN 29 | "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO 30 | PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 31 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | TOP := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) 5 | PGCOPYDB ?= $(TOP)src/bin/pgcopydb/pgcopydb 6 | 7 | all: bin ; 8 | 9 | GIT-VERSION-FILE: 10 | @$(SHELL_PATH) ./GIT-VERSION-GEN > /dev/null 2>&1 11 | 12 | bin: GIT-VERSION-FILE 13 | $(MAKE) -C src/bin/ all 14 | 15 | sqlite3: 16 | $(MAKE) -C src/bin/lib/sqlite $@ 17 | 18 | clean: 19 | rm -f GIT-VERSION-FILE 20 | $(MAKE) -C src/bin/ clean 21 | 22 | maintainer-clean: 23 | rm -f GIT-VERSION-FILE 24 | $(MAKE) -C src/bin/ maintainer-clean 25 | rm -f version 26 | 27 | docs: 28 | $(MAKE) -C docs clean man html 29 | 30 | update-docs: bin 31 | bash ./docs/update-help-messages.sh 32 | 33 | check-docs: 34 | cat Dockerfile ci/Dockerfile.docs.template > ci/Dockerfile.docs 35 | docker build --file=ci/Dockerfile.docs --tag test-docs . 36 | docker run test-docs 37 | 38 | test: build 39 | $(MAKE) -C tests all 40 | 41 | tests: test ; 42 | 43 | tests/ci: 44 | sh ./ci/banned.h.sh 45 | 46 | tests/*: build 47 | $(MAKE) -C tests $(notdir $@) 48 | 49 | install: bin 50 | $(MAKE) -C src/bin/ install 51 | 52 | indent: 53 | citus_indent 54 | 55 | build: version 56 | docker build -t pgcopydb . 57 | 58 | echo-version: GIT-VERSION-FILE 59 | @awk '{print $$3}' $< 60 | 61 | version: GIT-VERSION-FILE 62 | @awk '{print $$3}' $< > $@ 63 | @cat $@ 64 | 65 | # debian packages built from the current sources 66 | deb: 67 | docker build -f Dockerfile.debian -t pgcopydb_debian . 68 | 69 | debsh: deb 70 | docker run --rm -it pgcopydb_debian bash 71 | 72 | # debian packages built from latest tag, manually maintained in the Dockerfile 73 | deb-qa: 74 | docker build -f Dockerfile.debian-qa -t pgcopydb_debian_qa . 75 | 76 | debsh-qa: deb-qa 77 | docker run --rm -it pgcopydb_debian_qa bash 78 | 79 | .PHONY: all 80 | .PHONY: bin clean install docs maintainer-clean update-docs 81 | .PHONY: test tests tests/ci tests/* 82 | .PHONY: deb debsh deb-qa debsh-qa 83 | .PHONY: GIT-VERSION-FILE 84 | -------------------------------------------------------------------------------- /ci/Dockerfile.docs.template: -------------------------------------------------------------------------------- 1 | # This file requires a copy of the main Dockerfile in the beginning. 2 | # 3 | # Run `cat Dockerfile ci/Dockerfile.docs.template > ci/Dockerfile.docs` in the 4 | # top directory to generate a working Dockerfile. 5 | # 6 | # Notice that the "build" image that is not defined in here is used as a base 7 | # image. Since Docker does not support multi-stage builds with dependencies 8 | # across different Dockerfiles for now, we need to prepend a copy of the 9 | # original Dockerfile in the beginning of this file. 10 | 11 | FROM --platform=${TARGETPLATFORM} build 12 | RUN dpkg --add-architecture ${TARGETARCH:-arm64} && apt update \ 13 | && apt install -qqy --no-install-suggests --no-install-recommends \ 14 | git \ 15 | && apt clean \ 16 | && rm -rf /var/lib/apt/lists/* 17 | 18 | WORKDIR /usr/src/pgcopydb 19 | COPY ci/check-docs.mk . 20 | COPY docs docs 21 | 22 | ENTRYPOINT ["make", "-f", "check-docs.mk"] 23 | -------------------------------------------------------------------------------- /ci/check-docs.mk: -------------------------------------------------------------------------------- 1 | # check that docs are uptodate 2 | 3 | OK = "Docs are up to date" 4 | KO = "Docs are not up to date, please run 'make update-docs'" 5 | 6 | all: 7 | git init --initial-branch=main . 8 | git config user.email ci@pgcopydb.org 9 | git config user.name "CI" 10 | git add docs 11 | git commit -m "docs before update" 12 | make -s -j$(nproc) update-docs 13 | git add docs 14 | git diff --staged --exit-code || (echo $(KO) && false) && echo $(OK) 15 | -------------------------------------------------------------------------------- /debian/.editorconfig: -------------------------------------------------------------------------------- 1 | # Trick editorconfig to think that there are no rules for formatting in this 2 | # directory. 3 | root = true 4 | -------------------------------------------------------------------------------- /debian/clean: -------------------------------------------------------------------------------- 1 | debian/home/ 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: pgcopydb 2 | Section: database 3 | Priority: optional 4 | Maintainer: Dimitri Fontaine 5 | Uploaders: 6 | Christoph Berg , 7 | Build-Depends: 8 | debhelper-compat (= 13), 9 | libedit-dev, 10 | libgc-dev, 11 | libkrb5-dev, 12 | liblz4-dev, 13 | libncurses-dev, 14 | libpam0g-dev, 15 | libreadline-dev, 16 | libselinux1-dev, 17 | libssl-dev, 18 | libxslt1-dev, 19 | libzstd-dev, 20 | postgresql, 21 | postgresql-server-dev-all (>= 158~), 22 | python3-sphinx, 23 | zlib1g-dev, 24 | Standards-Version: 4.6.0 25 | Homepage: https://github.com/dimitri/pgcopydb 26 | Vcs-Browser: https://github.com/dimitri/pgcopydb 27 | Vcs-Git: https://github.com/dimitri/pgcopydb.git 28 | 29 | Package: pgcopydb 30 | Architecture: any 31 | Depends: 32 | postgresql-client, 33 | ${misc:Depends}, 34 | ${shlibs:Depends}, 35 | Description: Copy an entire PostgreSQL database from source to target 36 | This tool copies an entire PostgreSQL database from source to target. It 37 | implements `pg_dump | pg_restore` on steroids, including advanced concurrency 38 | tricks to make the operation faster. 39 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | VERSION = $(shell dpkg-parsechangelog --show-field version) 4 | 5 | override_dh_auto_build: 6 | $(MAKE) -C src/bin/pgcopydb GIT_VERSION=$(VERSION) git-version.h 7 | $(MAKE) -C src/bin/pgcopydb 8 | $(MAKE) -C docs man 9 | 10 | override_dh_auto_clean: 11 | $(MAKE) -C src/bin/pgcopydb clean 12 | $(MAKE) -C docs clean 13 | 14 | override_dh_auto_test: 15 | PATH=$(CURDIR)/src/bin/pgcopydb:$(PATH) debian/tests/copydb 16 | 17 | override_dh_auto_install: 18 | $(MAKE) -C src/bin/pgcopydb install \ 19 | DESTDIR=$(CURDIR)/debian/pgcopydb \ 20 | BINDIR=/usr/bin 21 | 22 | override_dh_installman: 23 | dh_installman docs/_build/man/* 24 | 25 | %: 26 | dh $@ 27 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /debian/tests/control: -------------------------------------------------------------------------------- 1 | Depends: @, postgresql 2 | Tests: copydb 3 | Restrictions: allow-stderr 4 | -------------------------------------------------------------------------------- /debian/tests/copydb: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -eux 4 | 5 | WORKDIR=$PWD/debian/home 6 | 7 | pg_virtualenv << EOF 8 | set -eux 9 | createdb src 10 | createdb dst 11 | psql -c 'create table foo as select 123+456' src 12 | rm -rf $WORKDIR 13 | HOME=$WORKDIR pgcopydb clone --source "dbname=src" --target "dbname=dst" --dir $WORKDIR 14 | pg_dump -t foo dst 15 | pg_dump -t foo dst | grep 579 16 | EOF 17 | -------------------------------------------------------------------------------- /debian/watch: -------------------------------------------------------------------------------- 1 | version=4 2 | https://github.com/dimitri/pgcopydb/tags .*/v(.*).tar.gz 3 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | .venv 3 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/include/compare-data.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb compare data: Compare source and target data 4 | usage: pgcopydb compare data --source ... 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --json Format the output using JSON 10 | 11 | -------------------------------------------------------------------------------- /docs/include/compare-schema.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb compare schema: Compare source and target schema 4 | usage: pgcopydb compare schema --source ... 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | 10 | -------------------------------------------------------------------------------- /docs/include/compare.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb compare: Compare source and target databases 4 | 5 | Available commands: 6 | pgcopydb compare 7 | schema Compare source and target schema 8 | data Compare source and target data 9 | 10 | -------------------------------------------------------------------------------- /docs/include/copy-blobs.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy blobs: Copy the blob data from the source database to the target 4 | usage: pgcopydb copy blobs --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --large-objects-jobs Number of concurrent Large Objects jobs to run 10 | --drop-if-exists On the target database, drop and create large objects 11 | --restart Allow restarting when temp files exist already 12 | --resume Allow resuming operations after a failure 13 | --not-consistent Allow taking a new snapshot on the source database 14 | --snapshot Use snapshot obtained with pg_export_snapshot 15 | 16 | -------------------------------------------------------------------------------- /docs/include/copy-constraints.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy constraints: Create all the constraints found in the source database in the target 4 | usage: pgcopydb copy constraints --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --filters Use the filters defined in 10 | --restart Allow restarting when temp files exist already 11 | --resume Allow resuming operations after a failure 12 | --not-consistent Allow taking a new snapshot on the source database 13 | 14 | -------------------------------------------------------------------------------- /docs/include/copy-data.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy data: Copy the data section from source to target 4 | usage: pgcopydb copy data --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --table-jobs Number of concurrent COPY jobs to run 10 | --index-jobs Number of concurrent CREATE INDEX jobs to run 11 | --restore-jobs Number of concurrent jobs for pg_restore 12 | --skip-large-objects Skip copying large objects (blobs) 13 | --filters Use the filters defined in 14 | --restart Allow restarting when temp files exist already 15 | --resume Allow resuming operations after a failure 16 | --not-consistent Allow taking a new snapshot on the source database 17 | --snapshot Use snapshot obtained with pg_export_snapshot 18 | 19 | -------------------------------------------------------------------------------- /docs/include/copy-db.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy db: Copy an entire database from source to target 4 | usage: pgcopydb copy db --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --table-jobs Number of concurrent COPY jobs to run 10 | --index-jobs Number of concurrent CREATE INDEX jobs to run 11 | --restore-jobs Number of concurrent jobs for pg_restore 12 | --drop-if-exists On the target database, clean-up from a previous run first 13 | --roles Also copy roles found on source to target 14 | --no-owner Do not set ownership of objects to match the original database 15 | --no-acl Prevent restoration of access privileges (grant/revoke commands). 16 | --no-comments Do not output commands to restore comments 17 | --no-tablespaces Do not output commands to select tablespaces 18 | --skip-large-objects Skip copying large objects (blobs) 19 | --filters Use the filters defined in 20 | --fail-fast Abort early in case of error 21 | --restart Allow restarting when temp files exist already 22 | --resume Allow resuming operations after a failure 23 | --not-consistent Allow taking a new snapshot on the source database 24 | --snapshot Use snapshot obtained with pg_export_snapshot 25 | --use-copy-binary Use the COPY BINARY format for COPY operations 26 | 27 | -------------------------------------------------------------------------------- /docs/include/copy-extensions.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy extensions: Copy the extensions from the source instance to the target instance 4 | usage: pgcopydb copy extensions --source ... --target ... 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --requirements List extensions requirements 10 | 11 | -------------------------------------------------------------------------------- /docs/include/copy-indexes.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy indexes: Create all the indexes found in the source database in the target 4 | usage: pgcopydb copy indexes --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --index-jobs Number of concurrent CREATE INDEX jobs to run 10 | --restore-jobs Number of concurrent jobs for pg_restore 11 | --filters Use the filters defined in 12 | --restart Allow restarting when temp files exist already 13 | --resume Allow resuming operations after a failure 14 | --not-consistent Allow taking a new snapshot on the source database 15 | 16 | -------------------------------------------------------------------------------- /docs/include/copy-roles.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy roles: Copy the roles from the source instance to the target instance 4 | usage: pgcopydb copy roles --source ... --target ... 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --no-role-passwords Do not dump passwords for roles 10 | 11 | -------------------------------------------------------------------------------- /docs/include/copy-schema.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy schema: Copy the database schema from source to target 4 | usage: pgcopydb copy schema --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --filters Use the filters defined in 10 | --restart Allow restarting when temp files exist already 11 | --resume Allow resuming operations after a failure 12 | --not-consistent Allow taking a new snapshot on the source database 13 | --snapshot Use snapshot obtained with pg_export_snapshot 14 | 15 | -------------------------------------------------------------------------------- /docs/include/copy-sequences.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy sequences: Copy the current value from all sequences in database from source to target 4 | usage: pgcopydb copy sequences --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --filters Use the filters defined in 10 | --restart Allow restarting when temp files exist already 11 | --resume Allow resuming operations after a failure 12 | --not-consistent Allow taking a new snapshot on the source database 13 | --snapshot Use snapshot obtained with pg_export_snapshot 14 | 15 | -------------------------------------------------------------------------------- /docs/include/copy-table-data.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy table-data: Copy the data from all tables in database from source to target 4 | usage: pgcopydb copy table-data --source ... --target ... [ --table-jobs ... --index-jobs ... ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --table-jobs Number of concurrent COPY jobs to run 10 | --filters Use the filters defined in 11 | --restart Allow restarting when temp files exist already 12 | --resume Allow resuming operations after a failure 13 | --not-consistent Allow taking a new snapshot on the source database 14 | --snapshot Use snapshot obtained with pg_export_snapshot 15 | 16 | -------------------------------------------------------------------------------- /docs/include/copy.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb copy: Implement the data section of the database copy 4 | 5 | Available commands: 6 | pgcopydb copy 7 | db Copy an entire database from source to target 8 | roles Copy the roles from the source instance to the target instance 9 | extensions Copy the extensions from the source instance to the target instance 10 | schema Copy the database schema from source to target 11 | data Copy the data section from source to target 12 | table-data Copy the data from all tables in database from source to target 13 | blobs Copy the blob data from the source database to the target 14 | sequences Copy the current value from all sequences in database from source to target 15 | indexes Create all the indexes found in the source database in the target 16 | constraints Create all the constraints found in the source database in the target 17 | 18 | -------------------------------------------------------------------------------- /docs/include/dump-roles.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb dump roles: Dump source database roles as custome file in work directory 4 | usage: pgcopydb dump roles --source 5 | 6 | --source Postgres URI to the source database 7 | --target Directory where to save the dump files 8 | --dir Work directory to use 9 | --no-role-passwords Do not dump passwords for roles 10 | 11 | -------------------------------------------------------------------------------- /docs/include/dump-schema.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb dump schema: Dump source database schema as custom files in work directory 4 | usage: pgcopydb dump schema --source 5 | 6 | --source Postgres URI to the source database 7 | --target Directory where to save the dump files 8 | --dir Work directory to use 9 | --skip-extensions Skip restoring extensions 10 | --filters Use the filters defined in 11 | --snapshot Use snapshot obtained with pg_export_snapshot 12 | 13 | -------------------------------------------------------------------------------- /docs/include/dump.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb dump: Dump database objects from a Postgres instance 4 | 5 | Available commands: 6 | pgcopydb dump 7 | schema Dump source database schema as custom files in work directory 8 | roles Dump source database roles as custome file in work directory 9 | 10 | -------------------------------------------------------------------------------- /docs/include/follow.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb follow: Replay changes from the source database to the target database 4 | usage: pgcopydb follow --source ... --target ... 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --filters Use the filters defined in 10 | --restart Allow restarting when temp files exist already 11 | --resume Allow resuming operations after a failure 12 | --not-consistent Allow taking a new snapshot on the source database 13 | --snapshot Use snapshot obtained with pg_export_snapshot 14 | --plugin Output plugin to use (test_decoding, wal2json) 15 | --wal2json-numeric-as-string Print numeric data type as string when using wal2json output plugin 16 | --slot-name Use this Postgres replication slot name 17 | --create-slot Create the replication slot 18 | --origin Use this Postgres replication origin node name 19 | --endpos Stop replaying changes when reaching this LSN 20 | 21 | -------------------------------------------------------------------------------- /docs/include/list-collations.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list collations: List all the source collations to copy 4 | usage: pgcopydb list collations --source ... 5 | 6 | --source Postgres URI to the source database 7 | 8 | -------------------------------------------------------------------------------- /docs/include/list-databases.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list databases: List databases 4 | usage: pgcopydb list databases --source ... 5 | 6 | --source Postgres URI to the source database 7 | 8 | -------------------------------------------------------------------------------- /docs/include/list-depends.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list depends: List all the dependencies to filter-out 4 | usage: pgcopydb list depends --source ... [ --schema-name [ --table-name ] ] 5 | 6 | --source Postgres URI to the source database 7 | --force Force fetching catalogs again 8 | --schema-name Name of the schema where to find the table 9 | --table-name Name of the target table 10 | --filter Use the filters defined in 11 | --list-skipped List only tables that are setup to be skipped 12 | 13 | -------------------------------------------------------------------------------- /docs/include/list-extensions.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list extensions: List all the source extensions to copy 4 | usage: pgcopydb list extensions --source ... 5 | 6 | --source Postgres URI to the source database 7 | --json Format the output using JSON 8 | --available-versions List available extension versions 9 | --requirements List extensions requirements 10 | 11 | -------------------------------------------------------------------------------- /docs/include/list-indexes.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list indexes: List all the indexes to create again after copying the data 4 | usage: pgcopydb list indexes --source ... [ --schema-name [ --table-name ] ] 5 | 6 | --source Postgres URI to the source database 7 | --force Force fetching catalogs again 8 | --schema-name Name of the schema where to find the table 9 | --table-name Name of the target table 10 | --filter Use the filters defined in 11 | --list-skipped List only tables that are setup to be skipped 12 | 13 | -------------------------------------------------------------------------------- /docs/include/list-progress.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list progress: List the progress 4 | usage: pgcopydb list progress --source ... 5 | 6 | --source Postgres URI to the source database 7 | --summary List the summary, requires --json 8 | --json Format the output using JSON 9 | --dir Work directory to use 10 | 11 | -------------------------------------------------------------------------------- /docs/include/list-schema.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list schema: List the schema to migrate, formatted in JSON 4 | usage: pgcopydb list schema --source ... 5 | 6 | --source Postgres URI to the source database 7 | --force Force fetching catalogs again 8 | --filter Use the filters defined in 9 | 10 | -------------------------------------------------------------------------------- /docs/include/list-sequences.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list sequences: List all the source sequences to copy data from 4 | usage: pgcopydb list sequences --source ... 5 | 6 | --source Postgres URI to the source database 7 | --force Force fetching catalogs again 8 | --filter Use the filters defined in 9 | --list-skipped List only tables that are setup to be skipped 10 | 11 | -------------------------------------------------------------------------------- /docs/include/list-table-parts.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list table-parts: List a source table copy partitions 4 | usage: pgcopydb list table-parts --source ... 5 | 6 | --source Postgres URI to the source database 7 | --force Force fetching catalogs again 8 | --schema-name Name of the schema where to find the table 9 | --table-name Name of the target table 10 | --split-tables-larger-than Size threshold to consider partitioning 11 | --split-max-parts Maximum number of jobs for Same-table concurrency 12 | --skip-split-by-ctid Skip the ctid split 13 | --estimate-table-sizes Allow using estimates for relation sizes 14 | 15 | -------------------------------------------------------------------------------- /docs/include/list-tables.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list tables: List all the source tables to copy data from 4 | usage: pgcopydb list tables --source ... 5 | 6 | --source Postgres URI to the source database 7 | --filter Use the filters defined in 8 | --force Force fetching catalogs again 9 | --list-skipped List only tables that are setup to be skipped 10 | --without-pkey List only tables that have no primary key 11 | 12 | -------------------------------------------------------------------------------- /docs/include/list.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb list: List database objects from a Postgres instance 4 | 5 | Available commands: 6 | pgcopydb list 7 | databases List databases 8 | extensions List all the source extensions to copy 9 | collations List all the source collations to copy 10 | tables List all the source tables to copy data from 11 | table-parts List a source table copy partitions 12 | sequences List all the source sequences to copy data from 13 | indexes List all the indexes to create again after copying the data 14 | depends List all the dependencies to filter-out 15 | schema List the schema to migrate, formatted in JSON 16 | progress List the progress 17 | 18 | -------------------------------------------------------------------------------- /docs/include/pgcopydb.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb: pgcopydb tool 4 | usage: pgcopydb [ --verbose --quiet ] 5 | 6 | 7 | Available commands: 8 | pgcopydb 9 | clone Clone an entire database from source to target 10 | fork Clone an entire database from source to target 11 | follow Replay changes from the source database to the target database 12 | snapshot Create and export a snapshot on the source database 13 | + compare Compare source and target databases 14 | + copy Implement the data section of the database copy 15 | + dump Dump database objects from a Postgres instance 16 | + restore Restore database objects into a Postgres instance 17 | + list List database objects from a Postgres instance 18 | + stream Stream changes from the source database 19 | ping Attempt to connect to the source and target instances 20 | help Print help message 21 | version Print pgcopydb version 22 | 23 | -------------------------------------------------------------------------------- /docs/include/ping.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb ping: Attempt to connect to the source and target instances 4 | usage: pgcopydb ping --source ... --target ... 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | 9 | -------------------------------------------------------------------------------- /docs/include/restore-parse-list.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb restore parse-list: Parse pg_restore --list output from custom file 4 | usage: pgcopydb restore parse-list [ ] 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --filters Use the filters defined in 10 | --skip-extensions Skip restoring extensions 11 | --skip-ext-comments Skip restoring COMMENT ON EXTENSION 12 | --restart Allow restarting when temp files exist already 13 | --resume Allow resuming operations after a failure 14 | --not-consistent Allow taking a new snapshot on the source database 15 | 16 | -------------------------------------------------------------------------------- /docs/include/restore-post-data.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb restore post-data: Restore a database post-data schema from custom file to target database 4 | usage: pgcopydb restore post-data --dir [ --source ] --target 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restore-jobs Number of concurrent jobs for pg_restore 10 | --no-owner Do not set ownership of objects to match the original database 11 | --no-acl Prevent restoration of access privileges (grant/revoke commands). 12 | --no-comments Do not output commands to restore comments 13 | --no-tablespaces Do not output commands to select tablespaces 14 | --skip-extensions Skip restoring extensions 15 | --skip-ext-comments Skip restoring COMMENT ON EXTENSION 16 | --filters Use the filters defined in 17 | --restart Allow restarting when temp files exist already 18 | --resume Allow resuming operations after a failure 19 | --not-consistent Allow taking a new snapshot on the source database 20 | 21 | -------------------------------------------------------------------------------- /docs/include/restore-pre-data.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb restore pre-data: Restore a database pre-data schema from custom file to target database 4 | usage: pgcopydb restore pre-data --dir [ --source ] --target 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restore-jobs Number of concurrent jobs for pg_restore 10 | --drop-if-exists On the target database, clean-up from a previous run first 11 | --no-owner Do not set ownership of objects to match the original database 12 | --no-acl Prevent restoration of access privileges (grant/revoke commands). 13 | --no-comments Do not output commands to restore comments 14 | --no-tablespaces Do not output commands to select tablespaces 15 | --skip-extensions Skip restoring extensions 16 | --skip-ext-comments Skip restoring COMMENT ON EXTENSION 17 | --filters Use the filters defined in 18 | --restart Allow restarting when temp files exist already 19 | --resume Allow resuming operations after a failure 20 | --not-consistent Allow taking a new snapshot on the source database 21 | 22 | -------------------------------------------------------------------------------- /docs/include/restore-roles.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb restore roles: Restore database roles from SQL file to target database 4 | usage: pgcopydb restore roles --dir [ --source ] --target 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restore-jobs Number of concurrent jobs for pg_restore 10 | 11 | -------------------------------------------------------------------------------- /docs/include/restore-schema.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb restore schema: Restore a database schema from custom files to target database 4 | usage: pgcopydb restore schema --dir [ --source ] --target 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restore-jobs Number of concurrent jobs for pg_restore 10 | --drop-if-exists On the target database, clean-up from a previous run first 11 | --no-owner Do not set ownership of objects to match the original database 12 | --no-acl Prevent restoration of access privileges (grant/revoke commands). 13 | --no-comments Do not output commands to restore comments 14 | --no-tablespaces Do not output commands to select tablespaces 15 | --filters Use the filters defined in 16 | --restart Allow restarting when temp files exist already 17 | --resume Allow resuming operations after a failure 18 | --not-consistent Allow taking a new snapshot on the source database 19 | 20 | -------------------------------------------------------------------------------- /docs/include/restore.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb restore: Restore database objects into a Postgres instance 4 | 5 | Available commands: 6 | pgcopydb restore 7 | schema Restore a database schema from custom files to target database 8 | pre-data Restore a database pre-data schema from custom file to target database 9 | post-data Restore a database post-data schema from custom file to target database 10 | roles Restore database roles from SQL file to target database 11 | parse-list Parse pg_restore --list output from custom file 12 | 13 | -------------------------------------------------------------------------------- /docs/include/snapshot.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb snapshot: Create and export a snapshot on the source database 4 | usage: pgcopydb snapshot --source ... 5 | 6 | --source Postgres URI to the source database 7 | --dir Work directory to use 8 | --follow Implement logical decoding to replay changes 9 | --plugin Output plugin to use (test_decoding, wal2json) 10 | --wal2json-numeric-as-string Print numeric data type as string when using wal2json output plugin 11 | --slot-name Use this Postgres replication slot name 12 | 13 | -------------------------------------------------------------------------------- /docs/include/stream-apply.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream apply: Apply changes from the source database into the target database 4 | usage: pgcopydb stream apply 5 | 6 | --target Postgres URI to the target database 7 | --dir Work directory to use 8 | --restart Allow restarting when temp files exist already 9 | --resume Allow resuming operations after a failure 10 | --not-consistent Allow taking a new snapshot on the source database 11 | --origin Name of the Postgres replication origin 12 | 13 | -------------------------------------------------------------------------------- /docs/include/stream-catchup.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream catchup: Apply prefetched changes from SQL files to the target database 4 | usage: pgcopydb stream catchup 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restart Allow restarting when temp files exist already 10 | --resume Allow resuming operations after a failure 11 | --not-consistent Allow taking a new snapshot on the source database 12 | --slot-name Stream changes recorded by this slot 13 | --endpos LSN position where to stop receiving changes 14 | --origin Name of the Postgres replication origin 15 | 16 | -------------------------------------------------------------------------------- /docs/include/stream-cleanup.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream cleanup: Cleanup source and target systems for logical decoding 4 | usage: pgcopydb stream cleanup 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --restart Allow restarting when temp files exist already 9 | --resume Allow resuming operations after a failure 10 | --not-consistent Allow taking a new snapshot on the source database 11 | --snapshot Use snapshot obtained with pg_export_snapshot 12 | --slot-name Stream changes recorded by this slot 13 | --origin Name of the Postgres replication origin 14 | 15 | -------------------------------------------------------------------------------- /docs/include/stream-prefetch.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream prefetch: Stream JSON changes from the source database and transform them to SQL 4 | usage: pgcopydb stream prefetch 5 | 6 | --source Postgres URI to the source database 7 | --dir Work directory to use 8 | --restart Allow restarting when temp files exist already 9 | --resume Allow resuming operations after a failure 10 | --not-consistent Allow taking a new snapshot on the source database 11 | --slot-name Stream changes recorded by this slot 12 | --endpos LSN position where to stop receiving changes 13 | -------------------------------------------------------------------------------- /docs/include/stream-receive.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream receive: Stream changes from the source database 4 | usage: pgcopydb stream receive 5 | 6 | --source Postgres URI to the source database 7 | --dir Work directory to use 8 | --to-stdout Stream logical decoding messages to stdout 9 | --restart Allow restarting when temp files exist already 10 | --resume Allow resuming operations after a failure 11 | --not-consistent Allow taking a new snapshot on the source database 12 | --slot-name Stream changes recorded by this slot 13 | --endpos LSN position where to stop receiving changes 14 | -------------------------------------------------------------------------------- /docs/include/stream-replay.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream replay: Replay changes from the source to the target database, live 4 | usage: pgcopydb stream replay 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restart Allow restarting when temp files exist already 10 | --resume Allow resuming operations after a failure 11 | --not-consistent Allow taking a new snapshot on the source database 12 | --slot-name Stream changes recorded by this slot 13 | --endpos LSN position where to stop receiving changes 14 | --origin Name of the Postgres replication origin 15 | 16 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-get.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel get: Get the sentinel table values 4 | usage: pgcopydb stream sentinel get 5 | 6 | --json Format the output using JSON 7 | --startpos Get only the startpos value 8 | --endpos Get only the endpos value 9 | --apply Get only the apply value 10 | --write-lsn Get only the write LSN value 11 | --flush-lsn Get only the flush LSN value 12 | --replay-lsn Get only the replay LSN value 13 | 14 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-set-apply.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel set apply: Set the sentinel apply mode 4 | usage: pgcopydb stream sentinel set apply 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-set-endpos.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel set endpos: Set the sentinel end position LSN 4 | usage: pgcopydb stream sentinel set endpos [ --source ... ] [ | --current ] 5 | 6 | --source Postgres URI to the source database 7 | --current Use pg_current_wal_flush_lsn() as the endpos 8 | 9 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-set-prefetch.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel set prefetch: Set the sentinel prefetch mode 4 | usage: pgcopydb stream sentinel set prefetch 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-set-startpos.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel set startpos: Set the sentinel start position LSN 4 | usage: pgcopydb stream sentinel set startpos 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-set.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel set: Set the sentinel table values 4 | 5 | Available commands: 6 | pgcopydb stream sentinel set 7 | startpos Set the sentinel start position LSN 8 | endpos Set the sentinel end position LSN 9 | apply Set the sentinel apply mode 10 | prefetch Set the sentinel prefetch mode 11 | 12 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel-setup.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel setup: Setup the sentinel table 4 | usage: pgcopydb stream sentinel setup 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/include/stream-sentinel.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream sentinel: Maintain a sentinel table 4 | 5 | Available commands: 6 | pgcopydb stream sentinel 7 | setup Setup the sentinel table 8 | get Get the sentinel table values 9 | + set Set the sentinel table values 10 | 11 | -------------------------------------------------------------------------------- /docs/include/stream-setup.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream setup: Setup source and target systems for logical decoding 4 | usage: pgcopydb stream setup 5 | 6 | --source Postgres URI to the source database 7 | --target Postgres URI to the target database 8 | --dir Work directory to use 9 | --restart Allow restarting when temp files exist already 10 | --resume Allow resuming operations after a failure 11 | --not-consistent Allow taking a new snapshot on the source database 12 | --snapshot Use snapshot obtained with pg_export_snapshot 13 | --plugin Output plugin to use (test_decoding, wal2json) 14 | --wal2json-numeric-as-string Print numeric data type as string when using wal2json output plugin 15 | --slot-name Stream changes recorded by this slot 16 | --origin Name of the Postgres replication origin 17 | 18 | -------------------------------------------------------------------------------- /docs/include/stream-transform.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream transform: Transform changes from the source database into SQL commands 4 | usage: pgcopydb stream transform 5 | 6 | --target Postgres URI to the target database 7 | --dir Work directory to use 8 | --restart Allow restarting when temp files exist already 9 | --resume Allow resuming operations after a failure 10 | --not-consistent Allow taking a new snapshot on the source database 11 | 12 | -------------------------------------------------------------------------------- /docs/include/stream.rst: -------------------------------------------------------------------------------- 1 | :: 2 | 3 | pgcopydb stream: Stream changes from the source database 4 | 5 | Available commands: 6 | pgcopydb stream 7 | setup Setup source and target systems for logical decoding 8 | cleanup Cleanup source and target systems for logical decoding 9 | prefetch Stream JSON changes from the source database and transform them to SQL 10 | catchup Apply prefetched changes from SQL files to the target database 11 | replay Replay changes from the source to the target database, live 12 | + sentinel Maintain a sentinel table 13 | receive Stream changes from the source database 14 | transform Transform changes from the source database into SQL commands 15 | apply Apply changes from the source database into the target database 16 | 17 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.https://www.sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | # Defining the exact version will make sure things don't break 2 | sphinx 3 | sphinx_rtd_theme 4 | readthedocs-sphinx-search 5 | -------------------------------------------------------------------------------- /src/bin/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | all: pgcopydb ; 5 | 6 | pgcopydb: 7 | $(MAKE) -C pgcopydb pgcopydb 8 | 9 | clean: 10 | $(MAKE) -C pgcopydb clean 11 | 12 | maintainer-clean: 13 | $(MAKE) -C pgcopydb clean 14 | $(MAKE) -C lib/sqlite clean 15 | 16 | install: $(pgcopydb) 17 | $(MAKE) -C pgcopydb install 18 | 19 | .PHONY: all pgcopydb install clean 20 | -------------------------------------------------------------------------------- /src/bin/lib/.editorconfig: -------------------------------------------------------------------------------- 1 | # Trick editorconfig to think that there are no rules for formatting in this 2 | # directory so that the libraries are not formatted. 3 | root = true 4 | -------------------------------------------------------------------------------- /src/bin/lib/README.md: -------------------------------------------------------------------------------- 1 | # Vendored-in librairies 2 | 3 | ## log.c 4 | 5 | A very simple lib for handling logs in C is available at 6 | 7 | https://github.com/rxi/log.c 8 | 9 | It says that 10 | 11 | log.c and log.h should be dropped into an existing project and compiled 12 | along with it. 13 | 14 | So this directory contains a _vendored-in_ copy of the log.c repository. 15 | 16 | ## SubCommands.c 17 | 18 | The single-header library is used to implement parsing "modern" command lines. 19 | 20 | ## JSON 21 | 22 | The parson librairy at https://github.com/kgabis/parson is a single C file 23 | and MIT licenced. It allows parsing from and serializing to JSON. 24 | 25 | ## Configuration file parsing 26 | 27 | We utilize the "ini.h" ini-file reader from https://github.com/mattiasgustavsson/libs 28 | 29 | ## pg 30 | 31 | We vendor-in some code from the Postgres project at 32 | https://git.postgresql.org/gitweb/?p=postgresql.git;a=summary. This code is 33 | licenced under The PostgreSQL Licence, a derivative of the BSD licence. 34 | 35 | ## uthash 36 | 37 | A hash in C that's available at 38 | 39 | https://github.com/troydhanson/uthash 40 | 41 | It says that 42 | 43 | All you need to do is copy the header file into your project, and include 44 | it. Since uthash is a header file only, there is no library code to link 45 | against. 46 | 47 | This directory contains only the `uthash.h` file, which implements hash tables. 48 | -------------------------------------------------------------------------------- /src/bin/lib/jenkins/Makefile: -------------------------------------------------------------------------------- 1 | all: clean lookup3 test ; 2 | 3 | test: lookup3 4 | ./lookup3 5 | 6 | lookup3.o: lookup3.c 7 | $(CC) -c -o $@ $< 8 | 9 | lookup3: lookup3.c 10 | $(CC) -D SELF_TEST -Wno-implicit-fallthrough -Wno-format -o $@ $< 11 | 12 | clean: 13 | rm -f lookup3.o lookup3 14 | 15 | .PHONY: all clean test 16 | -------------------------------------------------------------------------------- /src/bin/lib/jenkins/README.md: -------------------------------------------------------------------------------- 1 | # Jenkins hash function 2 | 3 | This directory contain public domain code from Bob Jenkins. The code is used 4 | to compute a hash value from a string. 5 | 6 | References: 7 | 8 | https://en.wikipedia.org/wiki/Jenkins_hash_function 9 | 10 | Code origin: 11 | 12 | http://burtleburtle.net/bob/c/lookup3.c 13 | -------------------------------------------------------------------------------- /src/bin/lib/log/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 rxi 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/bin/lib/log/README.md: -------------------------------------------------------------------------------- 1 | # log.c 2 | A simple logging library implemented in C99 3 | 4 | ![screenshot](https://cloud.githubusercontent.com/assets/3920290/23831970/a2415e96-0723-11e7-9886-f8f5d2de60fe.png) 5 | 6 | 7 | ## Usage 8 | **[log.c](src/log.c?raw=1)** and **[log.h](src/log.h?raw=1)** should be dropped 9 | into an existing project and compiled along with it. The library provides 6 10 | function-like macros for logging: 11 | 12 | ```c 13 | log_trace(const char *fmt, ...); 14 | log_debug(const char *fmt, ...); 15 | log_info(const char *fmt, ...); 16 | log_warn(const char *fmt, ...); 17 | log_error(const char *fmt, ...); 18 | log_fatal(const char *fmt, ...); 19 | ``` 20 | 21 | Each function takes a printf format string followed by additional arguments: 22 | 23 | ```c 24 | log_trace("Hello %s", "world") 25 | ``` 26 | 27 | Resulting in a line with the given format printed to stderr: 28 | 29 | ``` 30 | 20:18:26 TRACE src/main.c:11: Hello world 31 | ``` 32 | 33 | 34 | #### log_set_quiet(int enable) 35 | Quiet-mode can be enabled by passing `1` to the `log_set_quiet()` function. 36 | While this mode is enabled the library will not output anything to stderr, but 37 | will continue to write to the file if one is set. 38 | 39 | 40 | #### log_set_level(int level) 41 | The current logging level can be set by using the `log_set_level()` function. 42 | All logs below the given level will be ignored. By default the level is 43 | `LOG_TRACE`, such that nothing is ignored. 44 | 45 | 46 | #### log_set_fp(FILE *fp) 47 | A file pointer where the log should be written can be provided to the library by 48 | using the `log_set_fp()` function. The data written to the file output is 49 | of the following format: 50 | 51 | ``` 52 | 2047-03-11 20:18:26 TRACE src/main.c:11: Hello world 53 | ``` 54 | 55 | 56 | #### log_set_lock(log_LockFn fn) 57 | If the log will be written to from multiple threads a lock function can be set. 58 | The function is passed a `udata` value (set by `log_set_udata()`) and the 59 | integer `1` if the lock should be acquired or `0` if the lock should be 60 | released. 61 | 62 | 63 | #### log_use_colors(int enable) 64 | Colors in the log output can be enabled by passing `1` to the `log_use_colors()` 65 | function. 66 | 67 | 68 | ## License 69 | This library is free software; you can redistribute it and/or modify it under 70 | the terms of the MIT license. See [LICENSE](LICENSE) for details. 71 | -------------------------------------------------------------------------------- /src/bin/lib/log/src/log.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2017 rxi 3 | * 4 | * This library is free software; you can redistribute it and/or modify it 5 | * under the terms of the MIT license. See `log.c` for details. 6 | */ 7 | 8 | #ifndef LOG_H 9 | #define LOG_H 10 | 11 | #include 12 | #include 13 | 14 | #define LOG_VERSION "0.1.3" 15 | 16 | #define LOG_TFORMAT_LONG "%Y-%m-%d %H:%M:%S" 17 | #define LOG_TFORMAT_SHORT "%H:%M:%S" 18 | 19 | /* used for JSON logs format "log" message */ 20 | #define LOG_BUFSIZE 2048 21 | 22 | typedef void (*log_LockFn)(void *udata, int lock); 23 | 24 | enum { 25 | LOG_TRACE, 26 | LOG_SQLITE, 27 | LOG_DEBUG, 28 | LOG_SQL, 29 | LOG_NOTICE, 30 | LOG_INFO, 31 | LOG_WARN, 32 | LOG_ERROR, 33 | LOG_FATAL 34 | }; 35 | 36 | #define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) 37 | #define log_sqlite(...) log_log(LOG_SQLITE, __FILE__, __LINE__, __VA_ARGS__) 38 | #define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) 39 | #define log_sql(...) log_log(LOG_SQL, __FILE__, __LINE__, __VA_ARGS__) 40 | #define log_notice(...) log_log(LOG_NOTICE, __FILE__, __LINE__, __VA_ARGS__) 41 | #define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) 42 | #define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) 43 | #define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) 44 | #define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) 45 | 46 | #define log_level(level, ...) log_log(level, __FILE__, __LINE__, __VA_ARGS__) 47 | 48 | void log_set_udata(void *udata); 49 | void log_set_lock(log_LockFn fn); 50 | void log_set_fp(FILE *fp); 51 | void log_set_level(int level); 52 | int log_get_level(void); 53 | void log_set_quiet(int enable); 54 | void log_use_colors(int enable); 55 | void log_use_json(int enable); 56 | void log_use_json_file(int enable); 57 | void log_show_file_line(int enable); 58 | void log_set_tformat(const char *tformat); 59 | 60 | void log_log(int level, const char *file, int line, const char *fmt, ...) 61 | __attribute__((format(printf, 4, 5))); 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/bin/lib/parson/.gitignore: -------------------------------------------------------------------------------- 1 | testcpp 2 | testcpp.* 3 | test.dSYM 4 | -------------------------------------------------------------------------------- /src/bin/lib/parson/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2012 - 2020 Krzysztof Gabis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/bin/lib/parson/Makefile: -------------------------------------------------------------------------------- 1 | CC = gcc 2 | CFLAGS = -O0 -g -Wall -Wextra -std=c89 -pedantic-errors -DTESTS_MAIN 3 | 4 | CPPC = g++ 5 | CPPFLAGS = -O0 -g -Wall -Wextra -DTESTS_MAIN 6 | 7 | all: test testcpp test_hash_collisions 8 | 9 | .PHONY: test testcpp test_hash_collisions 10 | test: tests.c parson.c 11 | $(CC) $(CFLAGS) -o $@ tests.c parson.c 12 | ./$@ 13 | 14 | testcpp: tests.c parson.c 15 | $(CPPC) $(CPPFLAGS) -o $@ tests.c parson.c 16 | ./$@ 17 | 18 | test_hash_collisions: tests.c parson.c 19 | $(CC) $(CFLAGS) -DPARSON_FORCE_HASH_COLLISIONS -o $@ tests.c parson.c 20 | ./$@ 21 | 22 | clean: 23 | rm -f test *.o 24 | 25 | -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_1_1.txt: -------------------------------------------------------------------------------- 1 | 2 | [ 3 | "JSON Test Pattern pass1", 4 | {"object with 1 member":["array with 1 element"]}, 5 | {}, 6 | [], 7 | -42, 8 | true, 9 | false, 10 | null, 11 | { 12 | "integer": 1234567890, 13 | "real": -9876.543210, 14 | "e": 0.123456789e-12, 15 | "E": 1.234567890E+34, 16 | "": 23456789012E66, 17 | "zero": 0, 18 | "one": 1, 19 | "space": " ", 20 | "quote": "\"", 21 | "backslash": "\\", 22 | "controls": "\b\f\n\r\t", 23 | "slash": "/ & \/", 24 | "alpha": "abcdefghijklmnopqrstuvwyz", 25 | "ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ", 26 | "digit": "0123456789", 27 | "0123456789": "digit", 28 | "special": "`1~!@#$%^&*()_+-={':[,]}|;.?", 29 | "nullchar": "abc\u0000def", 30 | "hex": "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A", 31 | "true": true, 32 | "false": false, 33 | "null": null, 34 | "array":[ ], 35 | "object":{ }, 36 | "address": "50 St. James Street", 37 | "url": "http://www.JSON.org/", 38 | "comment": "// /* */": " ", 40 | " s p a c e d " :[1,2 , 3 41 | 42 | , 43 | 44 | 4 , 5 , 6 ,7 ],"compact":[1,2,3,4,5,6,7], 45 | "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", 46 | "quotes": "" \u0022 %22 0x22 034 "", 47 | "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" 48 | : "A key can be any string" 49 | }, 50 | 0.5 ,98.6 51 | , 52 | 99.44 53 | , 54 | 55 | 1066, 56 | 1e1, 57 | 0.1e1, 58 | 1e-1, 59 | 1e00,2e+00,2e-00 60 | ,"rosebud"] -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_1_3.txt: -------------------------------------------------------------------------------- 1 | { 2 | "JSON Test Pattern pass3": { 3 | "The outermost value": "must be an object or array.", 4 | "In this test": "It is an object." 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_2.txt: -------------------------------------------------------------------------------- 1 | { 2 | "string" : "lorem ipsum", 3 | "utf string" : "\u006corem\u0020ipsum", 4 | "utf-8 string": "あいうえお", 5 | "surrogate string": "lorem\uD834\uDD1Eipsum\uD834\uDF67lorem", 6 | "string with null": "abc\u0000def", 7 | "positive one" : 1, 8 | "negative one" : -1, 9 | "pi" : 3.14, 10 | "hard to parse number" : -3.14e-4, 11 | "big int": 2147483647, 12 | "big uint": 4294967295, 13 | "double underflow": 6.9041432094973937e-310, 14 | "boolean true" : true, 15 | "boolean false" : false, 16 | "null" : null, 17 | "string array" : ["lorem", "ipsum"], 18 | "x^2 array" : [0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100], 19 | "/*" : null, 20 | "object" : { "nested string" : "str", 21 | "nested true" : true, 22 | "nested false" : false, 23 | "nested null" : null, 24 | "nested number" : 123, 25 | "nested array" : ["lorem", "ipsum"], 26 | "nested object" : {"lorem": "ipsum"} 27 | }, 28 | "*/" : null, 29 | "/**/" : "comment", 30 | "//" : "comment", 31 | "url" : "https:\/\/www.example.com\/search?q=12345", 32 | "escaped chars" : "\" \\ \/", 33 | "empty object" : {}, 34 | "empty array" : [] 35 | } 36 | -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_2_comments.txt: -------------------------------------------------------------------------------- 1 | /* 2 | *Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor 3 | *ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud 4 | *dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. 5 | */ 6 | // Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor 7 | { /* lorem ipsum */ 8 | "string" : "lorem ipsum", // lorem ipsum 9 | "utf string" : "\u006corem\u0020ipsum", // lorem ipsum // 10 | "utf-8 string": "あいうえお", // /* lorem ipsum */ 11 | "surrogate string": "lorem\uD834\uDD1Eipsum\uD834\uDF67lorem", 12 | "string with null": "abc\u0000def", 13 | "positive one" : 1, 14 | "negative one" : -1, 15 | "pi" : 3.14, 16 | "hard to parse number" : -3.14e-4, 17 | "big int": 2147483647, 18 | "big uint": 4294967295, 19 | "double underflow": 6.9041432094973937e-310, 20 | "boolean true" : true, 21 | "boolean false" : false, 22 | "null" : null, 23 | "string array" : ["lorem",/*in array*/"ipsum"], 24 | "x^2 array" : [0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100], 25 | /* "x^2 array" : [], 26 | */ 27 | "/*" : null, 28 | "object" : { "nested string" : "str", 29 | "nested true" : /* lorem ipsum */ true, 30 | "nested false" : false, 31 | "nested null" : null, // lorem ipsum 32 | "nested number" : 123, 33 | "nested array" : ["lorem", "ipsum"], 34 | "nested object" : {"lorem": "ipsum"} 35 | }, 36 | "*/" : null, 37 | "/**/" : "comment", 38 | "//" : "comment", 39 | "url" : "https:\/\/www.example.com\/search?q=12345", 40 | "escaped chars" : "\" \\ \/", 41 | "empty object" : {}, 42 | "empty array" : [] 43 | } 44 | /**/ 45 | // -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_2_pretty.txt: -------------------------------------------------------------------------------- 1 | { 2 | "string": "lorem ipsum", 3 | "utf string": "lorem ipsum", 4 | "utf-8 string": "あいうえお", 5 | "surrogate string": "lorem𝄞ipsum𝍧lorem", 6 | "string with null": "abc\u0000def", 7 | "positive one": 1, 8 | "negative one": -1, 9 | "pi": 3.1400000000000001, 10 | "hard to parse number": -0.00031399999999999999, 11 | "big int": 2147483647, 12 | "big uint": 4294967295, 13 | "double underflow": 6.9041432094973937e-310, 14 | "boolean true": true, 15 | "boolean false": false, 16 | "null": null, 17 | "string array": [ 18 | "lorem", 19 | "ipsum" 20 | ], 21 | "x^2 array": [ 22 | 0, 23 | 1, 24 | 4, 25 | 9, 26 | 16, 27 | 25, 28 | 36, 29 | 49, 30 | 64, 31 | 81, 32 | 100 33 | ], 34 | "\/*": null, 35 | "object": { 36 | "nested string": "str", 37 | "nested true": true, 38 | "nested false": false, 39 | "nested null": null, 40 | "nested number": 123, 41 | "nested array": [ 42 | "lorem", 43 | "ipsum" 44 | ] 45 | }, 46 | "*\/": null, 47 | "\/**\/": "comment", 48 | "\/\/": "comment", 49 | "url": "https:\/\/www.example.com\/search?q=12345", 50 | "escaped chars": "\" \\ \/", 51 | "empty object": {}, 52 | "empty array": [] 53 | } -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_2_serialized.txt: -------------------------------------------------------------------------------- 1 | {"string":"lorem ipsum","utf string":"lorem ipsum","utf-8 string":"あいうえお","surrogate string":"lorem𝄞ipsum𝍧lorem","string with null":"abc\u0000def","positive one":1,"negative one":-1,"pi":3.1400000000000001,"hard to parse number":-0.00031399999999999999,"big int":2147483647,"big uint":4294967295,"double underflow":6.9041432094973937e-310,"boolean true":true,"boolean false":false,"null":null,"string array":["lorem","ipsum"],"x^2 array":[0,1,4,9,16,25,36,49,64,81,100],"\/*":null,"object":{"nested string":"str","nested true":true,"nested false":false,"nested null":null,"nested number":123,"nested array":["lorem","ipsum"],"nested object":{"lorem":"ipsum"}},"*\/":null,"\/**\/":"comment","\/\/":"comment","url":"https:\/\/www.example.com\/search?q=12345","escaped chars":"\" \\ \/","empty object":{},"empty array":[]} -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_2_serialized_pretty.txt: -------------------------------------------------------------------------------- 1 | { 2 | "string": "lorem ipsum", 3 | "utf string": "lorem ipsum", 4 | "utf-8 string": "あいうえお", 5 | "surrogate string": "lorem𝄞ipsum𝍧lorem", 6 | "string with null": "abc\u0000def", 7 | "positive one": 1, 8 | "negative one": -1, 9 | "pi": 3.1400000000000001, 10 | "hard to parse number": -0.00031399999999999999, 11 | "big int": 2147483647, 12 | "big uint": 4294967295, 13 | "double underflow": 6.9041432094973937e-310, 14 | "boolean true": true, 15 | "boolean false": false, 16 | "null": null, 17 | "string array": [ 18 | "lorem", 19 | "ipsum" 20 | ], 21 | "x^2 array": [ 22 | 0, 23 | 1, 24 | 4, 25 | 9, 26 | 16, 27 | 25, 28 | 36, 29 | 49, 30 | 64, 31 | 81, 32 | 100 33 | ], 34 | "\/*": null, 35 | "object": { 36 | "nested string": "str", 37 | "nested true": true, 38 | "nested false": false, 39 | "nested null": null, 40 | "nested number": 123, 41 | "nested array": [ 42 | "lorem", 43 | "ipsum" 44 | ] 45 | }, 46 | "*\/": null, 47 | "\/**\/": "comment", 48 | "\/\/": "comment", 49 | "url": "https:\/\/www.example.com\/search?q=12345", 50 | "escaped chars": "\" \\ \/", 51 | "empty object": {}, 52 | "empty array": [] 53 | } -------------------------------------------------------------------------------- /src/bin/lib/parson/tests/test_5.txt: -------------------------------------------------------------------------------- 1 | { 2 | "first": "John", 3 | "last": "Doe", 4 | "age": 25, 5 | "registered": true, 6 | "interests": [ "Reading", "Mountain Biking" ], 7 | "favorites": { 8 | "color": "blue", 9 | "sport": "running" 10 | }, 11 | "utf string" : "\u006corem\u0020ipsum", 12 | "utf-8 string": "あいうえお", 13 | "surrogate string": "lorem\uD834\uDD1Eipsum\uD834\uDF67lorem", 14 | "string with null": "abc\u0000def", 15 | "windows path": "C:\\Windows\\Path" 16 | } -------------------------------------------------------------------------------- /src/bin/lib/pg/README.md: -------------------------------------------------------------------------------- 1 | # Postgres code 2 | 3 | This directory contains PostgreSQL code that we have vendored-in. 4 | 5 | 6 | Some parts of pg_dump has been imported to deal with the lack of a libpgdump 7 | interface. In doing that, dependencies to common code (such as 8 | ScanKeywordLookup and ScanKeywordCategories) has been removed, including 9 | call sites to fmdId (now always double-quoted). 10 | -------------------------------------------------------------------------------- /src/bin/lib/pg/dumputils.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Utility routines for SQL dumping 4 | * 5 | * Basically this is stuff that is useful in both pg_dump and pg_dumpall. 6 | * 7 | * 8 | * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group 9 | * Portions Copyright (c) 1994, Regents of the University of California 10 | * 11 | * src/bin/pg_dump/dumputils.h 12 | * 13 | *------------------------------------------------------------------------- 14 | */ 15 | #ifndef DUMPUTILS_H 16 | #define DUMPUTILS_H 17 | 18 | #include 19 | 20 | #include "libpq-fe.h" 21 | #include "pqexpbuffer.h" 22 | 23 | /* 24 | * Preferred strftime(3) format specifier for printing timestamps in pg_dump 25 | * and friends. 26 | * 27 | * We don't print the timezone on Windows, because the names are long and 28 | * localized, which means they may contain characters in various random 29 | * encodings; this has been seen to cause encoding errors when reading the 30 | * dump script. Think not to get around that by using %z, because 31 | * (1) %z is not portable to pre-C99 systems, and 32 | * (2) %z doesn't actually act differently from %Z on Windows anyway. 33 | */ 34 | #ifndef WIN32 35 | #define PGDUMP_STRFTIME_FMT "%Y-%m-%d %H:%M:%S %Z" 36 | #else 37 | #define PGDUMP_STRFTIME_FMT "%Y-%m-%d %H:%M:%S" 38 | #endif 39 | 40 | 41 | extern bool variable_is_guc_list_quote(const char *name); 42 | 43 | extern bool SplitGUCList(char *rawstring, char separator, 44 | char ***namelist); 45 | 46 | extern void makeAlterConfigCommand(PGconn *conn, const char *configitem, 47 | const char *type, const char *name, 48 | const char *type2, const char *name2, 49 | PQExpBuffer buf); 50 | 51 | #endif /* DUMPUTILS_H */ 52 | -------------------------------------------------------------------------------- /src/bin/lib/pg/snprintf.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * port.h 4 | * Header for src/port/ compatibility functions. 5 | * 6 | * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group 7 | * Portions Copyright (c) 1994, Regents of the University of California 8 | * 9 | * src/include/port.h 10 | * 11 | *------------------------------------------------------------------------- 12 | */ 13 | #ifndef PG_SNPRINTF_H 14 | #define PG_SNPRINTF_H 15 | 16 | #include "postgres_fe.h" 17 | 18 | #ifndef USE_REPL_SNPRINTF 19 | 20 | int pg_vsnprintf(char *str, size_t count, const char *fmt, va_list args); 21 | int pg_snprintf(char *str, size_t count, const char *fmt,...) 22 | __attribute__((format(printf, 3, 4))); 23 | int pg_vsprintf(char *str, const char *fmt, va_list args); 24 | int pg_sprintf(char *str, const char *fmt,...) 25 | __attribute__((format(printf, 2, 3))); 26 | int pg_vfprintf(FILE *stream, const char *fmt, va_list args); 27 | int pg_fprintf(FILE *stream, const char *fmt,...) 28 | __attribute__((format(printf, 2, 3))); 29 | int pg_vprintf(const char *fmt, va_list args); 30 | int pg_printf(const char *fmt,...) 31 | __attribute__((format(printf, 1, 2))); 32 | 33 | /* This is also provided by snprintf.c */ 34 | int pg_strfromd(char *str, size_t count, int precision, double value); 35 | 36 | /* Replace strerror() with our own, somewhat more robust wrapper */ 37 | extern char *pg_strerror(int errnum); 38 | #define strerror pg_strerror 39 | 40 | /* Likewise for strerror_r(); note we prefer the GNU API for that */ 41 | extern char *pg_strerror_r(int errnum, char *buf, size_t buflen); 42 | #define strerror_r pg_strerror_r 43 | #define PG_STRERROR_R_BUFLEN 256 /* Recommended buffer size for strerror_r */ 44 | 45 | #endif /* USE_REPL_SNPRINTF */ 46 | 47 | #endif /* PG_SNPRINTF_H */ 48 | -------------------------------------------------------------------------------- /src/bin/lib/pg/string_utils.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * String-processing utility routines for frontend code 4 | * 5 | * Utility functions that interpret backend output or quote strings for 6 | * assorted contexts. 7 | * 8 | * 9 | * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group 10 | * Portions Copyright (c) 1994, Regents of the University of California 11 | * 12 | * src/include/fe_utils/string_utils.h 13 | * 14 | *------------------------------------------------------------------------- 15 | */ 16 | #ifndef STRING_UTILS_H 17 | #define STRING_UTILS_H 18 | 19 | #include "libpq-fe.h" 20 | #include "pqexpbuffer.h" 21 | 22 | extern void appendStringLiteral(PQExpBuffer buf, const char *str, 23 | int encoding, bool std_strings); 24 | extern void appendStringLiteralConn(PQExpBuffer buf, const char *str, 25 | PGconn *conn); 26 | 27 | #endif /* STRING_UTILS_H */ 28 | -------------------------------------------------------------------------------- /src/bin/lib/sqlite/.gitignore: -------------------------------------------------------------------------------- 1 | sqlite3 2 | -------------------------------------------------------------------------------- /src/bin/lib/sqlite/Makefile: -------------------------------------------------------------------------------- 1 | # See https://sqlite.org/howtocompile.html 2 | 3 | SRC = $(wildcard *.c) 4 | INC = $(wildcard *.h) 5 | OBJS = $(patsubst %.c,%.o,$(SRC)) 6 | 7 | PG_CONFIG ?= pg_config 8 | 9 | CC = $(shell $(PG_CONFIG) --cc) 10 | DEFAULT_CFLAGS = $(shell $(PG_CONFIG) --cflags) 11 | 12 | ifdef DEBUG 13 | # Use optimization option that provides good debugging experience 14 | # https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html. 15 | # It overrides '-O2' flag of pg_config. 16 | DEFAULT_CFLAGS += -Og 17 | endif 18 | 19 | SQLITE_CFLAGS += -DSQLITE_THREADSAFE=0 20 | SQLITE_CFLAGS += -DSQLITE_OMIT_LOAD_EXTENSION 21 | SQLITE_CFLAGS += -DHAVE_READLINE 22 | SQLITE_CFLAGS += -DSQLITE_ENABLE_MATH_FUNCTIONS 23 | #SQLITE_CFLAGS += -DSQLITE_ENABLE_NORMALIZE 24 | 25 | SQLITE_CFLAGS += -Wno-missing-prototypes 26 | SQLITE_CFLAGS += -Wno-implicit-fallthrough 27 | 28 | override CFLAGS := $(DEFAULT_CFLAGS) $(CFLAGS) $(SQLITE_CFLAGS) 29 | 30 | LIBS += -lm 31 | LIBS += -lreadline 32 | LIBS += -lncurses 33 | 34 | sqlite3: $(SRC) $(INC) 35 | $(CC) $(CFLAGS) $(SRC) $(LIBS) -o $@ 36 | 37 | sqlite3.o: sqlite3.c sqlite3.h sqlite3ext.h 38 | $(CC) -c $(CFLAGS) $< -o $@ 39 | 40 | clean: 41 | rm -f $(OBJS) sqlite3 42 | -------------------------------------------------------------------------------- /src/bin/lib/sqlite/README.md: -------------------------------------------------------------------------------- 1 | # SQLite lib 2 | 3 | See https://sqlite.org/amalgamation.html 4 | 5 | In order to know which features of SQLite can be used in the source code of 6 | pgcopydb (including SQL support such as functions to format time, FILTER 7 | aggregate clause, RETURNING for DML statements, etc), it's easier to just 8 | use the SQLite amalgamation release. 9 | -------------------------------------------------------------------------------- /src/bin/lib/subcommands.c/commandline.h: -------------------------------------------------------------------------------- 1 | /* 2 | * commandline.h 3 | * 4 | * Copyright (c) Microsoft Corporation. All rights reserved. 5 | * Licensed under the PostgreSQL License. 6 | * 7 | */ 8 | 9 | #ifndef COMMANDLINE_H 10 | #define COMMANDLINE_H 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | typedef int (*command_getopt)(int argc, char **argv); 18 | typedef void (*command_run)(int argc, char **argv); 19 | 20 | typedef struct CommandLine 21 | { 22 | const char *name; 23 | const char *shortDescription; 24 | const char *usageSuffix; 25 | const char *help; 26 | 27 | command_getopt getopt; 28 | command_run run; 29 | 30 | struct CommandLine **subcommands; 31 | char *breadcrumb; 32 | } CommandLine; 33 | 34 | extern CommandLine *current_command; 35 | 36 | #define make_command_set(name, desc, usage, help, getopt, set) \ 37 | { name, desc, usage, help, getopt, NULL, set, NULL } 38 | 39 | #define make_command(name, desc, usage, help, getopt, run) \ 40 | { name, desc, usage, help, getopt, run, NULL, NULL } 41 | 42 | bool commandline_run(CommandLine *command, int argc, char **argv); 43 | void commandline_help(FILE *stream); 44 | void commandline_print_usage(CommandLine *command, FILE *stream); 45 | void commandline_print_subcommands(CommandLine *command, FILE *stream); 46 | void commandline_print_command_tree(CommandLine *command, FILE *stream); 47 | void commandline_add_breadcrumb(CommandLine *command, CommandLine *subcommand); 48 | 49 | #define streq(a, b) (a != NULL && b != NULL && strcmp(a, b) == 0) 50 | 51 | #endif /* COMMANDLINE_H */ 52 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/.gitignore: -------------------------------------------------------------------------------- 1 | pgcopydb 2 | git-version.h 3 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/cli_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/cli_list.h 3 | * Implementation of a CLI which lets you run individual routines 4 | * directly 5 | */ 6 | 7 | #ifndef CLI_LIST_H 8 | #define CLI_LIST_H 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include "cli_common.h" 15 | #include "cli_root.h" 16 | #include "pgsql.h" 17 | #include "schema.h" 18 | 19 | typedef struct ListDBOptions 20 | { 21 | char dir[MAXPGPATH]; 22 | 23 | ConnStrings connStrings; 24 | 25 | char schema_name[PG_NAMEDATALEN]; 26 | char table_name[PG_NAMEDATALEN]; 27 | char filterFileName[MAXPGPATH]; 28 | 29 | bool listSkipped; 30 | bool noPKey; 31 | bool force; 32 | bool summary; 33 | bool availableVersions; 34 | bool requirements; 35 | bool resume; 36 | bool notConsistent; 37 | bool skipCtidSplit; 38 | 39 | char snapshot[BUFSIZE]; 40 | 41 | SplitTableLargerThan splitTablesLargerThan; 42 | int splitMaxParts; 43 | bool estimateTableSizes; 44 | } ListDBOptions; 45 | 46 | 47 | #endif /* CLI_LIST_H */ 48 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/cli_root.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/cli_root.h 3 | * Implementation of a CLI which lets you run individual keeper routines 4 | * directly 5 | */ 6 | 7 | #ifndef CLI_ROOT_H 8 | #define CLI_ROOT_H 9 | 10 | #include "commandline.h" 11 | #include "lock_utils.h" 12 | 13 | extern char pgcopydb_argv0[]; 14 | extern char pgcopydb_program[]; 15 | extern int logLevel; 16 | 17 | extern Semaphore log_semaphore; 18 | 19 | extern char *ps_buffer; 20 | extern size_t ps_buffer_size; 21 | extern size_t last_status_len; 22 | 23 | extern CommandLine help; 24 | extern CommandLine version; 25 | 26 | extern CommandLine root; 27 | extern CommandLine *root_subcommands[]; 28 | 29 | int root_options(int argc, char **argv); 30 | 31 | /* cli_clone_follow.c */ 32 | void cli_clone(int argc, char **argv); 33 | void cli_follow(int argc, char **argv); 34 | 35 | /* cli_copy.h */ 36 | extern CommandLine clone_command; 37 | extern CommandLine fork_command; 38 | extern CommandLine follow_command; 39 | extern CommandLine copy_commands; 40 | 41 | /* cli_snapshot.c */ 42 | extern CommandLine snapshot_command; 43 | 44 | /* cli_dump.c */ 45 | extern CommandLine dump_commands; 46 | 47 | /* cli_ping.c */ 48 | extern CommandLine ping_command; 49 | 50 | /* cli_restore.c */ 51 | extern CommandLine restore_commands; 52 | 53 | /* cli_list.c */ 54 | extern CommandLine list_commands; 55 | 56 | /* cli_stream.c */ 57 | extern CommandLine stream_commands; 58 | 59 | /* cli_sentinel.c */ 60 | extern CommandLine sentinel_commands; 61 | 62 | /* cli_compare.c */ 63 | extern CommandLine compare_commands; 64 | 65 | #endif /* CLI_ROOT_H */ 66 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/env_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/env_utils.h 3 | * Utility functions for interacting with environment settings. 4 | */ 5 | 6 | #ifndef ENV_UTILS_H 7 | #define ENV_UTILS_H 8 | 9 | #include "postgres_fe.h" 10 | 11 | enum EnvType 12 | { 13 | ENV_TYPE_STRING, 14 | ENV_TYPE_INT, 15 | ENV_TYPE_BOOL 16 | }; 17 | 18 | typedef struct EnvParser 19 | { 20 | char *envname; /* name of the environment variable */ 21 | enum EnvType type; 22 | void *target; 23 | int targetSize; /* for strings, 0 for other types */ 24 | bool lowerBounded; /* for integers */ 25 | int minValue; /* for integers, meaningful only when lowerBounded=true */ 26 | bool upperBounded; /* for integers */ 27 | int maxValue; /* for integers, meaningful only when upperBounded=true */ 28 | } EnvParser; 29 | 30 | typedef struct EnvParserArray 31 | { 32 | int count; 33 | EnvParser *array; 34 | } EnvParserArray; 35 | 36 | bool env_exists(const char *name); 37 | bool get_env_copy(const char *name, char *outbuffer, int maxLength); 38 | bool get_env_copy_with_fallback(const char *name, char *result, int maxLength, 39 | const char *fallback); 40 | bool get_env_dup(const char *name, char **result); 41 | bool get_env_dup_with_fallback(const char *name, char **result, const char *fallback); 42 | bool get_env_using_parsers(EnvParserArray *parsers); 43 | 44 | #endif /* ENV_UTILS_H */ 45 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/ini_implementation.c: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/ini_implementation.c 3 | * The file containing library code used to parse files with .INI syntax 4 | * 5 | * The main reason this is in a separate file is so you can exclude a file 6 | * during static analysis. This way we exclude vendored in library code, 7 | * but not our code using it. 8 | * 9 | * Licensed under the PostgreSQL License. 10 | */ 11 | #define INI_IMPLEMENTATION 12 | #include "ini.h" 13 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/lock_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/lock_utils.h 3 | * Utility functions for inter-process locking 4 | */ 5 | 6 | #ifndef LOCK_UTILS_H 7 | #define LOCK_UTILS_H 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | /* 16 | * pgcopydb uses semaphores as a locking mechanism protecting a critical 17 | * section, where a single worker/process is expected at any time. 18 | * 19 | * Some parts of the code might use the semaphore in a re-entrant way, for 20 | * instance SQLite iterator query where the caller hook function then runs a 21 | * query at each step of the iteration. 22 | */ 23 | typedef struct Semaphore 24 | { 25 | int semId; 26 | int initValue; 27 | pid_t owner; 28 | 29 | bool reentrant; 30 | int depth; 31 | 32 | bool debug; 33 | } Semaphore; 34 | 35 | 36 | bool semaphore_init(Semaphore *semaphore); 37 | bool semaphore_finish(Semaphore *semaphore); 38 | 39 | bool semaphore_create(Semaphore *semaphore); 40 | bool semaphore_open(Semaphore *semaphore); 41 | bool semaphore_unlink(Semaphore *semaphore); 42 | 43 | bool semaphore_cleanup(const char *pidfile); 44 | 45 | bool semaphore_lock(Semaphore *semaphore); 46 | bool semaphore_unlock(Semaphore *semaphore); 47 | 48 | void semaphore_log_lock_function(void *udata, int mode); 49 | 50 | #endif /* LOCK_UTILS_H */ 51 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/pgsql_timeline.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/pgsql_timeline.h 3 | * API for sending SQL commands about timelines to a PostgreSQL server 4 | */ 5 | #ifndef PGSQL_TIMELINE_H 6 | #define PGSQL_TIMELINE_H 7 | 8 | #include "pgsql.h" 9 | #include "schema.h" 10 | 11 | /* pgsql_timeline.c */ 12 | bool pgsql_identify_system(PGSQL *pgsql, IdentifySystem *system, 13 | char *cdcPathDir); 14 | bool parse_timeline_history_file(char *filename, 15 | DatabaseCatalog *catalog, 16 | uint32_t currentTimeline); 17 | 18 | #endif /* PGSQL_TIMELINE_H */ 19 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/pgsql_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/pgsql_utils.h 3 | * Common helper functions for interacting with a postgres server 4 | */ 5 | #ifndef PGSQL_UTILS_H 6 | #define PGSQL_UTILS_H 7 | 8 | 9 | #include "libpq-fe.h" 10 | 11 | /* pgsql.c */ 12 | bool is_response_ok(PGresult *result); 13 | bool clear_results(PGSQL *pgsql); 14 | 15 | #endif /* PGSQL_UTILS_H */ 16 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/pidfile.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/pidfile.h 3 | * Utilities to manage the pgcopydb pidfile. 4 | * 5 | * Copyright (c) Microsoft Corporation. All rights reserved. 6 | * Licensed under the PostgreSQL License. 7 | * 8 | */ 9 | #ifndef PIDFILE_H 10 | #define PIDFILE_H 11 | 12 | #include 13 | #include 14 | 15 | #include "postgres_fe.h" 16 | #include "pqexpbuffer.h" 17 | 18 | /* 19 | * As of pgcopydb 0.1, the contents of the pidfile is: 20 | * 21 | * line # 22 | * 1 PID 23 | * 2 version number (PGCOPYDB_VERSION) 24 | * 3 shared semaphore id (used to serialize log writes) 25 | * 26 | */ 27 | #define PIDFILE_LINE_PID 1 28 | #define PIDFILE_LINE_VERSION_STRING 2 29 | #define PIDFILE_LINE_SEM_ID 3 30 | 31 | bool create_pidfile(const char *pidfile, pid_t pid); 32 | bool prepare_pidfile_buffer(PQExpBuffer content, pid_t pid); 33 | bool read_pidfile(const char *pidfile, pid_t *pid); 34 | bool remove_pidfile(const char *pidfile); 35 | 36 | #endif /* PIDFILE_H */ 37 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/progress.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/progress.h 3 | * Implementation of a CLI to copy a database between two Postgres instances 4 | */ 5 | 6 | #ifndef PROGRESS_H 7 | #define PROGRESS_H 8 | 9 | #include "parson.h" 10 | 11 | #include "schema.h" 12 | #include "summary.h" 13 | 14 | 15 | typedef struct CopyTableSummaryArray 16 | { 17 | int count; 18 | int capacity; 19 | CopyTableSummary *array; /* malloc'ed area */ 20 | } CopyTableSummaryArray; 21 | 22 | 23 | typedef struct CopyIndexSummaryArray 24 | { 25 | int count; 26 | int capacity; 27 | CopyIndexSummary *array; /* malloc'ed area */ 28 | } CopyIndexSummaryArray; 29 | 30 | 31 | /* register progress being made, see `pgcopydb list progress` */ 32 | typedef struct CopyProgress 33 | { 34 | int tableCount; 35 | int tableDoneCount; 36 | SourceTableArray tableInProgress; 37 | CopyTableSummaryArray tableSummaryArray; 38 | 39 | int indexCount; 40 | int indexDoneCount; 41 | SourceIndexArray indexInProgress; 42 | CopyIndexSummaryArray indexSummaryArray; 43 | } CopyProgress; 44 | 45 | 46 | bool copydb_prepare_schema_json_file(CopyDataSpec *copySpecs); 47 | bool copydb_update_progress(CopyDataSpec *copySpecs, CopyProgress *progress); 48 | 49 | bool copydb_progress_as_json(CopyDataSpec *copySpecs, 50 | CopyProgress *progress, 51 | JSON_Value *js); 52 | 53 | 54 | #endif /* PROGRESS_H */ 55 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/queue_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/queue_utils.h 3 | * Utility functions for inter-process queueing 4 | */ 5 | 6 | #ifndef QUEUE_UTILS_H 7 | #define QUEUE_UTILS_H 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "postgres.h" 16 | 17 | typedef struct Queue 18 | { 19 | char *name; 20 | int qId; 21 | pid_t owner; 22 | } Queue; 23 | 24 | 25 | /* 26 | * Message types that we send on the queue. The only messages we send are Oid 27 | * from either table (to drive a vacuum analyze job) or an index oid (to drive 28 | * a CREATE INDEX job). 29 | */ 30 | typedef enum 31 | { 32 | QMSG_TYPE_UNKNOWN = 0, 33 | QMSG_TYPE_TABLEOID, /* table oid */ 34 | QMSG_TYPE_TABLEPOID, /* table oid, table partition number */ 35 | QMSG_TYPE_INDEXOID, /* index oid */ 36 | QMSG_TYPE_STREAM_TRANSFORM, /* lsn position for transform process */ 37 | QMSG_TYPE_BLOBOID, /* large object oid */ 38 | QMSG_TYPE_STOP 39 | } QMessageType; 40 | 41 | typedef struct QMessage 42 | { 43 | long type; 44 | union 45 | { 46 | uint32_t oid; 47 | uint64_t lsn; 48 | 49 | /* table parts (support for COPY partitionning) */ 50 | struct tp 51 | { 52 | uint32_t oid; 53 | uint32_t part; 54 | } tp; 55 | } data; 56 | } QMessage; 57 | 58 | bool queue_create(Queue *queue, char *name); 59 | bool queue_unlink(Queue *queue); 60 | 61 | bool queue_send(Queue *queue, QMessage *msg); 62 | bool queue_receive(Queue *queue, QMessage *msg); 63 | 64 | /* see struct msqid_ds in msgctl(2) */ 65 | typedef struct QueueStats 66 | { 67 | uint64_t msg_cbytes; /* number of bytes in use on the queue */ 68 | uint64_t msg_qnum; /* number of msgs in the queue */ 69 | pid_t msg_lspid; /* pid of last msgsnd() */ 70 | pid_t msg_lrpid; /* pid of last msgrcv() */ 71 | } QueueStats; 72 | 73 | bool queue_stats(Queue *queue, QueueStats *stats); 74 | 75 | #endif /* QUEUE_UTILS_H */ 76 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/signals.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/signals.h 3 | * Signal handlers for pgcopydb, used in loop.c and pgsetup.c 4 | */ 5 | 6 | #ifndef SIGNALS_H 7 | #define SIGNALS_H 8 | 9 | #include 10 | #include 11 | 12 | /* This flag controls termination of the main loop. */ 13 | extern volatile sig_atomic_t asked_to_stop; /* SIGTERM */ 14 | extern volatile sig_atomic_t asked_to_stop_fast; /* SIGINT */ 15 | extern volatile sig_atomic_t asked_to_reload; /* SIGHUP */ 16 | extern volatile sig_atomic_t asked_to_quit; /* SIGQUIT */ 17 | 18 | #define CHECK_FOR_FAST_SHUTDOWN { if (asked_to_stop_fast) { break; } \ 19 | } 20 | 21 | void set_signal_handlers(bool exitOnQuit); 22 | void catch_reload(int sig); 23 | void catch_int(int sig); 24 | void catch_term(int sig); 25 | void catch_quit(int sig); 26 | void catch_quit_and_exit(int sig); 27 | void unset_signal_flags(void); 28 | 29 | int get_current_signal(int defaultSignal); 30 | char * signal_to_string(int signal); 31 | bool signal_is_handled(int signal); 32 | 33 | #endif /* SIGNALS_H */ 34 | -------------------------------------------------------------------------------- /src/bin/pgcopydb/string_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * src/bin/pgcopydb/string_utils.h 3 | * Utility functions for string handling 4 | */ 5 | #ifndef STRING_UTILS_H 6 | #define STRING_UTILS_H 7 | 8 | #include 9 | 10 | #define IS_EMPTY_STRING_BUFFER(strbuf) (strbuf[0] == '\0') 11 | 12 | #define NULL_AS_EMPTY_STRING(str) (str == NULL ? "" : str) 13 | 14 | #define streq(a, b) (a != NULL && b != NULL && strcmp(a, b) == 0) 15 | 16 | #define strneq(x, y) \ 17 | ((x != NULL) && (y != NULL) && (strcmp(x, y) != 0)) 18 | 19 | /* maximum decimal int64 length with minus and NUL */ 20 | #define INTSTRING_MAX_DIGITS 21 21 | typedef struct IntString 22 | { 23 | int64_t intValue; 24 | char strValue[INTSTRING_MAX_DIGITS]; 25 | } IntString; 26 | 27 | IntString intToString(int64_t number); 28 | 29 | bool stringToInt(const char *str, int *number); 30 | bool stringToUInt(const char *str, unsigned int *number); 31 | 32 | bool stringToInt64(const char *str, int64_t *number); 33 | bool stringToUInt64(const char *str, uint64_t *number); 34 | 35 | bool stringToUInt32(const char *str, uint32_t *number); 36 | 37 | bool hexStringToUInt32(const char *str, uint32_t *number); 38 | 39 | bool IntervalToString(uint64_t millisecs, char *buffer, size_t size); 40 | 41 | typedef struct LinesBuffer 42 | { 43 | char *buffer; 44 | uint64_t count; 45 | char **lines; /* malloc'ed area */ 46 | } LinesBuffer; 47 | 48 | uint64_t countLines(char *buffer); 49 | bool splitLines(LinesBuffer *lbuf, char *buffer); 50 | 51 | void processBufferCallback(const char *buffer, bool error); 52 | 53 | void pretty_print_bytes(char *buffer, size_t size, uint64_t bytes); 54 | void pretty_print_bytes_per_second(char *buffer, size_t size, uint64_t bytes, 55 | uint64_t durationMs); 56 | void pretty_print_count(char *buffer, size_t size, uint64_t count); 57 | 58 | #endif /* STRING_UTILS_h */ 59 | -------------------------------------------------------------------------------- /tests/.psqlrc: -------------------------------------------------------------------------------- 1 | \set QUIET on 2 | 3 | \set ON_ERROR_STOP on 4 | \set ON_ERROR_ROLLBACK interactive 5 | 6 | \set VERBOSITY verbose 7 | 8 | \pset linestyle 'unicode' 9 | 10 | set intervalstyle to 'postgres_verbose'; 11 | 12 | \set QUIET off 13 | -------------------------------------------------------------------------------- /tests/.sqliterc: -------------------------------------------------------------------------------- 1 | .headers on 2 | .mode column 3 | .echo on 4 | -------------------------------------------------------------------------------- /tests/Dockerfile.pagila: -------------------------------------------------------------------------------- 1 | ARG PGVERSION=16 2 | 3 | FROM debian:bullseye-slim 4 | 5 | ARG PGVERSION 6 | 7 | RUN apt-get update \ 8 | && apt-get install -y --no-install-recommends \ 9 | ca-certificates \ 10 | curl \ 11 | gdb \ 12 | git \ 13 | gnupg \ 14 | htop \ 15 | jq \ 16 | libgc1 \ 17 | libpq5 \ 18 | lsof \ 19 | make \ 20 | openssl \ 21 | postgresql-client-common \ 22 | psmisc \ 23 | strace \ 24 | sudo \ 25 | tmux \ 26 | watch \ 27 | && rm -rf /var/lib/apt/lists/* 28 | 29 | RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - 30 | RUN echo "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main ${PGVERSION}" > /etc/apt/sources.list.d/pgdg.list 31 | 32 | # bypass initdb of a "main" cluster 33 | RUN echo 'create_main_cluster = false' | sudo tee -a /etc/postgresql-common/createcluster.conf 34 | RUN apt-get update \ 35 | && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 36 | postgresql-${PGVERSION} \ 37 | && rm -rf /var/lib/apt/lists/* 38 | 39 | WORKDIR /usr/src/ 40 | RUN git clone --depth 1 https://github.com/devrimgunduz/pagila.git 41 | 42 | #RUN adduser --disabled-password --gecos '' --home /var/lib/postgres docker 43 | #RUN adduser docker sudo 44 | RUN useradd -rm -d /var/lib/postgres -s /bin/bash -g postgres -G sudo docker 45 | RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers 46 | 47 | COPY --from=pgcopydb /usr/local/bin/sqlite3 /usr/local/bin 48 | COPY --from=pgcopydb /usr/local/bin/pgcopydb /usr/local/bin 49 | 50 | COPY .psqlrc /var/lib/postgres 51 | COPY .sqliterc /var/lib/postgres 52 | COPY cleanup.mk /var/lib/postgres 53 | -------------------------------------------------------------------------------- /tests/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | PGVERSION ?= 16 5 | BUILD_ARGS = --build-arg PGVERSION=$(PGVERSION) 6 | 7 | all: pagila pagila-multi-steps blobs unit filtering extensions \ 8 | cdc-wal2json cdc-test-decoding cdc-endpos-between-transaction cdc-low-level \ 9 | follow-wal2json follow-9.6 follow-data-only \ 10 | endpos-in-multi-wal-txn; 11 | 12 | pagila: build 13 | $(MAKE) -C $@ 14 | 15 | pagila-multi-steps: build 16 | $(MAKE) -C $@ 17 | 18 | pagila-standby: build 19 | $(MAKE) -C $@ 20 | 21 | blobs: build 22 | $(MAKE) -C $@ 23 | 24 | unit: build 25 | $(MAKE) -C $@ 26 | 27 | filtering: build 28 | $(MAKE) -C $@ 29 | 30 | extensions: build 31 | $(MAKE) -C $@ 32 | 33 | cdc-wal2json: build 34 | $(MAKE) -C $@ 35 | 36 | cdc-test-decoding: build 37 | $(MAKE) -C $@ 38 | 39 | cdc-endpos-between-transaction: build 40 | $(MAKE) -C $@ 41 | 42 | cdc-low-level: build 43 | $(MAKE) -C $@ 44 | 45 | follow-wal2json: build 46 | $(MAKE) -C $@ 47 | 48 | follow-9.6: build 49 | $(MAKE) -C $@ 50 | 51 | follow-data-only: build 52 | $(MAKE) -C $@ 53 | 54 | endpos-in-multi-wal-txn: build 55 | $(MAKE) -C $@ 56 | 57 | timescaledb: build 58 | $(MAKE) -C $@ 59 | 60 | build: 61 | docker build $(BUILD_ARGS) -t pagila -f Dockerfile.pagila . 62 | 63 | .PHONY: all build 64 | .PHONY: pagila pagila-multi-steps blobs unit filtering extensions 65 | .PHONY: cdc-wal2json cdc-test-decoding cdc-low-level 66 | .PHONY: follow-wal2json follow-9.6 67 | .PHONY: endpos-in-multi-wal-txn 68 | -------------------------------------------------------------------------------- /tests/blobs/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pgcopydb 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./import.sql import.sql 6 | COPY ./imgs imgs 7 | 8 | USER docker 9 | CMD ["/usr/src/pgcopydb/copydb.sh"] 10 | -------------------------------------------------------------------------------- /tests/blobs/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | import: 16 | psql --single-transaction --no-psqlrc -f import.sql 17 | 18 | 19 | .PHONY: run down build test import 20 | -------------------------------------------------------------------------------- /tests/blobs/README.md: -------------------------------------------------------------------------------- 1 | # Testing Large Objects 2 | 3 | To be able to test our large objects support, we first need to import some 4 | large object data in our source database. For that open data images have 5 | been imported from [unsplash](https://unsplash.com/s/photos/elephants). 6 | -------------------------------------------------------------------------------- /tests/blobs/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | target: 11 | image: postgres:13-bullseye 12 | expose: 13 | - 5432 14 | environment: 15 | POSTGRES_USER: postgres 16 | POSTGRES_PASSWORD: h4ckm3 17 | POSTGRES_HOST_AUTH_METHOD: trust 18 | test: 19 | build: . 20 | environment: 21 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 22 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 23 | PGCOPYDB_TABLE_JOBS: 4 24 | PGCOPYDB_INDEX_JOBS: 2 25 | depends_on: 26 | - source 27 | - target 28 | -------------------------------------------------------------------------------- /tests/blobs/imgs/aj-robbie-BuQ1RZckYW4-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimitri/pgcopydb/c1e07da3d257d0d3a1fc321f3c7e1a4063eed6bd/tests/blobs/imgs/aj-robbie-BuQ1RZckYW4-unsplash.jpg -------------------------------------------------------------------------------- /tests/blobs/imgs/bisakha-datta--6SmukZ_w6s-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimitri/pgcopydb/c1e07da3d257d0d3a1fc321f3c7e1a4063eed6bd/tests/blobs/imgs/bisakha-datta--6SmukZ_w6s-unsplash.jpg -------------------------------------------------------------------------------- /tests/blobs/imgs/geran-de-klerk-AX9sJ-mPoL4-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimitri/pgcopydb/c1e07da3d257d0d3a1fc321f3c7e1a4063eed6bd/tests/blobs/imgs/geran-de-klerk-AX9sJ-mPoL4-unsplash.jpg -------------------------------------------------------------------------------- /tests/blobs/imgs/nam-anh-QJbyG6O0ick-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimitri/pgcopydb/c1e07da3d257d0d3a1fc321f3c7e1a4063eed6bd/tests/blobs/imgs/nam-anh-QJbyG6O0ick-unsplash.jpg -------------------------------------------------------------------------------- /tests/blobs/imgs/redcharlie-Y--zr3CPaPs-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimitri/pgcopydb/c1e07da3d257d0d3a1fc321f3c7e1a4063eed6bd/tests/blobs/imgs/redcharlie-Y--zr3CPaPs-unsplash.jpg -------------------------------------------------------------------------------- /tests/blobs/imgs/richard-jacobs-8oenpCXktqQ-unsplash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dimitri/pgcopydb/c1e07da3d257d0d3a1fc321f3c7e1a4063eed6bd/tests/blobs/imgs/richard-jacobs-8oenpCXktqQ-unsplash.jpg -------------------------------------------------------------------------------- /tests/blobs/import.sql: -------------------------------------------------------------------------------- 1 | \lo_import 'imgs/aj-robbie-BuQ1RZckYW4-unsplash.jpg' 2 | \lo_import 'imgs/bisakha-datta--6SmukZ_w6s-unsplash.jpg' 3 | \lo_import 'imgs/geran-de-klerk-AX9sJ-mPoL4-unsplash.jpg' 4 | \lo_import 'imgs/nam-anh-QJbyG6O0ick-unsplash.jpg' 5 | \lo_import 'imgs/redcharlie-Y--zr3CPaPs-unsplash.jpg' 6 | \lo_import 'imgs/richard-jacobs-8oenpCXktqQ-unsplash.jpg' 7 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/000000010000000000000002.sql: -------------------------------------------------------------------------------- 1 | -- KEEPALIVE {"lsn":"0/2448730","timestamp":"2023-12-21 17:07:18.986320+0000"} 2 | BEGIN; -- {"xid":491,"lsn":"0/244B450","timestamp":"2023-12-21 17:07:19.3436+0000","commit_lsn":"0/244B750"} 3 | PREPARE 86e87d54 AS INSERT INTO "public"."category" ("category_id", "name", "last_update") overriding system value VALUES ($1, $2, $3), ($4, $5, $6), ($7, $8, $9), ($10, $11, $12), ($13, $14, $15); 4 | EXECUTE 86e87d54["1000","Fantasy","2022-12-08 00:00:01+00","1001","History","2022-12-09 00:00:01+00","1002","Adventure","2022-12-10 00:00:01+00","1003","Musical","2022-12-11 00:00:01+00","1004","Western","2022-12-12 00:00:01+00"]; 5 | COMMIT; -- {"xid":491,"lsn":"0/244B750","timestamp":"2023-12-21 17:07:19.3436+0000"} 6 | BEGIN; -- {"xid":492,"lsn":"0/244B750","timestamp":"2023-12-21 17:07:19.3543+0000"} 7 | PREPARE 918852ce AS INSERT INTO "public"."category" ("category_id", "name", "last_update") overriding system value VALUES ($1, $2, $3), ($4, $5, $6), ($7, $8, $9); 8 | EXECUTE 918852ce["1005","Mystery","2022-12-13 00:00:01+00","1006","Historical drama","2022-12-14 00:00:01+00","1008","Thriller","2022-12-15 00:00:01+00"]; 9 | -- KEEPALIVE {"lsn":"0/244B878","timestamp":"2023-12-21 17:07:19.3615+0000"} 10 | -- ENDPOS {"lsn":"0/244B878"} 11 | ROLLBACK; -- {"xid":492,"lsn":"0/244B878","timestamp":"2023-12-21 17:07:19.3543+0000"} 12 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./dml.sql dml.sql 6 | COPY ./dml1.sql dml1.sql 7 | COPY ./dml2.sql dml2.sql 8 | COPY ./ddl.sql ddl.sql 9 | COPY ./000000010000000000000002.json 000000010000000000000002.json 10 | COPY ./000000010000000000000002.sql 000000010000000000000002.sql 11 | 12 | USER docker 13 | CMD ["/usr/src/pgcopydb/copydb.sh"] 14 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:13-bullseye 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends postgresql-13-wal2json \ 5 | && rm -rf /var/lib/apt/lists/* 6 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.pg 6 | expose: 7 | - 5432 8 | environment: 9 | POSTGRES_USER: postgres 10 | POSTGRES_PASSWORD: h4ckm3 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | command: > 13 | -c wal_level=logical 14 | -c ssl=on 15 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 16 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 17 | target: 18 | image: postgres:13-bullseye 19 | expose: 20 | - 5432 21 | environment: 22 | POSTGRES_USER: postgres 23 | POSTGRES_PASSWORD: h4ckm3 24 | POSTGRES_HOST_AUTH_METHOD: trust 25 | command: > 26 | -c ssl=on 27 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 28 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 29 | test: 30 | build: 31 | context: . 32 | dockerfile: Dockerfile 33 | environment: 34 | PGSSLMODE: "require" 35 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 36 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 37 | PGCOPYDB_TABLE_JOBS: 4 38 | PGCOPYDB_INDEX_JOBS: 2 39 | PGCOPYDB_OUTPUT_PLUGIN: wal2json 40 | depends_on: 41 | - source 42 | - target 43 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | alter table payment_p2022_01 replica identity full; 8 | alter table payment_p2022_02 replica identity full; 9 | alter table payment_p2022_03 replica identity full; 10 | alter table payment_p2022_04 replica identity full; 11 | alter table payment_p2022_05 replica identity full; 12 | alter table payment_p2022_06 replica identity full; 13 | alter table payment_p2022_07 replica identity full; 14 | commit; 15 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/dml.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database. 5 | 6 | \ir dml1.sql 7 | \ir dml2.sql 8 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/dml1.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database. 5 | 6 | -- first transaction 7 | begin; 8 | 9 | insert into category(category_id, name, last_update) values (1000, 'Fantasy', '2022-12-08 00:00:01'); 10 | insert into category(category_id, name, last_update) values (1001, 'History', '2022-12-09 00:00:01'); 11 | insert into category(category_id, name, last_update) values (1002, 'Adventure', '2022-12-10 00:00:01'); 12 | insert into category(category_id, name, last_update) values (1003, 'Musical', '2022-12-11 00:00:01'); 13 | insert into category(category_id, name, last_update) values (1004, 'Western', '2022-12-12 00:00:01'); 14 | 15 | commit; 16 | -------------------------------------------------------------------------------- /tests/cdc-endpos-between-transaction/dml2.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database. 5 | 6 | -- second transaction 7 | begin; 8 | 9 | insert into category(category_id, name, last_update) values (1005, 'Mystery', '2022-12-13 00:00:01'); 10 | insert into category(category_id, name, last_update) values (1006, 'Historical drama', '2022-12-14 00:00:01'); 11 | insert into category(category_id, name, last_update) values (1008, 'Thriller', '2022-12-15 00:00:01'); 12 | insert into category(category_id, name, last_update) values (1007, 'Satire', '2022-12-16 00:00:01'); 13 | insert into category(category_id, name, last_update) values (1009, 'Romance', '2022-12-17 00:00:01'); 14 | 15 | commit; 16 | -------------------------------------------------------------------------------- /tests/cdc-low-level/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./dml.sql dml.sql 6 | COPY ./ddl.sql ddl.sql 7 | COPY ./000000010000000000000002.json 000000010000000000000002.json 8 | COPY ./000000010000000000000002.sql 000000010000000000000002.sql 9 | COPY ./pipeline-deadlock.sql pipeline-deadlock.sql 10 | 11 | USER docker 12 | CMD ["/usr/src/pgcopydb/copydb.sh"] 13 | -------------------------------------------------------------------------------- /tests/cdc-low-level/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/cdc-low-level/README.md: -------------------------------------------------------------------------------- 1 | Change Data Capture 2 | =================== 3 | 4 | pgcopydb implements logical decoding through using the wal2json plugin: 5 | 6 | https://github.com/eulerto/wal2json 7 | 8 | This means that changes made to the source database during the copying of 9 | the data can be replayed to the target database. 10 | 11 | This directory implements testing for the change data capture capabilities 12 | of pgcopydb. Tests are using the pagila database, and a set of SQL scripts 13 | that run some DML trafic. 14 | -------------------------------------------------------------------------------- /tests/cdc-low-level/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | command: > 11 | -c wal_level=logical 12 | -c ssl=on 13 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 14 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 15 | target: 16 | image: postgres:13-bullseye 17 | expose: 18 | - 5432 19 | environment: 20 | POSTGRES_USER: postgres 21 | POSTGRES_PASSWORD: h4ckm3 22 | POSTGRES_HOST_AUTH_METHOD: trust 23 | command: > 24 | -c ssl=on 25 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 26 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 27 | test: 28 | build: 29 | context: . 30 | dockerfile: Dockerfile 31 | cap_add: 32 | - SYS_ADMIN 33 | - SYS_PTRACE 34 | environment: 35 | PGSSLMODE: "require" 36 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 37 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 38 | PGCOPYDB_TABLE_JOBS: 4 39 | PGCOPYDB_INDEX_JOBS: 2 40 | depends_on: 41 | - source 42 | - target 43 | -------------------------------------------------------------------------------- /tests/cdc-low-level/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | alter table payment_p2022_01 replica identity full; 8 | alter table payment_p2022_02 replica identity full; 9 | alter table payment_p2022_03 replica identity full; 10 | alter table payment_p2022_04 replica identity full; 11 | alter table payment_p2022_05 replica identity full; 12 | alter table payment_p2022_06 replica identity full; 13 | alter table payment_p2022_07 replica identity full; 14 | commit; 15 | 16 | begin; 17 | 18 | create table metrics(id int, time timestamptz, name text, value numeric); 19 | 20 | -- Create a trigger function that artificially raises notice on insert 21 | -- to send large message which fills the libpq connection's read buffer 22 | -- which is 16KB by default. 23 | -- https://github.com/postgres/postgres/blob/master/src/interfaces/libpq/fe-connect.c#L4616 24 | CREATE OR REPLACE FUNCTION notify_on_insert() RETURNS TRIGGER AS $$ 25 | BEGIN 26 | RAISE NOTICE 'A new record has been inserted into the metrics table! %', repeat('a', 32 * 1024); 27 | RETURN NEW; 28 | END; 29 | $$ LANGUAGE plpgsql; 30 | 31 | CREATE TRIGGER insert_notice_trigger 32 | AFTER INSERT ON metrics 33 | FOR EACH ROW 34 | EXECUTE FUNCTION notify_on_insert(); 35 | 36 | -- Apply executes on session_replication_role = 'replica' 37 | -- Lets enable the trigger for the replica mode 38 | ALTER TABLE metrics ENABLE REPLICA TRIGGER insert_notice_trigger; 39 | 40 | commit; 41 | -------------------------------------------------------------------------------- /tests/cdc-low-level/dml.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database. 5 | 6 | \set customerid1 291 7 | \set customerid2 293 8 | 9 | \set staffid1 1 10 | \set staffid2 2 11 | 12 | \set inventoryid1 371 13 | \set inventoryid2 373 14 | 15 | begin; 16 | 17 | insert into rental(rental_date, inventory_id, customer_id, staff_id, last_update) 18 | values 19 | ('2022-06-01', :inventoryid1, :customerid1, :staffid1, '2022-06-01'), 20 | ('2022-06-01', :inventoryid2, :customerid2, :staffid2, '2022-06-01'); 21 | 22 | insert into payment(customer_id, staff_id, rental_id, amount, payment_date) 23 | select customer_id, staff_id, rental_id, 5.99, rental_date 24 | from rental where rental_date='2022-06-01'; 25 | 26 | commit; 27 | 28 | -- update 10 rows in a single UPDATE command 29 | update public.payment set amount = 11.95 where amount = 11.99; 30 | 31 | begin; 32 | 33 | delete from payment 34 | using rental 35 | where rental.rental_id = payment.rental_id 36 | and rental.last_update = '2022-06-01'; 37 | 38 | delete from rental where rental.last_update = '2022-06-01'; 39 | 40 | commit; 41 | 42 | -- 43 | -- update the payments back to their original values 44 | -- 45 | begin; 46 | 47 | update public.payment set amount = 11.99 where amount = 11.95; 48 | 49 | commit; 50 | -------------------------------------------------------------------------------- /tests/cdc-test-decoding/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./dml.sql dml.sql 6 | COPY ./ddl.sql ddl.sql 7 | COPY ./000000010000000000000002.json 000000010000000000000002.json 8 | COPY ./000000010000000000000002.sql 000000010000000000000002.sql 9 | COPY ./continued-txn.json continued-txn.json 10 | COPY ./continued-txn.sql continued-txn.sql 11 | 12 | USER docker 13 | CMD ["/usr/src/pgcopydb/copydb.sh"] 14 | -------------------------------------------------------------------------------- /tests/cdc-test-decoding/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/cdc-test-decoding/README.md: -------------------------------------------------------------------------------- 1 | Change Data Capture 2 | =================== 3 | 4 | pgcopydb implements logical decoding through using the wal2json plugin: 5 | 6 | https://github.com/eulerto/wal2json 7 | 8 | This means that changes made to the source database during the copying of 9 | the data can be replayed to the target database. 10 | 11 | This directory implements testing for the change data capture capabilities 12 | of pgcopydb. Tests are using the pagila database, and a set of SQL scripts 13 | that run some DML trafic. 14 | -------------------------------------------------------------------------------- /tests/cdc-test-decoding/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | command: > 11 | -c wal_level=logical 12 | -c ssl=on 13 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 14 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 15 | target: 16 | image: postgres:13-bullseye 17 | expose: 18 | - 5432 19 | environment: 20 | POSTGRES_USER: postgres 21 | POSTGRES_PASSWORD: h4ckm3 22 | POSTGRES_HOST_AUTH_METHOD: trust 23 | command: > 24 | -c ssl=on 25 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 26 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 27 | test: 28 | build: 29 | context: . 30 | dockerfile: Dockerfile 31 | environment: 32 | PGSSLMODE: "require" 33 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 34 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 35 | PGCOPYDB_TABLE_JOBS: 4 36 | PGCOPYDB_INDEX_JOBS: 2 37 | depends_on: 38 | - source 39 | - target 40 | -------------------------------------------------------------------------------- /tests/cdc-test-decoding/continued-txn.json: -------------------------------------------------------------------------------- 1 | {"action":"I","xid":"0","lsn":"6F/140005C0","timestamp":"2024-02-22 19:55:32.604734+0000","message":"table public.readings: INSERT: \"time\"[timestamp with time zone]:'2017-07-15 02:54:30+00' tags_id[integer]:48 latitude[double precision]:3.46436 longitude[double precision]:68.56976 elevation[double precision]:115 velocity[double precision]:55 heading[double precision]:30 grade[double precision]:78 fuel_consumption[double precision]:1.8 additional_tags[jsonb]:null"} 2 | {"action":"C","xid":"6730937","lsn":"6F/14003AD8","timestamp":"2024-02-22 19:55:32.604741+0000","message":"COMMIT 6730937"} 3 | {"action":"X","lsn":"6F/15004368"} 4 | -------------------------------------------------------------------------------- /tests/cdc-test-decoding/continued-txn.sql: -------------------------------------------------------------------------------- 1 | PREPARE b286b954 AS INSERT INTO public.readings ("time", tags_id, latitude, longitude, elevation, velocity, heading, grade, fuel_consumption, additional_tags) overriding system value VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10); 2 | EXECUTE b286b954["2017-07-15 02:54:30+00","48","3.46436","68.56976","115","55","30","78","1.8",null]; 3 | COMMIT; -- {"xid":6730937,"lsn":"6F/14003AD8","timestamp":"2024-02-22 19:55:32.604741+0000"} 4 | -- SWITCH WAL {"lsn":"6F/15004368"} 5 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./dml.sql dml.sql 6 | COPY ./ddl.sql ddl.sql 7 | COPY ./special-dml.sql special-dml.sql 8 | COPY ./special-ddl.sql special-ddl.sql 9 | COPY ./000000010000000000000002.json 000000010000000000000002.json 10 | COPY ./000000010000000000000002.sql 000000010000000000000002.sql 11 | 12 | USER docker 13 | CMD ["/usr/src/pgcopydb/copydb.sh"] 14 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:13-bullseye 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends postgresql-13-wal2json \ 5 | && rm -rf /var/lib/apt/lists/* 6 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | COMPOSE_EXIT = --exit-code-from=test --abort-on-container-exit 5 | 6 | test: down run down ; 7 | 8 | up: down build 9 | docker compose up $(COMPOSE_EXIT) 10 | 11 | run: build 12 | docker compose run test 13 | 14 | down: 15 | docker compose down 16 | 17 | build: 18 | docker compose build --quiet 19 | 20 | .PHONY: run down build test 21 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/README.md: -------------------------------------------------------------------------------- 1 | Change Data Capture 2 | =================== 3 | 4 | pgcopydb implements logical decoding through using the wal2json plugin: 5 | 6 | https://github.com/eulerto/wal2json 7 | 8 | This means that changes made to the source database during the copying of 9 | the data can be replayed to the target database. 10 | 11 | This directory implements testing for the change data capture capabilities 12 | of pgcopydb. Tests are using the pagila database, and a set of SQL scripts 13 | that run some DML trafic. 14 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.pg 6 | expose: 7 | - 5432 8 | environment: 9 | POSTGRES_USER: postgres 10 | POSTGRES_PASSWORD: h4ckm3 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | command: > 13 | -c wal_level=logical 14 | -c ssl=on 15 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 16 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 17 | target: 18 | image: postgres:13-bullseye 19 | expose: 20 | - 5432 21 | environment: 22 | POSTGRES_USER: postgres 23 | POSTGRES_PASSWORD: h4ckm3 24 | POSTGRES_HOST_AUTH_METHOD: trust 25 | command: > 26 | -c ssl=on 27 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 28 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 29 | test: 30 | build: 31 | context: . 32 | dockerfile: Dockerfile 33 | environment: 34 | PGSSLMODE: "require" 35 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 36 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 37 | PGCOPYDB_TABLE_JOBS: 4 38 | PGCOPYDB_INDEX_JOBS: 2 39 | PGCOPYDB_OUTPUT_PLUGIN: wal2json 40 | depends_on: 41 | - source 42 | - target 43 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | alter table payment_p2022_01 replica identity full; 8 | alter table payment_p2022_02 replica identity full; 9 | alter table payment_p2022_03 replica identity full; 10 | alter table payment_p2022_04 replica identity full; 11 | alter table payment_p2022_05 replica identity full; 12 | alter table payment_p2022_06 replica identity full; 13 | alter table payment_p2022_07 replica identity full; 14 | commit; 15 | 16 | begin; 17 | alter table address replica identity full; 18 | commit; 19 | 20 | begin; 21 | create table if not exists generated_column_test 22 | ( 23 | id bigint primary key, 24 | name text, 25 | greet_hello text generated always as ('Hello ' || name) stored, 26 | greet_hi text generated always as ('Hi ' || name) stored, 27 | -- tests identifier escaping 28 | time text generated always as ('identifier 1' || 'now') stored, 29 | email text not null, 30 | -- tests identifier escaping 31 | "table" text generated always as ('identifier 2' || name) stored, 32 | """table""" text generated always as ('identifier 3' || name) stored, 33 | """hel""lo""" text generated always as ('identifier 4' || name) stored 34 | ); 35 | commit; 36 | 37 | begin; 38 | -- table with single column to test update is not failing when value is not changed 39 | create table if not exists single_column_table 40 | ( 41 | id bigint 42 | ); 43 | alter table single_column_table replica identity full; 44 | 45 | -- table with 3 columns to test update is not failing when value is not changed 46 | create table if not exists multi_column_table 47 | ( 48 | id bigint, 49 | name text, 50 | email text 51 | ); 52 | alter table multi_column_table replica identity full; 53 | commit; 54 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/special-ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL to create postgres objects covering special cases 5 | -- like maximum identifiers lengths, identifiers that requires double quotes, 6 | -- etc. 7 | 8 | -- 9 | -- create schemas and tables with names that needs double-quoting 10 | -- 11 | 12 | begin; 13 | 14 | create schema if not exists "Sp1eCial .Char"; 15 | 16 | create table "Sp1eCial .Char"."source1testing"( 17 | "s0" serial primary key, 18 | "s""1" int not null 19 | ); 20 | 21 | commit; 22 | 23 | -- 24 | -- create schema and table with length of NAMEDATALEN 25 | -- 26 | 27 | begin; 28 | 29 | create schema "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456"; 30 | 31 | create table "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456"."abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890123456"( 32 | "abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890123456" serial primary key, 33 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456" int not null 34 | ); 35 | 36 | commit; 37 | -------------------------------------------------------------------------------- /tests/cdc-wal2json/special-dml.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database and covers edge 5 | -- cases of postgres objects like identifiers length, identifiers that requires 6 | -- double quotes 7 | truncate "Sp1eCial .Char"."source1testing"; 8 | 9 | insert into "Sp1eCial .Char"."source1testing"("s""1") 10 | select 11 | x 12 | from 13 | generate_series(1, 5) as t(x); 14 | 15 | update 16 | "Sp1eCial .Char"."source1testing" 17 | set 18 | "s""1" = "s""1" * 2; 19 | 20 | delete from "Sp1eCial .Char"."source1testing" 21 | where ("s""1" % 3) = 0; 22 | 23 | insert into "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456"."abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890123456"("ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456") 24 | select 25 | x 26 | from 27 | generate_series(1, 5) as t(x); 28 | 29 | update 30 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456"."abcdefghijklmnopqrstuvwxyz0123456789012345678901234567890123456" 31 | set 32 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456" = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789012345678901234567890123456" * 2; 33 | 34 | -------------------------------------------------------------------------------- /tests/cleanup.mk: -------------------------------------------------------------------------------- 1 | # cleanup the external volume between docker-compose runs. 2 | 3 | cleanup: 4 | sudo rm -rf $(TMPDIR)/pgcopydb 5 | sudo install -d -o docker $(TMPDIR)/pgcopydb 6 | 7 | sudo rm -rf $(XDG_DATA_HOME) 8 | sudo install -d -o docker $(XDG_DATA_HOME) 9 | -------------------------------------------------------------------------------- /tests/endpos-in-multi-wal-txn/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./ddl.sql ddl.sql 6 | COPY ./multi-wal-txn.sql multi-wal-txn.sql 7 | 8 | USER docker 9 | CMD ["/usr/src/pgcopydb/copydb.sh"] 10 | -------------------------------------------------------------------------------- /tests/endpos-in-multi-wal-txn/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:13-bullseye 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends postgresql-13-wal2json \ 5 | && rm -rf /var/lib/apt/lists/* 6 | -------------------------------------------------------------------------------- /tests/endpos-in-multi-wal-txn/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/endpos-in-multi-wal-txn/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.pg 6 | expose: 7 | - 5432 8 | environment: 9 | POSTGRES_USER: postgres 10 | POSTGRES_PASSWORD: h4ckm3 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | command: > 13 | -c wal_level=logical 14 | -c ssl=on 15 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 16 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 17 | target: 18 | image: postgres:13-bullseye 19 | expose: 20 | - 5432 21 | environment: 22 | POSTGRES_USER: postgres 23 | POSTGRES_PASSWORD: h4ckm3 24 | POSTGRES_HOST_AUTH_METHOD: trust 25 | command: > 26 | -c ssl=on 27 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 28 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 29 | test: 30 | build: 31 | context: . 32 | dockerfile: Dockerfile 33 | environment: 34 | PGSSLMODE: "require" 35 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 36 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 37 | PGCOPYDB_TABLE_JOBS: 4 38 | PGCOPYDB_INDEX_JOBS: 2 39 | PGCOPYDB_OUTPUT_PLUGIN: wal2json 40 | depends_on: 41 | - source 42 | - target 43 | -------------------------------------------------------------------------------- /tests/endpos-in-multi-wal-txn/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | 8 | CREATE TABLE table_a (id serial PRIMARY KEY, f1 int4, f2 text); 9 | CREATE TABLE table_b (id serial PRIMARY KEY, f1 int4, f2 text[]); 10 | 11 | commit; 12 | -------------------------------------------------------------------------------- /tests/endpos-in-multi-wal-txn/multi-wal-txn.sql: -------------------------------------------------------------------------------- 1 | -- transaction that spans multiple WAL files 2 | 3 | BEGIN; 4 | 5 | INSERT INTO table_a(f1) VALUES ((random() * 100 + 1)::int); 6 | 7 | SELECT 8 | pg_switch_wal(); 9 | 10 | INSERT INTO table_a(f1) VALUES ((random() * 100 + 1)::int); 11 | 12 | SELECT 13 | pg_switch_wal(); 14 | 15 | INSERT INTO table_a(f1) VALUES (10001001), (101), (10001002), (104), (10001003); 16 | 17 | SELECT 18 | pg_switch_wal(); 19 | 20 | INSERT INTO table_a(f1) VALUES ((random() * 100 + 1)::int); 21 | 22 | SELECT 23 | pg_switch_wal(); 24 | 25 | COMMIT; 26 | -------------------------------------------------------------------------------- /tests/extensions/.editorconfig: -------------------------------------------------------------------------------- 1 | # Ignore one file that is from an outside source 2 | [countries.sql] 3 | charset = unset 4 | end_of_line = unset 5 | insert_final_newline = unset 6 | trim_trailing_whitespace = unset 7 | indent_style = unset 8 | indent_size = unset 9 | -------------------------------------------------------------------------------- /tests/extensions/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./ddl.sql ddl.sql 6 | COPY ./countries.sql countries.sql 7 | 8 | USER docker 9 | CMD ["/usr/src/pgcopydb/copydb.sh"] 10 | -------------------------------------------------------------------------------- /tests/extensions/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:13-bullseye 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends \ 5 | postgresql-13-wal2json \ 6 | postgresql-13-partman \ 7 | postgresql-13-postgis-3 \ 8 | postgis \ 9 | && rm -rf /var/lib/apt/lists/* 10 | -------------------------------------------------------------------------------- /tests/extensions/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/extensions/README.md: -------------------------------------------------------------------------------- 1 | # Testing extensions 2 | 3 | * PostGIS test data from https://github.com/adityatoshniwal/postgis-sample-dataset 4 | -------------------------------------------------------------------------------- /tests/extensions/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.pg 6 | expose: 7 | - 5432 8 | environment: 9 | POSTGRES_USER: postgres 10 | POSTGRES_PASSWORD: h4ckm3 11 | POSTGRES_HOST_AUTH_METHOD: trust 12 | command: > 13 | -c wal_level=logical 14 | -c ssl=on 15 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 16 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 17 | target: 18 | build: 19 | context: . 20 | dockerfile: Dockerfile.pg 21 | expose: 22 | - 5432 23 | environment: 24 | POSTGRES_USER: postgres 25 | POSTGRES_PASSWORD: h4ckm3 26 | POSTGRES_HOST_AUTH_METHOD: trust 27 | command: > 28 | -c ssl=on 29 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 30 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 31 | test: 32 | build: . 33 | environment: 34 | PGSSLMODE: "require" 35 | POSTGRES_SOURCE: postgres://postgres:h4ckm3@source/postgres 36 | POSTGRES_TARGET: postgres://postgres:h4ckm3@target/postgres 37 | PGCOPYDB_SOURCE_PGURI_SU: postgres://postgres:h4ckm3@source/pagila 38 | PGCOPYDB_TARGET_PGURI_SU: postgres://postgres:h4ckm3@target/pagila 39 | PGCOPYDB_SOURCE_PGURI: postgres://pagila:own3d@source/pagila 40 | PGCOPYDB_TARGET_PGURI: postgres://pagila:own3d@target/pagila 41 | PGCOPYDB_TABLE_JOBS: 4 42 | PGCOPYDB_INDEX_JOBS: 2 43 | depends_on: 44 | - source 45 | - target 46 | -------------------------------------------------------------------------------- /tests/extensions/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/extensions/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the test database. 5 | 6 | begin; 7 | -- Create new tables with foreign key relationships to test 8 | -- the query for `schema_list_extensions()` 9 | create table one (id int primary key, two_id int); 10 | create table two (id int primary key, one_id int, three_id int); 11 | create table three (id int primary key); 12 | alter table one add constraint one_two_fk foreign key (two_id) references two(id); 13 | alter table two add constraint two_one_fk foreign key (one_id) references one(id); 14 | alter table two add constraint two_three_fk foreign key (three_id) references three(id); 15 | commit; 16 | -------------------------------------------------------------------------------- /tests/filtering/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./include.ini include.ini 6 | COPY ./exclude.ini exclude.ini 7 | COPY ./extra.sql extra.sql 8 | 9 | # unit tests 10 | COPY ./include ./test/include 11 | COPY ./exclude ./test/exclude 12 | 13 | USER docker 14 | WORKDIR /usr/src/pgcopydb/test/ 15 | CMD ["/usr/src/pgcopydb/copydb.sh"] 16 | -------------------------------------------------------------------------------- /tests/filtering/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/filtering/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | command: > 11 | -c ssl=on 12 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 13 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 14 | target: 15 | image: postgres:13-bullseye 16 | expose: 17 | - 5432 18 | environment: 19 | POSTGRES_USER: postgres 20 | POSTGRES_PASSWORD: h4ckm3 21 | POSTGRES_HOST_AUTH_METHOD: trust 22 | command: > 23 | -c ssl=on 24 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 25 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 26 | test: 27 | build: . 28 | environment: 29 | PGSSLMODE: "require" 30 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 31 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 32 | PGCOPYDB_TABLE_JOBS: 4 33 | PGCOPYDB_INDEX_JOBS: 2 34 | depends_on: 35 | - source 36 | - target 37 | -------------------------------------------------------------------------------- /tests/filtering/exclude.ini: -------------------------------------------------------------------------------- 1 | [exclude-schema] 2 | public 3 | bar 4 | schema_name_20_chars 5 | 6 | [exclude-table] 7 | ; public.actor 8 | ; public.category 9 | ; public.film 10 | ; public.film_actor 11 | ; public.film_category 12 | ; public.language 13 | ; public.rental 14 | foo.test_tbl_2 15 | app.foo 16 | foo.matview_1_exclude_as_table 17 | 18 | [exclude-table-data] 19 | foo.matview_1_exclude_data 20 | partitioned_tables.sellers_archive 21 | -------------------------------------------------------------------------------- /tests/filtering/exclude/expected/1-ignore-matview.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ]----------------------- 2 | schemaname | foo 3 | matviewname | matview_1 4 | ispopulated | t 5 | -[ RECORD 2 ]----------------------- 6 | schemaname | foo 7 | matviewname | matview_1_exclude_data 8 | ispopulated | f 9 | 10 | -------------------------------------------------------------------------------- /tests/filtering/exclude/expected/2-sequences.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ]--- 2 | last_value | 667 3 | 4 | -[ RECORD 1 ]--- 5 | last_value | 668 6 | 7 | -[ RECORD 1 ]--- 8 | last_value | 669 9 | 10 | -[ RECORD 1 ]--- 11 | last_value | 670 12 | 13 | -[ RECORD 1 ]--- 14 | last_value | 671 15 | 16 | -------------------------------------------------------------------------------- /tests/filtering/exclude/sql/1-ignore-matview.sql: -------------------------------------------------------------------------------- 1 | select schemaname, matviewname, ispopulated from pg_matviews order by 1, 2; 2 | -------------------------------------------------------------------------------- /tests/filtering/exclude/sql/2-sequences.sql: -------------------------------------------------------------------------------- 1 | select last_value from seq.default_table_id_seq; 2 | select last_value from seq.identity_table_id_seq; 3 | select last_value from seq.standalone_id_seq; 4 | select last_value from seq.standalone_smallint_id_seq; 5 | select last_value from seq.standalone_minvalue_id_seq; 6 | -------------------------------------------------------------------------------- /tests/filtering/include.ini: -------------------------------------------------------------------------------- 1 | [include-only-schema] 2 | public 3 | schema_name_20_chars 4 | 5 | [include-only-table] 6 | public.actor 7 | public.category 8 | public.film 9 | public.film_actor 10 | public.film_category 11 | public.language 12 | public.rental 13 | schema_name_20_chars.very______long______table______name_______50_chars 14 | 15 | [exclude-index] 16 | public.idx_actor_last_name 17 | 18 | [exclude-table-data] 19 | public.rental 20 | -------------------------------------------------------------------------------- /tests/filtering/include/expected/1-empty-rental.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ] 2 | count | 0 3 | 4 | -------------------------------------------------------------------------------- /tests/filtering/include/expected/2-payment-does-not-exists.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ] 2 | exists | f 3 | 4 | -------------------------------------------------------------------------------- /tests/filtering/include/expected/3-schema-bar-does-not-exists.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ]----------------- 2 | nspname | app 3 | -[ RECORD 2 ]----------------- 4 | nspname | copy 5 | -[ RECORD 3 ]----------------- 6 | nspname | foo 7 | -[ RECORD 4 ]----------------- 8 | nspname | partitioned_tables 9 | -[ RECORD 5 ]----------------- 10 | nspname | public 11 | -[ RECORD 6 ]----------------- 12 | nspname | schema_name_20_chars 13 | -[ RECORD 7 ]----------------- 14 | nspname | seq 15 | 16 | -------------------------------------------------------------------------------- /tests/filtering/include/expected/4-exclude-index.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ] 2 | exists | f 3 | 4 | -------------------------------------------------------------------------------- /tests/filtering/include/sql/1-empty-rental.sql: -------------------------------------------------------------------------------- 1 | select count(*) from public.rental; 2 | -------------------------------------------------------------------------------- /tests/filtering/include/sql/2-payment-does-not-exists.sql: -------------------------------------------------------------------------------- 1 | select exists 2 | ( 3 | select 1 4 | from pg_class c 5 | join pg_namespace n on n.oid = c.relnamespace 6 | where n.nspname = 'public' and c.relname = 'payment' 7 | ); 8 | -------------------------------------------------------------------------------- /tests/filtering/include/sql/3-schema-bar-does-not-exists.sql: -------------------------------------------------------------------------------- 1 | select nspname 2 | from pg_namespace 3 | where nspname !~ '^pg_' and nspname <> 'information_schema' 4 | order by nspname; 5 | -------------------------------------------------------------------------------- /tests/filtering/include/sql/4-exclude-index.sql: -------------------------------------------------------------------------------- 1 | select exists(select schemaname, tablename, indexname 2 | from pg_indexes 3 | where schemaname = 'public' 4 | and tablename = 'actor' 5 | and indexname = 'idx_actor_last_name' 6 | ); 7 | -------------------------------------------------------------------------------- /tests/follow-9.6/.dockerignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | README.md 3 | Dockerfile 4 | Dockerfile.pg 5 | Dockerfile.inject 6 | docker-compose.yml 7 | -------------------------------------------------------------------------------- /tests/follow-9.6/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./dml.sql dml.sql 6 | COPY ./ddl.sql ddl.sql 7 | 8 | USER docker 9 | CMD ["/usr/src/pgcopydb/copydb.sh"] 10 | -------------------------------------------------------------------------------- /tests/follow-9.6/Dockerfile.inject: -------------------------------------------------------------------------------- 1 | FROM pgcopydb 2 | 3 | USER root 4 | 5 | RUN apt-get update \ 6 | && apt-get install -y --no-install-recommends jq \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | WORKDIR /usr/src/pgcopydb 10 | COPY ./inject.sh inject.sh 11 | COPY ./dml.sql dml.sql 12 | 13 | USER docker 14 | CMD ["/usr/src/pgcopydb/inject.sh"] 15 | -------------------------------------------------------------------------------- /tests/follow-9.6/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | ARG PGVERSION=9.6 2 | 3 | # 4 | # We use the Docker image for postgres so as to benefit from entry point 5 | # scripts with some level of tweakability. We could use a bare debian image 6 | # instead, but we would have to provide and maintain those scripts. 7 | # 8 | FROM postgres:${PGVERSION} 9 | 10 | ARG PGVERSION 11 | 12 | COPY allow-logical-rep.sh /docker-entrypoint-initdb.d 13 | -------------------------------------------------------------------------------- /tests/follow-9.6/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | PGVERSION ?= 9.6 5 | export PGVERSION 6 | 7 | COMPOSE_EXIT = --exit-code-from=test --abort-on-container-exit 8 | 9 | test: down run down ; 10 | 11 | up: down build 12 | docker compose up $(COMPOSE_EXIT) 13 | 14 | run: build fix-volumes 15 | docker compose run test 16 | 17 | down: 18 | docker compose down --volumes --remove-orphans 19 | 20 | build: 21 | docker compose build 22 | 23 | VPATH = /var/run/pgcopydb 24 | CNAME = follow-9.6 25 | VNAME = follow-9.6 26 | VOLUMES = -v $(VNAME):$(VPATH) 27 | OPTS = --env-file=../paths.env $(VOLUMES) 28 | CLEANUP = make -f /var/lib/postgres/cleanup.mk 29 | 30 | fix-volumes: 31 | docker run --rm $(OPTS) $(CNAME) $(CLEANUP) 32 | 33 | attach: 34 | docker run --rm -it $(OPTS) $(CNAME) bash 35 | 36 | .PHONY: run down build test fix-volumes 37 | -------------------------------------------------------------------------------- /tests/follow-9.6/README.md: -------------------------------------------------------------------------------- 1 | Change Data Capture 2 | =================== 3 | 4 | pgcopydb implements logical decoding through using the wal2json plugin: 5 | 6 | https://github.com/eulerto/wal2json 7 | 8 | This means that changes made to the source database during the copying of 9 | the data can be replayed to the target database. 10 | 11 | This directory implements testing for the change data capture capabilities 12 | of pgcopydb. Tests are using the pagila database, and a set of SQL scripts 13 | that run some DML trafic. 14 | 15 | Follow 9.6 test 16 | =============== 17 | 18 | This test is meant to test pgcopydb Logical Decoding support compatibility 19 | with Postgres 9.6 as a source server. It has also be made compatible with 20 | Postgres 9.5 and Postgres 10, as all of those docker images are using the 21 | same debian stretch distribution. 22 | 23 | ``` 24 | $ make -C tests/follow-9.6 PGVERSION=9.5 up 25 | $ make -C tests/follow-9.6 PGVERSION=9.6 up 26 | $ make -C tests/follow-9.6 PGVERSION=10 up 27 | ``` 28 | -------------------------------------------------------------------------------- /tests/follow-9.6/allow-logical-rep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "host replication all all trust" >> ${PGDATA}/pg_hba.conf 4 | pg_ctl reload 5 | -------------------------------------------------------------------------------- /tests/follow-9.6/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.pg 6 | args: 7 | - PGVERSION 8 | expose: 9 | - 5432 10 | env_file: 11 | - ../postgres.env 12 | command: > 13 | -c wal_level=logical 14 | -c max_replication_slots=4 15 | -c max_wal_senders=4 16 | -c ssl=on 17 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 18 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 19 | 20 | target: 21 | image: postgres:13-bullseye 22 | expose: 23 | - 5432 24 | env_file: 25 | - ../postgres.env 26 | command: > 27 | -c ssl=on 28 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 29 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 30 | 31 | inject: 32 | image: follow-9.6-inject 33 | build: 34 | context: . 35 | dockerfile: Dockerfile.inject 36 | environment: 37 | PGVERSION: 38 | env_file: 39 | - ../uris.env 40 | - ../paths.env 41 | # share TMPDIR between inject and test services 42 | volumes: 43 | - follow-9.6:/var/run/pgcopydb 44 | 45 | test: 46 | image: follow-9.6 47 | build: . 48 | environment: 49 | PGVERSION: 50 | PGCOPYDB_TABLE_JOBS: 4 51 | PGCOPYDB_INDEX_JOBS: 2 52 | PGCOPYDB_SPLIT_TABLES_LARGER_THAN: 200kB 53 | env_file: 54 | - ../uris.env 55 | - ../paths.env 56 | # share TMPDIR between inject and test services 57 | volumes: 58 | - follow-9.6:/var/run/pgcopydb 59 | depends_on: 60 | - source 61 | - target 62 | - inject 63 | 64 | volumes: 65 | follow-9.6: 66 | external: true 67 | -------------------------------------------------------------------------------- /tests/follow-9.6/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | alter table payment replica identity full; 8 | alter table payment_p2022_01 replica identity full; 9 | alter table payment_p2022_02 replica identity full; 10 | alter table payment_p2022_03 replica identity full; 11 | alter table payment_p2022_04 replica identity full; 12 | alter table payment_p2022_05 replica identity full; 13 | alter table payment_p2022_06 replica identity full; 14 | alter table payment_p2022_07 replica identity full; 15 | commit; 16 | -------------------------------------------------------------------------------- /tests/follow-9.6/dml.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database. 5 | 6 | \set customerid1 291 7 | \set customerid2 292 8 | 9 | \set staffid1 1 10 | \set staffid2 2 11 | 12 | \set inventoryid1 371 13 | \set inventoryid2 1097 14 | 15 | begin; 16 | 17 | with r as 18 | ( 19 | insert into rental(rental_date, inventory_id, customer_id, staff_id, last_update) 20 | select '2022-06-01', :inventoryid1, :customerid1, :staffid1, '2022-06-01' 21 | returning rental_id, customer_id, staff_id 22 | ) 23 | insert into payment(customer_id, staff_id, rental_id, amount, payment_date) 24 | select customer_id, staff_id, rental_id, 5.99, '2022-06-01' 25 | from r; 26 | 27 | commit; 28 | 29 | -- update 10 rows in a single UPDATE command 30 | update public.payment set amount = 11.95 where amount = 11.99; 31 | 32 | begin; 33 | 34 | delete from payment 35 | using rental 36 | where rental.rental_id = payment.rental_id 37 | and rental.last_update = '2022-06-01'; 38 | 39 | delete from rental where rental.last_update = '2022-06-01'; 40 | 41 | commit; 42 | 43 | -- 44 | -- update the payments back to their original values 45 | -- 46 | begin; 47 | 48 | update public.payment set amount = 11.99 where amount = 11.95; 49 | 50 | commit; 51 | -------------------------------------------------------------------------------- /tests/follow-data-only/.dockerignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | README.md 3 | Dockerfile 4 | Dockerfile.pg 5 | Dockerfile.inject 6 | docker-compose.yml 7 | -------------------------------------------------------------------------------- /tests/follow-data-only/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./run-background-traffic.sh run-background-traffic.sh 6 | COPY ./dml.sql dml.sql 7 | COPY ./dml-bufsize.sql dml-bufsize.sql 8 | COPY ./ddl.sql ddl.sql 9 | COPY ./multi-wal-txn.sql multi-wal-txn.sql 10 | 11 | USER docker 12 | CMD ["/usr/src/pgcopydb/copydb.sh"] 13 | -------------------------------------------------------------------------------- /tests/follow-data-only/Dockerfile.inject: -------------------------------------------------------------------------------- 1 | FROM pgcopydb 2 | 3 | USER root 4 | 5 | RUN apt-get update \ 6 | && apt-get install -y --no-install-recommends jq sqlite3 \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | WORKDIR /usr/src/pgcopydb 10 | COPY ./inject.sh inject.sh 11 | COPY ./dml.sql dml.sql 12 | COPY ./dml-bufsize.sql dml-bufsize.sql 13 | 14 | USER docker 15 | CMD ["/usr/src/pgcopydb/inject.sh"] 16 | -------------------------------------------------------------------------------- /tests/follow-data-only/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:13-bullseye 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends postgresql-13-wal2json \ 5 | && rm -rf /var/lib/apt/lists/* 6 | -------------------------------------------------------------------------------- /tests/follow-data-only/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | COMPOSE_EXIT = --exit-code-from=test --abort-on-container-exit 5 | 6 | test: down run down ; 7 | 8 | up: down build 9 | docker compose up $(COMPOSE_EXIT) 10 | 11 | run: build fix-volumes 12 | docker compose run test 13 | 14 | down: 15 | docker compose down --volumes --remove-orphans 16 | 17 | build: 18 | docker compose build --quiet 19 | 20 | VPATH = /var/run/pgcopydb 21 | CNAME = follow-data-only 22 | VNAME = follow-data-only 23 | VOLUMES = -v $(VNAME):$(VPATH) 24 | OPTS = --env-file=../paths.env $(VOLUMES) 25 | CLEANUP = make -f /var/lib/postgres/cleanup.mk 26 | 27 | fix-volumes: 28 | docker run --rm $(OPTS) $(CNAME) $(CLEANUP) 29 | 30 | attach: 31 | docker run --rm -it $(OPTS) $(CNAME) bash 32 | 33 | .PHONY: run down build test fix-volumes 34 | -------------------------------------------------------------------------------- /tests/follow-data-only/README.md: -------------------------------------------------------------------------------- 1 | Change Data Capture 2 | =================== 3 | 4 | pgcopydb implements logical decoding through using the wal2json plugin: 5 | 6 | https://github.com/eulerto/wal2json 7 | 8 | This means that changes made to the source database during the copying of 9 | the data can be replayed to the target database. 10 | 11 | This directory implements testing for the change data capture capabilities 12 | of pgcopydb. Tests are using the pagila database, and a set of SQL scripts 13 | that run some DML trafic. 14 | -------------------------------------------------------------------------------- /tests/follow-data-only/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile.pg 6 | expose: 7 | - 5432 8 | env_file: 9 | - ../postgres.env 10 | command: > 11 | -c wal_level=logical 12 | -c ssl=on 13 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 14 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 15 | 16 | target: 17 | image: postgres:15-bullseye 18 | expose: 19 | - 5432 20 | env_file: 21 | - ../postgres.env 22 | command: > 23 | -c ssl=on 24 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 25 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 26 | 27 | inject: 28 | image: follow-data-only-inject 29 | build: 30 | context: . 31 | dockerfile: Dockerfile.inject 32 | env_file: 33 | - ../uris.env 34 | - ../paths.env 35 | # share TMPDIR between inject and test services 36 | volumes: 37 | - follow-data-only:/var/run/pgcopydb 38 | 39 | test: 40 | image: follow-data-only 41 | build: . 42 | environment: 43 | PGCOPYDB_TABLE_JOBS: 4 44 | PGCOPYDB_INDEX_JOBS: 2 45 | env_file: 46 | - ../uris.env 47 | - ../paths.env 48 | # share TMPDIR between inject and test services 49 | volumes: 50 | - follow-data-only:/var/run/pgcopydb 51 | depends_on: 52 | - source 53 | - target 54 | - inject 55 | 56 | volumes: 57 | follow-data-only: 58 | external: true 59 | -------------------------------------------------------------------------------- /tests/follow-data-only/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | 8 | CREATE TABLE table_a (id serial PRIMARY KEY, f1 int4, f2 text); 9 | CREATE TABLE table_b (id serial PRIMARY KEY, f1 int4, f2 text[]); 10 | 11 | commit; 12 | 13 | begin; 14 | 15 | CREATE TABLE IF NOT EXISTS update_test 16 | ( 17 | id bigint primary key, 18 | name text 19 | ); 20 | 21 | commit; 22 | -------------------------------------------------------------------------------- /tests/follow-data-only/dml-bufsize.sql: -------------------------------------------------------------------------------- 1 | -- SELECT x, pg_size_pretty(x::bigint) 2 | -- FROM generate_series(1024, 2 * 1024 * 1024, 128 * 1024) as t(x); 3 | -- ┌─────────┬────────────────┐ 4 | -- │ x │ pg_size_pretty │ 5 | -- ├─────────┼────────────────┤ 6 | -- │ 1024 │ 1024 bytes │ 7 | -- │ 132096 │ 129 kB │ 8 | -- │ 263168 │ 257 kB │ 9 | -- │ 394240 │ 385 kB │ 10 | -- │ 525312 │ 513 kB │ 11 | -- │ 656384 │ 641 kB │ 12 | -- │ 787456 │ 769 kB │ 13 | -- │ 918528 │ 897 kB │ 14 | -- │ 1049600 │ 1025 kB │ 15 | -- │ 1180672 │ 1153 kB │ 16 | -- │ 1311744 │ 1281 kB │ 17 | -- │ 1442816 │ 1409 kB │ 18 | -- │ 1573888 │ 1537 kB │ 19 | -- │ 1704960 │ 1665 kB │ 20 | -- │ 1836032 │ 1793 kB │ 21 | -- │ 1967104 │ 1921 kB │ 22 | -- └─────────┴────────────────┘ 23 | -- (16 rows) 24 | 25 | INSERT INTO table_a (f1, f2) 26 | SELECT x, repeat('x', x) 27 | FROM generate_series(1024, 2 * 1024 * 1024, 128 * 1024) as t(x); 28 | -------------------------------------------------------------------------------- /tests/follow-data-only/dml.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO table_a (f1, f2) 2 | SELECT x, 3 | (array[E'test new\nline', 4 | E'\\\\Client\\S$\\2023\\Dir1\\Test_Doc №2', 5 | E'test \rreturn', 6 | E'test ''single'' quote'])[(x - 1) % 4 + 1] 7 | FROM generate_series(:a,:b) as t(x); 8 | 9 | INSERT INTO table_b(f1, f2) 10 | SELECT x, (regexp_split_to_array(lorem.ipsum, '[ ,.]'))[x:x+5] 11 | FROM (values('Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.')) as lorem(ipsum), 12 | generate_series(:a,:b) as t(x); 13 | -------------------------------------------------------------------------------- /tests/follow-data-only/multi-wal-txn.sql: -------------------------------------------------------------------------------- 1 | -- transaction that spans multiple WAL files 2 | 3 | BEGIN; 4 | 5 | INSERT INTO table_a(f1) VALUES ((random() * 100 + 1)::int); 6 | 7 | SELECT 8 | pg_switch_wal(); 9 | 10 | INSERT INTO table_a(f1) VALUES ((random() * 100 + 1)::int); 11 | 12 | SELECT 13 | pg_switch_wal(); 14 | 15 | INSERT INTO table_a(f1) VALUES ((random() * 100 + 1)::int); 16 | 17 | SELECT 18 | pg_switch_wal(); 19 | 20 | COMMIT; 21 | -------------------------------------------------------------------------------- /tests/follow-data-only/run-background-traffic.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -e 4 | 5 | # Run transactions that insert a variable number of rows at a time. The number 6 | # of rows inserted varies between 1 and 100. Each insert will occur in the 7 | # background and finish quickly which avoids excessive connections. 8 | sql="INSERT INTO table_a(f1) SELECT generate_series(1,(random() * 100 + 1)::int);" 9 | 10 | while true; do 11 | psql -d ${PGCOPYDB_SOURCE_PGURI} -c "${sql}" & 12 | # To control the rate, we include a sleep of 0.1 seconds between each insert 13 | # operation. 14 | sleep 0.1 15 | done 16 | -------------------------------------------------------------------------------- /tests/follow-wal2json/.dockerignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | README.md 3 | Dockerfile 4 | Dockerfile.pg 5 | Dockerfile.inject 6 | docker-compose.yml 7 | -------------------------------------------------------------------------------- /tests/follow-wal2json/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | COPY ./dml.sql dml.sql 6 | COPY ./ddl.sql ddl.sql 7 | 8 | USER docker 9 | CMD ["/usr/src/pgcopydb/copydb.sh"] 10 | -------------------------------------------------------------------------------- /tests/follow-wal2json/Dockerfile.inject: -------------------------------------------------------------------------------- 1 | FROM pgcopydb 2 | 3 | USER root 4 | 5 | RUN apt-get update \ 6 | && apt-get install -y --no-install-recommends jq \ 7 | && rm -rf /var/lib/apt/lists/* 8 | 9 | WORKDIR /usr/src/pgcopydb 10 | COPY ./inject.sh inject.sh 11 | COPY ./dml.sql dml.sql 12 | 13 | USER docker 14 | CMD ["/usr/src/pgcopydb/inject.sh"] 15 | -------------------------------------------------------------------------------- /tests/follow-wal2json/Dockerfile.pg: -------------------------------------------------------------------------------- 1 | FROM postgres:13-bullseye 2 | 3 | RUN apt-get update \ 4 | && apt-get install -y --no-install-recommends postgresql-13-wal2json \ 5 | && rm -rf /var/lib/apt/lists/* 6 | -------------------------------------------------------------------------------- /tests/follow-wal2json/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | COMPOSE_EXIT = --exit-code-from=test --abort-on-container-exit 5 | 6 | test: down run down ; 7 | 8 | up: down build 9 | docker compose up $(COMPOSE_EXIT) 10 | 11 | run: build fix-volumes 12 | docker compose run test 13 | 14 | down: 15 | docker compose down 16 | 17 | build: 18 | docker compose build --quiet 19 | 20 | VPATH = /var/run/pgcopydb 21 | CNAME = follow-wal2json 22 | VNAME = follow-wal2json 23 | VOLUMES = -v $(VNAME):$(VPATH) 24 | OPTS = --env-file=../paths.env $(VOLUMES) 25 | CLEANUP = make -f /var/lib/postgres/cleanup.mk 26 | 27 | fix-volumes: 28 | docker run --rm $(OPTS) $(CNAME) $(CLEANUP) 29 | 30 | attach: 31 | docker run --rm -it $(OPTS) $(CNAME) bash 32 | 33 | .PHONY: run down build test fix-volumes 34 | -------------------------------------------------------------------------------- /tests/follow-wal2json/README.md: -------------------------------------------------------------------------------- 1 | Change Data Capture 2 | =================== 3 | 4 | pgcopydb implements logical decoding through using the wal2json plugin: 5 | 6 | https://github.com/eulerto/wal2json 7 | 8 | This means that changes made to the source database during the copying of 9 | the data can be replayed to the target database. 10 | 11 | This directory implements testing for the change data capture capabilities 12 | of pgcopydb. Tests are using the pagila database, and a set of SQL scripts 13 | that run some DML trafic. 14 | -------------------------------------------------------------------------------- /tests/follow-wal2json/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres-wal2json:13-bullseye 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.pg 7 | expose: 8 | - 5432 9 | env_file: 10 | - ../postgres.env 11 | command: > 12 | -c wal_level=logical 13 | -c ssl=on 14 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 15 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 16 | 17 | target: 18 | image: postgres:16-bullseye 19 | expose: 20 | - 5432 21 | env_file: 22 | - ../postgres.env 23 | command: > 24 | -c ssl=on 25 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 26 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 27 | 28 | inject: 29 | build: 30 | context: . 31 | dockerfile: Dockerfile.inject 32 | env_file: 33 | - ../uris.env 34 | - ../paths.env 35 | # share TMPDIR between inject and test services 36 | volumes: 37 | - follow-wal2json:/var/run/pgcopydb 38 | 39 | test: 40 | image: follow-wal2json 41 | build: . 42 | cap_add: 43 | - SYS_ADMIN 44 | - SYS_PTRACE 45 | environment: 46 | PGCOPYDB_TABLE_JOBS: 4 47 | PGCOPYDB_INDEX_JOBS: 2 48 | env_file: 49 | - ../uris.env 50 | - ../paths.env 51 | # share TMPDIR between inject and test services 52 | volumes: 53 | - follow-wal2json:/var/run/pgcopydb 54 | depends_on: 55 | - source 56 | - target 57 | - inject 58 | 59 | volumes: 60 | follow-wal2json: 61 | external: true 62 | -------------------------------------------------------------------------------- /tests/follow-wal2json/copydb.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | # - PGCOPYDB_TARGET_PGURI 10 | # - PGCOPYDB_TABLE_JOBS 11 | # - PGCOPYDB_INDEX_JOBS 12 | 13 | # make sure source and target databases are ready 14 | pgcopydb ping 15 | 16 | psql -o /tmp/s.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /usr/src/pagila/pagila-schema.sql 17 | psql -o /tmp/d.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /usr/src/pagila/pagila-data.sql 18 | 19 | # alter the pagila schema to allow capturing DDLs without pkey 20 | psql -d ${PGCOPYDB_SOURCE_PGURI} -f /usr/src/pgcopydb/ddl.sql 21 | 22 | # pgcopydb clone uses the environment variables 23 | pgcopydb clone --follow --plugin wal2json 24 | 25 | # cleanup 26 | pgcopydb stream sentinel get 27 | 28 | # make sure the inject service has had time to see the final sentinel values 29 | sleep 2 30 | pgcopydb stream cleanup 31 | -------------------------------------------------------------------------------- /tests/follow-wal2json/ddl.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/ddl.sql 3 | --- 4 | --- This file implements DDL changes in the pagila database. 5 | 6 | begin; 7 | alter table payment_p2022_01 replica identity full; 8 | alter table payment_p2022_02 replica identity full; 9 | alter table payment_p2022_03 replica identity full; 10 | alter table payment_p2022_04 replica identity full; 11 | alter table payment_p2022_05 replica identity full; 12 | alter table payment_p2022_06 replica identity full; 13 | alter table payment_p2022_07 replica identity full; 14 | commit; 15 | -------------------------------------------------------------------------------- /tests/follow-wal2json/dml.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- pgcopydb test/cdc/dml.sql 3 | --- 4 | --- This file implements DML changes in the pagila database. 5 | 6 | \set customerid1 291 7 | \set customerid2 292 8 | 9 | \set staffid1 1 10 | \set staffid2 2 11 | 12 | \set inventoryid1 371 13 | \set inventoryid2 1097 14 | 15 | begin; 16 | 17 | with r as 18 | ( 19 | insert into rental(rental_date, inventory_id, customer_id, staff_id, last_update) 20 | select '2022-06-01', :inventoryid1, :customerid1, :staffid1, '2022-06-01' 21 | returning rental_id, customer_id, staff_id 22 | ) 23 | insert into payment(customer_id, staff_id, rental_id, amount, payment_date) 24 | select customer_id, staff_id, rental_id, 5.99, '2022-06-01' 25 | from r; 26 | 27 | commit; 28 | 29 | -- update 10 rows in a single UPDATE command 30 | update public.payment set amount = 11.95 where amount = 11.99; 31 | 32 | begin; 33 | 34 | delete from payment 35 | using rental 36 | where rental.rental_id = payment.rental_id 37 | and rental.last_update = '2022-06-01'; 38 | 39 | delete from rental where rental.last_update = '2022-06-01'; 40 | 41 | commit; 42 | 43 | -- 44 | -- update the payments back to their original values 45 | -- 46 | begin; 47 | 48 | update public.payment set amount = 11.99 where amount = 11.95; 49 | 50 | commit; 51 | -------------------------------------------------------------------------------- /tests/follow-wal2json/inject.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | # - PGCOPYDB_TARGET_PGURI 10 | # - PGCOPYDB_TABLE_JOBS 11 | # - PGCOPYDB_INDEX_JOBS 12 | 13 | pgcopydb ping 14 | 15 | # 16 | # Only start injecting DML traffic on the source database when the pagila 17 | # schema and base data set has been deployed already. Our proxy to know that 18 | # that's the case is the existence of the pgcopydb.sentinel table on the 19 | # source database. 20 | # 21 | dbfile=${TMPDIR}/pgcopydb/schema/source.db 22 | 23 | until [ -s ${dbfile} ] 24 | do 25 | sleep 1 26 | done 27 | 28 | # 29 | # Inject changes from our DML file in a loop, again and again. 30 | # 31 | # Every other round of DML changes, we also force the source server to 32 | # switch to another WAL file, to test that our streaming solution can follow 33 | # WAL file changes. 34 | # 35 | for i in `seq 5` 36 | do 37 | psql -d ${PGCOPYDB_SOURCE_PGURI} -f /usr/src/pgcopydb/dml.sql 38 | sleep 1 39 | 40 | psql -d ${PGCOPYDB_SOURCE_PGURI} -f /usr/src/pgcopydb/dml.sql 41 | sleep 1 42 | 43 | psql -d ${PGCOPYDB_SOURCE_PGURI} -c 'select pg_switch_wal()' 44 | sleep 1 45 | done 46 | 47 | # grab the current LSN, it's going to be our streaming end position 48 | lsn=`psql -At -d ${PGCOPYDB_SOURCE_PGURI} -c 'select pg_current_wal_flush_lsn()'` 49 | 50 | pgcopydb stream sentinel set endpos --current --debug 51 | pgcopydb stream sentinel get 52 | 53 | endpos=`pgcopydb stream sentinel get --endpos 2>/dev/null` 54 | 55 | if [ ${endpos} = "0/0" ] 56 | then 57 | echo "expected ${lsn} endpos, found ${endpos}" 58 | exit 1 59 | fi 60 | 61 | # 62 | # Because we're using docker-compose --abort-on-container-exit make sure 63 | # that the other process in the pgcopydb service is done before exiting 64 | # here. 65 | # 66 | flushlsn="0/0" 67 | 68 | while [ ${flushlsn} \< ${endpos} ] 69 | do 70 | flushlsn=`pgcopydb stream sentinel get --flush-lsn 2>/dev/null` 71 | sleep 1 72 | done 73 | 74 | # 75 | # Still give some time to the pgcopydb service to finish its processing, 76 | # with the cleanup and all. 77 | # 78 | sleep 10 79 | -------------------------------------------------------------------------------- /tests/pagila-multi-steps/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | COPY ./copydb.sh copydb.sh 5 | 6 | USER docker 7 | CMD ["/usr/src/pgcopydb/copydb.sh"] 8 | -------------------------------------------------------------------------------- /tests/pagila-multi-steps/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/pagila-multi-steps/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | target: 11 | image: postgres:13-bullseye 12 | expose: 13 | - 5432 14 | environment: 15 | POSTGRES_USER: postgres 16 | POSTGRES_PASSWORD: h4ckm3 17 | POSTGRES_HOST_AUTH_METHOD: trust 18 | test: 19 | build: . 20 | environment: 21 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 22 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 23 | PGCOPYDB_TABLE_JOBS: 2 24 | PGCOPYDB_INDEX_JOBS: 2 25 | depends_on: 26 | - source 27 | - target 28 | -------------------------------------------------------------------------------- /tests/pagila-multi-steps/copydb.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | # - PGCOPYDB_TARGET_PGURI 10 | # - PGCOPYDB_TABLE_JOBS 11 | # - PGCOPYDB_INDEX_JOBS 12 | 13 | # make sure source and target databases are ready 14 | pgcopydb ping 15 | 16 | psql -o /tmp/d.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /usr/src/pagila/pagila-schema.sql 17 | psql -o /tmp/s.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /usr/src/pagila/pagila-data.sql 18 | 19 | psql -d ${PGCOPYDB_TARGET_PGURI} < /tmp/pagila-schema.sql 21 | 22 | psql -o /tmp/s.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /tmp/pagila-schema.sql 23 | psql -o /tmp/d.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /usr/src/pagila/pagila-data.sql 24 | 25 | pgcopydb clone --skip-ext-comments --notice \ 26 | --source ${PGCOPYDB_SOURCE_STANDBY_PGURI} \ 27 | --target ${PGCOPYDB_TARGET_PGURI} 28 | 29 | pgcopydb compare schema \ 30 | --source ${PGCOPYDB_SOURCE_STANDBY_PGURI} \ 31 | --target ${PGCOPYDB_TARGET_PGURI} 32 | 33 | pgcopydb compare data \ 34 | --source ${PGCOPYDB_SOURCE_STANDBY_PGURI} \ 35 | --target ${PGCOPYDB_TARGET_PGURI} 36 | -------------------------------------------------------------------------------- /tests/pagila/Dockerfile: -------------------------------------------------------------------------------- 1 | # that image is built by docker compose build: project-service 2 | FROM pagila 3 | 4 | COPY --from=pgcopydb /usr/local/bin/pgcopydb /usr/local/bin 5 | 6 | WORKDIR /usr/src/pgcopydb 7 | COPY copydb.sh copydb.sh 8 | 9 | USER docker 10 | CMD ["/usr/src/pgcopydb/copydb.sh"] 11 | -------------------------------------------------------------------------------- /tests/pagila/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: down build test 16 | -------------------------------------------------------------------------------- /tests/pagila/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | command: > 11 | -c ssl=on 12 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 13 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 14 | target: 15 | image: postgres:13-bullseye 16 | expose: 17 | - 5432 18 | environment: 19 | POSTGRES_USER: postgres 20 | POSTGRES_PASSWORD: h4ckm3 21 | POSTGRES_HOST_AUTH_METHOD: trust 22 | command: > 23 | -c ssl=on 24 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 25 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 26 | test: 27 | build: 28 | context: . 29 | dockerfile: Dockerfile 30 | cap_add: 31 | - SYS_ADMIN 32 | - SYS_PTRACE 33 | environment: 34 | PGSSLMODE: "require" 35 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 36 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 37 | PGCOPYDB_TABLE_JOBS: 4 38 | PGCOPYDB_INDEX_JOBS: 4 39 | PGCOPYDB_LARGE_OBJECTS_JOBS: 4 40 | PGCOPYDB_SPLIT_TABLES_LARGER_THAN: 200kB 41 | PGCOPYDB_SPLIT_MAX_PARTS: 5 42 | PGCOPYDB_FAIL_FAST: "true" 43 | depends_on: 44 | - source 45 | - target 46 | -------------------------------------------------------------------------------- /tests/pagila/copydb.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | # - PGCOPYDB_TARGET_PGURI 10 | # - PGCOPYDB_TABLE_JOBS 11 | # - PGCOPYDB_INDEX_JOBS 12 | 13 | # make sure source and target databases are ready 14 | pgcopydb ping 15 | 16 | # 17 | # Now create a user that's going to be the owner of our database 18 | # 19 | psql -a ${PGCOPYDB_SOURCE_PGURI} < /tmp/pagila-schema.sql 56 | 57 | psql -o /tmp/s.out -d ${PAGILA_SOURCE_PGURI} -1 -f /tmp/pagila-schema.sql 58 | psql -o /tmp/d.out -d ${PAGILA_SOURCE_PGURI} -1 -f /usr/src/pagila/pagila-data.sql 59 | 60 | pgcopydb clone --notice \ 61 | --skip-ext-comments \ 62 | --skip-db-properties \ 63 | --estimate-table-sizes \ 64 | --use-copy-binary \ 65 | --source ${PAGILA_SOURCE_PGURI} \ 66 | --target ${PAGILA_TARGET_PGURI} 67 | 68 | pgcopydb compare schema \ 69 | --source ${PAGILA_SOURCE_PGURI} \ 70 | --target ${PAGILA_TARGET_PGURI} 71 | 72 | pgcopydb compare data \ 73 | --source ${PAGILA_SOURCE_PGURI} \ 74 | --target ${PAGILA_TARGET_PGURI} 75 | -------------------------------------------------------------------------------- /tests/paths.env: -------------------------------------------------------------------------------- 1 | TMPDIR=/var/run/pgcopydb 2 | XDG_DATA_HOME=/var/run/pgcopydb/cdc 3 | -------------------------------------------------------------------------------- /tests/postgres.env: -------------------------------------------------------------------------------- 1 | POSTGRES_USER=postgres 2 | POSTGRES_PASSWORD=h4ckm3 3 | POSTGRES_HOST_AUTH_METHOD=trust 4 | -------------------------------------------------------------------------------- /tests/timescaledb/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM pagila 2 | 3 | WORKDIR /usr/src/pgcopydb 4 | 5 | COPY ./copydb.sh copydb.sh 6 | COPY ./rides.sql rides.sql 7 | COPY ./fares.sql fares.sql 8 | COPY ./nyc_data_rides.10k.csv nyc_data_rides.10k.csv 9 | 10 | USER docker 11 | CMD ["/usr/src/pgcopydb/copydb.sh"] 12 | -------------------------------------------------------------------------------- /tests/timescaledb/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | COMPOSE_EXIT = --exit-code-from=test --abort-on-container-exit 5 | 6 | test: down run down ; 7 | 8 | up: down build 9 | docker compose up $(COMPOSE_EXIT) 10 | 11 | run: build 12 | docker compose run test 13 | 14 | down: 15 | docker compose down 16 | 17 | build: 18 | docker compose build --quiet 19 | 20 | .PHONY: run down build test 21 | -------------------------------------------------------------------------------- /tests/timescaledb/compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: timescale/timescaledb:latest-pg13 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | target: 11 | image: timescale/timescaledb:latest-pg13 12 | expose: 13 | - 5432 14 | environment: 15 | POSTGRES_USER: postgres 16 | POSTGRES_PASSWORD: h4ckm3 17 | POSTGRES_HOST_AUTH_METHOD: trust 18 | test: 19 | build: . 20 | environment: 21 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 22 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 23 | PGCOPYDB_TABLE_JOBS: 4 24 | PGCOPYDB_INDEX_JOBS: 2 25 | depends_on: 26 | - source 27 | - target 28 | -------------------------------------------------------------------------------- /tests/timescaledb/copydb.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | # - PGCOPYDB_TARGET_PGURI 10 | # - PGCOPYDB_TABLE_JOBS 11 | # - PGCOPYDB_INDEX_JOBS 12 | 13 | # make sure source and target databases are ready 14 | pgcopydb ping 15 | 16 | # copying roles needs superuser 17 | # and we use the postgres database here still 18 | pgcopydb copy roles 19 | 20 | psql -o /tmp/c.out -d ${PGCOPYDB_SOURCE_PGURI} -1 -f /usr/src/pgcopydb/rides.sql 21 | 22 | copy="\COPY rides FROM nyc_data_rides.10k.csv CSV" 23 | psql -d ${PGCOPYDB_SOURCE_PGURI} -1 -c "${copy}" 24 | 25 | # take a snapshot using role tsdb on source database 26 | coproc ( pgcopydb snapshot --debug ) 27 | 28 | sleep 1 29 | 30 | # copy the extensions separately, needs superuser (both on source and target) 31 | pgcopydb copy extensions 32 | 33 | # now clone with superuser privileges, seems to be required for timescaledb 34 | pgcopydb clone --skip-extensions --no-owner 35 | 36 | kill -TERM ${COPROC_PID} 37 | wait ${COPROC_PID} 38 | 39 | s=/tmp/source_fares.out 40 | t=/tmp/target_fares.out 41 | psql -o $s -d ${PGCOPYDB_SOURCE_PGURI} -f /usr/src/pgcopydb/fares.sql 42 | psql -o $t -d ${PGCOPYDB_TARGET_PGURI} -f /usr/src/pgcopydb/fares.sql 43 | 44 | diff $s $t 45 | 46 | pgcopydb compare schema 47 | pgcopydb compare data 48 | 49 | # check timescaledb hypertables and chunks catalogs 50 | psql -d ${PGCOPYDB_SOURCE_PGURI} -1 -c "SELECT * FROM timescaledb_information.hypertables" > /tmp/source_hypertables.out 51 | psql -d ${PGCOPYDB_TARGET_PGURI} -1 -c "SELECT * FROM timescaledb_information.hypertables" > /tmp/target_hypertables.out 52 | 53 | diff /tmp/source_hypertables.out /tmp/target_hypertables.out 54 | 55 | psql -d ${PGCOPYDB_SOURCE_PGURI} -1 -c "SELECT * FROM timescaledb_information.chunks" > /tmp/source_chunks.out 56 | psql -d ${PGCOPYDB_TARGET_PGURI} -1 -c "SELECT * FROM timescaledb_information.chunks" > /tmp/target_chunks.out 57 | 58 | diff /tmp/source_chunks.out /tmp/target_chunks.out 59 | -------------------------------------------------------------------------------- /tests/timescaledb/fares.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- https://docs.timescale.com/tutorials/latest/nyc-taxi-cab/query-nyc/ 3 | -- 4 | 5 | SELECT rate_code, COUNT(vendor_id) AS num_trips 6 | FROM rides 7 | WHERE pickup_datetime < '2016-01-08' 8 | GROUP BY rate_code 9 | ORDER BY rate_code; 10 | -------------------------------------------------------------------------------- /tests/timescaledb/rides.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- https://docs.timescale.com/tutorials/latest/nyc-taxi-cab/dataset-nyc/ 3 | -- 4 | 5 | BEGIN; 6 | 7 | CREATE EXTENSION IF NOT EXISTS timescaledb; 8 | 9 | CREATE TABLE "rides"( 10 | vendor_id TEXT, 11 | pickup_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, 12 | dropoff_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, 13 | passenger_count NUMERIC, 14 | trip_distance NUMERIC, 15 | pickup_longitude NUMERIC, 16 | pickup_latitude NUMERIC, 17 | rate_code INTEGER, 18 | dropoff_longitude NUMERIC, 19 | dropoff_latitude NUMERIC, 20 | payment_type INTEGER, 21 | fare_amount NUMERIC, 22 | extra NUMERIC, 23 | mta_tax NUMERIC, 24 | tip_amount NUMERIC, 25 | tolls_amount NUMERIC, 26 | improvement_surcharge NUMERIC, 27 | total_amount NUMERIC 28 | ); 29 | SELECT create_hypertable('rides', 'pickup_datetime', 'payment_type', 2, create_default_indexes=>FALSE); 30 | CREATE INDEX ON rides (vendor_id, pickup_datetime desc); 31 | CREATE INDEX ON rides (pickup_datetime desc, vendor_id); 32 | CREATE INDEX ON rides (rate_code, pickup_datetime DESC); 33 | CREATE INDEX ON rides (passenger_count, pickup_datetime desc); 34 | 35 | CREATE TABLE IF NOT EXISTS "payment_types"( 36 | payment_type INTEGER, 37 | description TEXT 38 | ); 39 | INSERT INTO payment_types(payment_type, description) VALUES 40 | (1, 'credit card'), 41 | (2, 'cash'), 42 | (3, 'no charge'), 43 | (4, 'dispute'), 44 | (5, 'unknown'), 45 | (6, 'voided trip'); 46 | 47 | CREATE TABLE IF NOT EXISTS "rates"( 48 | rate_code INTEGER, 49 | description TEXT 50 | ); 51 | INSERT INTO rates(rate_code, description) VALUES 52 | (1, 'standard rate'), 53 | (2, 'JFK'), 54 | (3, 'Newark'), 55 | (4, 'Nassau or Westchester'), 56 | (5, 'negotiated fare'), 57 | (6, 'group ride'); 58 | 59 | COMMIT; 60 | -------------------------------------------------------------------------------- /tests/unit/Dockerfile: -------------------------------------------------------------------------------- 1 | # that image is built by docker compose build: project-service 2 | FROM pagila 3 | 4 | WORKDIR /usr/src/pgcopydb 5 | COPY ./copydb.sh copydb.sh 6 | COPY ./setup ./test/setup 7 | COPY ./sql ./test/sql 8 | COPY ./script ./test/script 9 | COPY ./expected ./test/expected 10 | 11 | USER docker 12 | WORKDIR /usr/src/pgcopydb/test/ 13 | CMD ["/usr/src/pgcopydb/copydb.sh"] 14 | -------------------------------------------------------------------------------- /tests/unit/Makefile: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2021 The PostgreSQL Global Development Group. 2 | # Licensed under the PostgreSQL License. 3 | 4 | test: down run down ; 5 | 6 | run: build 7 | docker compose run test 8 | 9 | down: 10 | docker compose down 11 | 12 | build: 13 | docker compose build --quiet 14 | 15 | .PHONY: run down build test 16 | -------------------------------------------------------------------------------- /tests/unit/README.md: -------------------------------------------------------------------------------- 1 | # Unit/Regression Testing for pgcopydb 2 | 3 | In addition to the pagila and large object testing, when a specific issue is 4 | opened on pgcopydb we might want to add testing that covers just the failing 5 | bits. 6 | 7 | This testing directory is meant to allow for covering those extra regression 8 | testing / unit testing. At the moment there are three modes of operations: 9 | 10 | 1. the pgcopydb command is expected to return a zero return code (unix 11 | command success) 12 | 13 | 2. the pgcopydb is expected to have done something specific on the target 14 | database and we want to check that. 15 | 16 | 3. the pgcopydb command should generate the expected output without performing 17 | the full migration. 18 | 19 | ## Regression testing 20 | 21 | In the spirit of pg_regress, the regression testing is done in the following 22 | three steps: 23 | 24 | 1. run the setup/*.sql file with psql 25 | 26 | This creates the testing environment with tables, constraints, data, etc 27 | 28 | 2. for each file in the sql and scripts directory, run it with psql (against 29 | the target database) or bash, respectively, and capture the output 30 | 31 | 3. compare the previous step output to the expected/${test}.out file 32 | -------------------------------------------------------------------------------- /tests/unit/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | source: 3 | image: postgres:13-bullseye 4 | expose: 5 | - 5432 6 | environment: 7 | POSTGRES_USER: postgres 8 | POSTGRES_PASSWORD: h4ckm3 9 | POSTGRES_HOST_AUTH_METHOD: trust 10 | target: 11 | image: postgres:13-bullseye 12 | expose: 13 | - 5432 14 | environment: 15 | POSTGRES_USER: postgres 16 | POSTGRES_PASSWORD: h4ckm3 17 | POSTGRES_HOST_AUTH_METHOD: trust 18 | test: 19 | build: . 20 | environment: 21 | PGCOPYDB_SOURCE_PGURI: postgres://postgres:h4ckm3@source/postgres 22 | PGCOPYDB_TARGET_PGURI: postgres://postgres:h4ckm3@target/postgres 23 | PGCOPYDB_TABLE_JOBS: 4 24 | PGCOPYDB_INDEX_JOBS: 2 25 | depends_on: 26 | - source 27 | - target 28 | -------------------------------------------------------------------------------- /tests/unit/copydb.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | # - PGCOPYDB_TARGET_PGURI 10 | # - PGCOPYDB_TABLE_JOBS 11 | # - PGCOPYDB_INDEX_JOBS 12 | 13 | # make sure source and target databases are ready 14 | pgcopydb ping 15 | 16 | sql='ALTER DATABASE postgres SET search_path TO public, """abc""";' 17 | psql -a -d "${PGCOPYDB_SOURCE_PGURI}" -c "${sql}" 18 | psql -a -d "${PGCOPYDB_SOURCE_PGURI}" -1 -f ./setup/setup.sql 19 | 20 | # create the target needed collation manually for the test 21 | psql -a -d "${PGCOPYDB_TARGET_PGURI}" -1 < $r 47 | test -f $e || cat $r 48 | diff $e $r || exit 1 49 | done 50 | 51 | 52 | for f in ./script/*.sh 53 | do 54 | t=`basename $f .sh` 55 | r=/tmp/results/${t}.out 56 | e=./expected/${t}.out 57 | bash $f > $r 58 | test -f $e || cat $r 59 | # exclude logs, whitespaces and blank lines 60 | DIFFOPTS='-B -w -I INFO -I WARN' 61 | diff ${DIFFOPTS} $e $r || cat $r 62 | diff ${DIFFOPTS} $e $r || exit 1 63 | done 64 | -------------------------------------------------------------------------------- /tests/unit/expected/1-exclusion-constraints.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ]--------+------------------------------------------------------------------- 2 | nspname | public 3 | relname | constraint_fail_idx 4 | nspname | public 5 | relname | exclcon 6 | indisprimary | f 7 | indisunique | f 8 | cols | a 9 | pg_get_indexdef | CREATE INDEX constraint_fail_idx ON public.exclcon USING btree (a) 10 | conname | constraint_fail_idx 11 | pg_get_constraintdef | EXCLUDE USING btree (a WITH =) 12 | 13 | -------------------------------------------------------------------------------- /tests/unit/expected/2-inherit.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ]---- 2 | id | 11000 3 | a | 1 4 | b | f 5 | c | [1, 2, 3, 4] 6 | -[ RECORD 2 ]---- 7 | id | 10999 8 | a | 1 9 | b | f 10 | c | [1, 2, 3, 4] 11 | 12 | -------------------------------------------------------------------------------- /tests/unit/expected/3-collations.out: -------------------------------------------------------------------------------- 1 | OID | Name | Object name 2 | -----------+----------------------+--------------------- 3 | 12329 | en_US.utf8 | database postgres 4 | 12440 | de-DE-x-icu | column f2 of table colls 5 | 12559 | en-US-x-icu | column f2 of table mycoltab 6 | 12658 | fr-FR-x-icu | column f1 of table colls 7 | 16418 | mycol | column f1 of table mycoltab 8 | -------------------------------------------------------------------------------- /tests/unit/expected/3-string-escape.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ] 2 | id | 1 3 | f1 | aaa + 4 | | aaa 5 | -[ RECORD 2 ] 6 | id | 2 7 | f1 | bbb\r+ 8 | | bbb 9 | -[ RECORD 3 ] 10 | id | 3 11 | f1 | ccc 12 | 13 | -------------------------------------------------------------------------------- /tests/unit/expected/4-parent-data.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ] 2 | count | 0 3 | 4 | -------------------------------------------------------------------------------- /tests/unit/expected/5-attgenerated.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ] 2 | nb | 1 3 | 4 | -------------------------------------------------------------------------------- /tests/unit/expected/6-sequences.out: -------------------------------------------------------------------------------- 1 | -[ RECORD 1 ]--- 2 | last_value | 667 3 | 4 | -[ RECORD 1 ]--- 5 | last_value | 668 6 | 7 | -[ RECORD 1 ]--- 8 | last_value | 669 9 | 10 | -[ RECORD 1 ]--- 11 | last_value | 670 12 | 13 | -[ RECORD 1 ]--- 14 | last_value | 671 15 | 16 | -------------------------------------------------------------------------------- /tests/unit/script/3-collations.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variables to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | 10 | pgcopydb list collations -q --dir /tmp/collations 2>&1 11 | -------------------------------------------------------------------------------- /tests/unit/script/4-list-table-split.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -x 4 | set -e 5 | 6 | # This script expects the following environment variable(s) to be set: 7 | # 8 | # - PGCOPYDB_SOURCE_PGURI 9 | 10 | # `pgcopydb list table-parts` will use the table-size cache populated in 11 | # `pgcopydb.table_size` in '4-list-table-split.sql'. 12 | # 13 | # The values stored in pgcopydb.table_size cache may not accurate, but the 14 | # pgcopydb commands will make decisions based on the available information in 15 | # the cache. 16 | 17 | DIR=/tmp/unit/split 18 | OPTS="--not-consistent --split-tables-larger-than 10kB" 19 | 20 | pgcopydb list schema --dir ${DIR} ${OPTS} >/dev/null 21 | 22 | # Cached size of table_1 is 100 KB, so this will be split into 10 parts 23 | pgcopydb list table-parts --dir ${DIR} \ 24 | --schema-name "public" --table-name "table_1" \ 25 | --split-tables-larger-than "10 kB" 2>&1 26 | 27 | # table_2 is identical to table_1 but with the size of 50 KB, so this will be 28 | # split into 5 parts 29 | pgcopydb list table-parts --dir ${DIR} \ 30 | --schema-name "public" --table-name "table_2" \ 31 | --split-tables-larger-than "10 kB" 2>&1 32 | 33 | # table_3 doesn't have size in cache, therefore it will not be split 34 | pgcopydb list table-parts --dir ${DIR} \ 35 | --schema-name "public" --table-name "table_3" \ 36 | --split-tables-larger-than "10 kB" 2>&1 37 | 38 | # table_ctid_candidate doesn't have a unique integer, therefore it will be split by ctid 39 | pgcopydb list table-parts --dir ${DIR} \ 40 | --schema-name "public" --table-name "table_ctid_candidate" \ 41 | --split-tables-larger-than "10 kB" 2>&1 42 | 43 | # ctid split is disabled, no partitioning will be done 44 | pgcopydb list table-parts --dir ${DIR} \ 45 | --schema-name "public" --table-name "table_ctid_candidate_skip" \ 46 | --split-tables-larger-than "10 kB" \ 47 | --skip-split-by-ctid 2>&1 48 | 49 | 50 | # Now we will test the limits on the number of parts 51 | DIR=/tmp/unit/split-with-limits 52 | OPTS+=" --split-max-parts 3" 53 | 54 | pgcopydb list schema --dir ${DIR} ${OPTS} >/dev/null 55 | 56 | # limits on the number of parts will be respected, and we will see 3 parts now 57 | pgcopydb list table-parts --dir ${DIR} \ 58 | --schema-name "public" --table-name "table_1" \ 59 | --split-tables-larger-than "10 kB" --split-max-parts 3 2>&1 60 | -------------------------------------------------------------------------------- /tests/unit/setup/1-exclusion-constraints.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS exclcon; 2 | 3 | CREATE TABLE exclcon (a integer, b integer); 4 | 5 | insert into exclcon values (1,2), (3,4); 6 | 7 | ALTER TABLE exclcon 8 | ADD CONSTRAINT constraint_fail_idx EXCLUDE USING btree (a WITH=); 9 | -------------------------------------------------------------------------------- /tests/unit/setup/10-search-path-index-enum.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- Index creation fails when referencing an object due to search_path issues 3 | --- 4 | --- See https://github.com/dimitri/pgcopydb/issues/489 5 | 6 | BEGIN; 7 | 8 | CREATE SCHEMA """abc"""; 9 | 10 | CREATE TYPE """abc""".state AS ENUM ( 11 | 'PROPOSED', 12 | 'SCHEDULED', 13 | 'STARTED' 14 | ); 15 | 16 | CREATE TABLE """abc""".job ( 17 | id bigint NOT NULL, 18 | state """abc""".state DEFAULT 'SCHEDULED'::"""abc""".state NOT NULL, 19 | date date NOT NULL 20 | ); 21 | 22 | CREATE INDEX indexname ON """abc""".job USING btree (state) WHERE (state = 'SCHEDULED'::state); 23 | 24 | COMMIT; 25 | -------------------------------------------------------------------------------- /tests/unit/setup/11-check-constraint.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE public.employees ( 2 | name character varying NOT NULL, 3 | emp_id uuid 4 | ); 5 | 6 | -- 7 | -- See https://github.com/dimitri/pgcopydb/issues/498 8 | -- 9 | -- Migration fails when handling CHECK CONSTRAINT 10 | -- 11 | 12 | ALTER TABLE public.employees 13 | ADD CONSTRAINT chk_employees_name_not_abc CHECK (((name)::text !~ 'abc'::text)) NOT VALID; 14 | -------------------------------------------------------------------------------- /tests/unit/setup/12-generated-column.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- See https://github.com/dimitri/pgcopydb/issues/496 3 | -- 4 | -- Migration fails when table has generated columns 5 | -- 6 | 7 | CREATE TABLE ref_admin_commune ( 8 | id character varying NOT NULL, 9 | nom character varying(50), 10 | test text GENERATED ALWAYS AS (upper((nom)::text)) STORED 11 | ); 12 | -------------------------------------------------------------------------------- /tests/unit/setup/13-parent-data.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS public.parent_model CASCADE; 2 | 3 | CREATE TABLE IF NOT EXISTS public.parent_model 4 | ( 5 | id integer NOT NULL, 6 | a integer NOT NULL, 7 | b boolean NOT NULL, 8 | CONSTRAINT pk_parent_model PRIMARY KEY (id) 9 | ); 10 | 11 | CREATE TABLE IF NOT EXISTS public.child_model_1 12 | ( 13 | -- Inherited from table public.parent_model: id integer NOT NULL, 14 | -- Inherited from table public.parent_model: a integer NOT NULL, 15 | -- Inherited from table public.parent_model: b boolean NOT NULL, 16 | c jsonb NOT NULL, 17 | CONSTRAINT pk_child_model_1 PRIMARY KEY (id) 18 | ) 19 | INHERITS (public.parent_model); 20 | 21 | INSERT INTO child_model_1 (id, a, b, c) 22 | SELECT 23 | 10000 + s, 24 | 1, 25 | false, 26 | '[1, 2, 3, 4]' 27 | FROM 28 | generate_series(1, 1000) s; 29 | -------------------------------------------------------------------------------- /tests/unit/setup/14-uuid.sql: -------------------------------------------------------------------------------- 1 | create extension "uuid-ossp"; 2 | 3 | create table uuid(id uuid default uuid_generate_v4(), f1 text); 4 | 5 | insert into uuid(f1) 6 | select x::text from generate_series(100, 199, 1) as t(x); 7 | -------------------------------------------------------------------------------- /tests/unit/setup/15-attgenerated.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- See https://github.com/dimitri/pgcopydb/issues/757 3 | --- 4 | 5 | create table gen 6 | ( 7 | nb int generated always as (1) stored 8 | ); 9 | 10 | insert into gen values(default); 11 | -------------------------------------------------------------------------------- /tests/unit/setup/16-sequences.sql: -------------------------------------------------------------------------------- 1 | --- 2 | --- See https://github.com/dimitri/pgcopydb/issues/777 3 | --- 4 | 5 | -- A sequence used as default 6 | create sequence default_table_id_seq; 7 | create table default_table (id integer primary key default nextval('default_table_id_seq')); 8 | select setval('default_table_id_seq', 667); 9 | 10 | -- A sequence used as identity 11 | create table identity_table (id integer primary key generated always as identity); 12 | select setval('identity_table_id_seq', 668); 13 | 14 | -- A standalone sequence 15 | create sequence standalone_id_seq; 16 | select setval('standalone_id_seq', 669); 17 | 18 | -- A standalone sequence smallint 19 | create sequence standalone_smallint_id_seq as smallint; 20 | select setval('standalone_smallint_id_seq', 670); 21 | 22 | -- A standalone sequence with a minvalue that has not been set 23 | create sequence standalone_minvalue_id_seq minvalue 671; 24 | -------------------------------------------------------------------------------- /tests/unit/setup/2-inherit.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS public.parent_model CASCADE; 2 | 3 | CREATE TABLE IF NOT EXISTS public.parent_model 4 | ( 5 | id integer NOT NULL, 6 | a integer NOT NULL, 7 | b boolean NOT NULL, 8 | CONSTRAINT pk_parent_model PRIMARY KEY (id) 9 | ); 10 | 11 | CREATE TABLE IF NOT EXISTS public.child_model_1 12 | ( 13 | -- Inherited from table public.parent_model: id integer NOT NULL, 14 | -- Inherited from table public.parent_model: a integer NOT NULL, 15 | -- Inherited from table public.parent_model: b boolean NOT NULL, 16 | c jsonb NOT NULL, 17 | CONSTRAINT pk_child_model_1 PRIMARY KEY (id) 18 | ) 19 | INHERITS (public.parent_model); 20 | 21 | CREATE TABLE IF NOT EXISTS public.child_model_2 22 | ( 23 | -- Inherited from table public.parent_model: id integer NOT NULL, 24 | -- Inherited from table public.parent_model: a integer NOT NULL, 25 | -- Inherited from table public.parent_model: b boolean NOT NULL, 26 | d integer NOT NULL, 27 | CONSTRAINT pk_child_model_2 PRIMARY KEY (id) 28 | ) 29 | INHERITS (public.parent_model); 30 | 31 | INSERT INTO child_model_1 (id, a, b, c) 32 | SELECT 33 | 10000 + s, 34 | 1, 35 | false, 36 | '[1, 2, 3, 4]' 37 | FROM 38 | generate_series(1, 1000) s; 39 | 40 | 41 | INSERT INTO child_model_2 (id, a, b, d) 42 | SELECT 43 | 20000 + s, 44 | 1, 45 | false, 46 | 1313 47 | FROM 48 | generate_series(1, 1000) s; 49 | -------------------------------------------------------------------------------- /tests/unit/setup/3-collations.sql: -------------------------------------------------------------------------------- 1 | create table colls 2 | ( 3 | id bigserial not null primary key, 4 | f1 text collate "fr-FR-x-icu", 5 | f2 text collate "de-DE-x-icu" 6 | ); 7 | 8 | create collation if not exists mycol 9 | ( 10 | locale = 'fr-FR-x-icu', 11 | provider = 'icu' 12 | ); 13 | 14 | create table mycoltab 15 | ( 16 | id bigserial not null primary key, 17 | f1 text collate mycol, 18 | f2 text collate "en-US-x-icu" 19 | ); 20 | -------------------------------------------------------------------------------- /tests/unit/setup/6-multiline-table-name.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE public."with MyTableName 2 | 3 | AS (SELECT row_id 4 | 5 | " ( 6 | row_id uuid NOT NULL 7 | ); 8 | 9 | -- 10 | -- See https://github.com/dimitri/pgcopydb/issues/430 11 | -- 12 | -- Migration fails when table has double quotes in them 13 | -- 14 | 15 | CREATE TABLE IF NOT EXISTS public."""dqname""" 16 | ( 17 | id integer 18 | ); 19 | 20 | 21 | -- 22 | -- See https://github.com/dimitri/pgcopydb/issues/483 23 | -- 24 | -- Migrations fails when the column name of a table is very long 25 | -- 26 | CREATE TABLE """long""" 27 | ( 28 | """aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa""" INT 29 | ); 30 | 31 | INSERT INTO """long""" VALUES (1); 32 | -------------------------------------------------------------------------------- /tests/unit/setup/7-identity.sql: -------------------------------------------------------------------------------- 1 | create table test_tbl 2 | ( 3 | id bigint not null generated always as identity primary key, 4 | f1 text 5 | ); 6 | -------------------------------------------------------------------------------- /tests/unit/setup/8-string-escape.sql: -------------------------------------------------------------------------------- 1 | create table test_str_escape 2 | ( 3 | id bigint not null generated always as identity primary key, 4 | f1 text 5 | ); 6 | 7 | insert into test_str_escape (f1) 8 | values (E'aaa\naaa'), 9 | (E'bbb\r\nbbb'), 10 | (E'ccc'); 11 | -------------------------------------------------------------------------------- /tests/unit/setup/9-table-with-zero-cols.sql: -------------------------------------------------------------------------------- 1 | create table nocols(); 2 | -------------------------------------------------------------------------------- /tests/unit/setup/setup.sql: -------------------------------------------------------------------------------- 1 | \ir 1-exclusion-constraints.sql 2 | \ir 2-inherit.sql 3 | \ir 3-collations.sql 4 | \ir 4-list-table-split.sql 5 | \ir 5-long-index-def.sql 6 | \ir 6-multiline-table-name.sql 7 | \ir 7-identity.sql 8 | \ir 8-string-escape.sql 9 | \ir 9-table-with-zero-cols.sql 10 | \ir 10-search-path-index-enum.sql 11 | \ir 11-check-constraint.sql 12 | \ir 12-generated-column.sql 13 | \ir 14-uuid.sql 14 | \ir 15-attgenerated.sql 15 | \ir 16-sequences.sql 16 | -------------------------------------------------------------------------------- /tests/unit/sql/1-exclusion-constraints.sql: -------------------------------------------------------------------------------- 1 | select n.nspname, i.relname, 2 | rn.nspname, r.relname, 3 | indisprimary, 4 | indisunique, 5 | (select string_agg(attname, ',') 6 | from pg_attribute 7 | where attrelid = r.oid 8 | and array[attnum::integer] <@ indkey::integer[] 9 | ) as cols, 10 | pg_get_indexdef(indexrelid), 11 | c.conname, 12 | pg_get_constraintdef(c.oid) 13 | from pg_index x 14 | join pg_class i ON i.oid = x.indexrelid 15 | join pg_class r ON r.oid = x.indrelid 16 | join pg_namespace n ON n.oid = i.relnamespace 17 | join pg_namespace rn ON rn.oid = r.relnamespace 18 | left join pg_depend d 19 | on d.classid = 'pg_class'::regclass 20 | and d.objid = i.oid 21 | and d.refclassid = 'pg_constraint'::regclass 22 | and d.deptype = 'i' 23 | left join pg_constraint c ON c.oid = d.refobjid 24 | where r.relkind = 'r' and r.relpersistence = 'p' 25 | and n.nspname !~ '^pg_' and n.nspname <> 'information_schema' 26 | and rn.nspname = 'public' and r.relname = 'exclcon' 27 | order by n.nspname, r.relname; 28 | -------------------------------------------------------------------------------- /tests/unit/sql/2-inherit.sql: -------------------------------------------------------------------------------- 1 | select * 2 | from public.child_model_1 3 | order by id desc 4 | limit 2; 5 | -------------------------------------------------------------------------------- /tests/unit/sql/3-string-escape.sql: -------------------------------------------------------------------------------- 1 | table test_str_escape; 2 | -------------------------------------------------------------------------------- /tests/unit/sql/4-parent-data.sql: -------------------------------------------------------------------------------- 1 | select count(*) from only public.parent_model; 2 | -------------------------------------------------------------------------------- /tests/unit/sql/5-attgenerated.sql: -------------------------------------------------------------------------------- 1 | table public.gen; 2 | -------------------------------------------------------------------------------- /tests/unit/sql/6-sequences.sql: -------------------------------------------------------------------------------- 1 | select last_value from default_table_id_seq; 2 | select last_value from identity_table_id_seq; 3 | select last_value from standalone_id_seq; 4 | select last_value from standalone_smallint_id_seq; 5 | select last_value from standalone_minvalue_id_seq; 6 | -------------------------------------------------------------------------------- /tests/uris.env: -------------------------------------------------------------------------------- 1 | PGSSLMODE="require" 2 | PGCOPYDB_SOURCE_PGURI="postgres://postgres:h4ckm3@source/postgres" 3 | PGCOPYDB_TARGET_PGURI="postgres://postgres:h4ckm3@target/postgres" 4 | --------------------------------------------------------------------------------