├── .github ├── dependabot.yml └── workflows │ ├── ci.yaml │ └── smoke.yaml ├── .gitignore ├── .prettierignore ├── .rspec ├── .rubocop.yml ├── .ruby-version ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── Rakefile ├── assets └── mascot.png ├── bin ├── pg_easy_replicate └── pg_easy_replicate_console ├── docker-compose.yml ├── lib ├── pg_easy_replicate.rb └── pg_easy_replicate │ ├── cli.rb │ ├── ddl_audit.rb │ ├── ddl_manager.rb │ ├── group.rb │ ├── helper.rb │ ├── index_manager.rb │ ├── orchestrate.rb │ ├── query.rb │ ├── stats.rb │ └── version.rb ├── package.json ├── pg_easy_replicate.gemspec ├── scripts ├── e2e-bootstrap.sh ├── e2e-start.sh └── release.sh ├── spec ├── database_helpers.rb ├── pg_easy_replicate │ ├── ddl_audit_spec.rb │ ├── ddl_manager_spec.rb │ ├── group_spec.rb │ ├── index_manager_spec.rb │ ├── orchestrate_spec.rb │ ├── query_spec.rb │ └── stats_spec.rb ├── pg_easy_replicate_spec.rb ├── smoke_spec.rb └── spec_helper.rb └── yarn.lock /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "bundler" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - "v**" 8 | 9 | pull_request: 10 | 11 | concurrency: 12 | group: branch-ci-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | rspec: 17 | runs-on: ubuntu-20.04 18 | timeout-minutes: 30 19 | name: Ruby ${{ matrix.ruby }} - PG ${{ matrix.pg.from }} -> PG ${{ matrix.pg.to }} 20 | strategy: 21 | matrix: 22 | ruby: ["3.0.5", "3.1.4", "3.2.1", "3.3.6"] 23 | pg: 24 | [ 25 | { from: 10, to: 11 }, 26 | { from: 11, to: 12 }, 27 | { from: 12, to: 13 }, 28 | { from: 13, to: 14 }, 29 | { from: 14, to: 15 }, 30 | { from: 10, to: 15 }, 31 | ] 32 | steps: 33 | - uses: actions/checkout@v1 34 | 35 | - name: Set up Ruby 36 | uses: ruby/setup-ruby@v1 37 | with: 38 | ruby-version: ${{ matrix.ruby }} 39 | bundler-cache: true 40 | 41 | - name: Bundle install 42 | env: 43 | RAILS_ENV: test 44 | run: | 45 | gem install bundler 46 | bundle install --jobs 4 --retry 3 --path vendor/bundle 47 | 48 | - name: Run Lint 49 | run: bundle exec rubocop 50 | # - name: Setup upterm session 51 | # uses: lhotari/action-upterm@v1 52 | - name: "Setup PG databases" 53 | run: | 54 | set -euvxo pipefail 55 | 56 | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' 57 | wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null 58 | 59 | sudo apt-get update 60 | 61 | sudo apt-get install --yes --no-install-recommends postgresql-${{ matrix.pg.from }} postgresql-client-${{ matrix.pg.from }} 62 | sudo apt-get install --yes --no-install-recommends postgresql-${{ matrix.pg.to }} postgresql-client-${{ matrix.pg.to }} 63 | 64 | sudo systemctl restart postgresql@${{ matrix.pg.from }}-main.service 65 | sudo systemctl restart postgresql@${{ matrix.pg.to }}-main.service 66 | sudo systemctl restart postgresql 67 | 68 | # Escape the quote because we are setting the password in PG 69 | # String: james-bond123@7!''3aaR 70 | export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR' 71 | 72 | sudo su - postgres -c "createuser -p 5432 -d -s -e -l james-bond" 73 | sudo -u postgres psql -p 5432 -c 'alter user "james-bond" with encrypted password '"'"''"$PGPASSWORD"''"'"';' 74 | sudo su - postgres -c "createdb -p 5432 postgres-db" 75 | sudo -u postgres psql -p 5432 -c "grant all privileges on database \"postgres-db\" to \"james-bond\";" 76 | 77 | sudo su - postgres -c "createuser -p 5433 -d -s -e -l james-bond" 78 | sudo -u postgres psql -p 5433 -c 'alter user "james-bond" with encrypted password '"'"''"$PGPASSWORD"''"'"';' 79 | sudo su - postgres -c "createdb -p 5433 postgres-db" 80 | sudo -u postgres psql -p 5433 -c "grant all privileges on database \"postgres-db\" to \"james-bond\";" 81 | 82 | # Remove the escaped quote since we are passing the pwd to psql 83 | # String: james-bond123@7!'3aaR 84 | export PGPASSWORD='james-bond123@7!'"'"'3aaR' 85 | psql -h localhost -d postgres-db -U james-bond -p 5432 -c 'ALTER SYSTEM SET wal_level = logical;' 86 | psql -h localhost -d postgres-db -U james-bond -p 5433 -c 'ALTER SYSTEM SET wal_level = logical;' 87 | 88 | sudo systemctl restart postgresql@${{ matrix.pg.from }}-main.service 89 | sudo systemctl restart postgresql@${{ matrix.pg.to }}-main.service 90 | sudo systemctl restart postgresql 91 | 92 | psql -h localhost -d postgres-db -U james-bond -p 5432 -c 'show wal_level;' 93 | psql -h localhost -d postgres-db -U james-bond -p 5433 -c 'show wal_level;' 94 | 95 | sudo -u postgres psql -p 5432 -c "ALTER SYSTEM SET max_connections = '500';" 96 | sudo -u postgres psql -p 5433 -c "ALTER SYSTEM SET max_connections = '500';" 97 | 98 | sudo systemctl restart postgresql@${{ matrix.pg.from }}-main.service 99 | sudo systemctl restart postgresql@${{ matrix.pg.to }}-main.service 100 | sudo systemctl restart postgresql 101 | 102 | # Verify the changes 103 | psql -h localhost -d postgres-db -U james-bond -p 5432 -c 'SHOW max_connections;' 104 | psql -h localhost -d postgres-db -U james-bond -p 5433 -c 'SHOW max_connections;' 105 | 106 | - name: Run RSpec 107 | run: bundle exec rspec 108 | build-push-image: 109 | if: startsWith(github.ref, 'refs/tags/v') 110 | runs-on: ubuntu-20.04 111 | timeout-minutes: 30 112 | needs: [rspec] 113 | steps: 114 | - name: Set up QEMU 115 | uses: docker/setup-qemu-action@v2 116 | - name: Set up Docker Buildx 117 | uses: docker/setup-buildx-action@v2 118 | - name: Login to Docker Hub 119 | uses: docker/login-action@v2 120 | with: 121 | username: ${{ secrets.DOCKERHUB_USERNAME }} 122 | password: ${{ secrets.DOCKERHUB_TOKEN }} 123 | - name: Branch name 124 | id: version_name 125 | run: | 126 | echo ::set-output name=no_v_tag::${GITHUB_REF_NAME:1} 127 | - name: Build and push 128 | uses: docker/build-push-action@v4 129 | with: 130 | platforms: linux/amd64,linux/arm64 131 | push: true 132 | build-args: VERSION=${{ steps.version_name.outputs.no_v_tag }} 133 | tags: shayonj/pg_easy_replicate:latest, shayonj/pg_easy_replicate:${{ steps.version_name.outputs.no_v_tag }} 134 | -------------------------------------------------------------------------------- /.github/workflows/smoke.yaml: -------------------------------------------------------------------------------- 1 | name: Smoke spec 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: 7 | - "v**" 8 | 9 | pull_request: 10 | 11 | concurrency: 12 | group: branch-smoke-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | smoke-spec: 17 | runs-on: ubuntu-latest 18 | timeout-minutes: 30 19 | name: Ruby ${{ matrix.ruby }} - PG ${{ matrix.pg.from }} -> PG ${{ matrix.pg.to }} 20 | strategy: 21 | matrix: 22 | ruby: ["3.0.5", "3.1.4", "3.2.1", "3.3.6"] 23 | pg: 24 | [ 25 | { from: 10, to: 11 }, 26 | { from: 11, to: 12 }, 27 | { from: 12, to: 13 }, 28 | { from: 13, to: 14 }, 29 | { from: 14, to: 15 }, 30 | { from: 10, to: 15 }, 31 | { from: 12, to: 15 }, 32 | ] 33 | steps: 34 | - uses: actions/checkout@v1 35 | 36 | - name: Set up Ruby 37 | uses: ruby/setup-ruby@v1 38 | with: 39 | ruby-version: ${{ matrix.ruby }} 40 | bundler-cache: true 41 | 42 | - name: Bundle install 43 | env: 44 | RAILS_ENV: test 45 | run: | 46 | gem install bundler 47 | bundle install --jobs 4 --retry 3 --path vendor/bundle 48 | 49 | - name: Run Lint 50 | run: bundle exec rubocop 51 | # - name: Setup upterm session 52 | # uses: lhotari/action-upterm@v1 53 | - name: "Setug PG databases" 54 | run: | 55 | set -euvxo pipefail 56 | 57 | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' 58 | wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null 59 | 60 | sudo apt-get update 61 | 62 | sudo apt-get install --yes --no-install-recommends postgresql-${{ matrix.pg.from }} postgresql-client-${{ matrix.pg.from }} 63 | sudo apt-get install --yes --no-install-recommends postgresql-${{ matrix.pg.to }} postgresql-client-${{ matrix.pg.to }} 64 | 65 | sudo systemctl restart postgresql@${{ matrix.pg.from }}-main.service 66 | sudo systemctl restart postgresql@${{ matrix.pg.to }}-main.service 67 | sudo systemctl restart postgresql 68 | 69 | # Escape the quote because we are setting the password in PG 70 | # String: james-bond123@7!''3aaR 71 | export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR' 72 | 73 | sudo su - postgres -c "createuser -p 5432 -d -s -e -l james-bond" 74 | sudo -u postgres psql -p 5432 -c 'alter user "james-bond" with encrypted password '"'"''"$PGPASSWORD"''"'"';' 75 | sudo su - postgres -c "createdb -p 5432 postgres-db" 76 | sudo -u postgres psql -p 5432 -c "grant all privileges on database \"postgres-db\" to \"james-bond\";" 77 | 78 | sudo su - postgres -c "createuser -p 5433 -d -s -e -l james-bond" 79 | sudo -u postgres psql -p 5433 -c 'alter user "james-bond" with encrypted password '"'"''"$PGPASSWORD"''"'"';' 80 | sudo su - postgres -c "createdb -p 5433 postgres-db" 81 | sudo -u postgres psql -p 5433 -c "grant all privileges on database \"postgres-db\" to \"james-bond\";" 82 | 83 | # Remove the escaped quote since we are passing the pwd to psql 84 | # String: james-bond123@7!'3aaR 85 | export PGPASSWORD='james-bond123@7!'"'"'3aaR' 86 | psql -h localhost -d postgres-db -U james-bond -p 5432 -c 'ALTER SYSTEM SET wal_level = logical;' 87 | psql -h localhost -d postgres-db -U james-bond -p 5433 -c 'ALTER SYSTEM SET wal_level = logical;' 88 | 89 | sudo systemctl restart postgresql@${{ matrix.pg.from }}-main.service 90 | sudo systemctl restart postgresql@${{ matrix.pg.to }}-main.service 91 | sudo systemctl restart postgresql 92 | 93 | psql -h localhost -d postgres-db -U james-bond -p 5432 -c 'show wal_level;' 94 | psql -h localhost -d postgres-db -U james-bond -p 5433 -c 'show wal_level;' 95 | 96 | - name: Run RSpec 97 | run: bundle exec rspec spec/smoke_spec.rb 98 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | # rspec failure tracking 11 | .rspec_status 12 | 13 | # .gem 14 | .gem 15 | 16 | node_modules 17 | .DS_Store 18 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /spec/fixtures 2 | /docs 3 | /scripts 4 | *.sql 5 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | --fail-fast 5 | --exclude-pattern spec/smoke_spec.rb 6 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | require: 2 | - rubocop-rspec 3 | - rubocop-rake 4 | - rubocop-performance 5 | 6 | inherit_mode: 7 | merge: 8 | - Include 9 | - Exclude 10 | - AllowedMethods 11 | 12 | AllCops: 13 | NewCops: enable 14 | Exclude: 15 | - "**/.git/**/*" 16 | - "**/node_modules/**/*" 17 | - "**/Brewfile" 18 | TargetRubyVersion: 3.0 19 | 20 | Bundler/OrderedGems: 21 | Include: 22 | - "**/Gemfile" 23 | 24 | Style/FrozenStringLiteralComment: 25 | EnforcedStyle: always 26 | 27 | Style/MutableConstant: 28 | EnforcedStyle: literals 29 | 30 | Style/MethodCallWithArgsParentheses: 31 | Enabled: true 32 | EnforcedStyle: require_parentheses 33 | AllowedMethods: 34 | - yield 35 | - raise 36 | - fail 37 | - puts 38 | - require 39 | - require_relative 40 | - render 41 | - redirect_to 42 | - head 43 | - throw 44 | # RSpec 45 | - to 46 | - not_to 47 | - to_not 48 | - and 49 | - or 50 | Exclude: 51 | - "**/Gemfile" 52 | - "**/db/migrate/*" 53 | - "**/db/schema.rb" 54 | 55 | Style/RedundantInitialize: 56 | Enabled: false 57 | 58 | Layout: 59 | Enabled: false 60 | 61 | Metrics: 62 | Enabled: false 63 | 64 | Naming/AccessorMethodName: 65 | Enabled: false 66 | 67 | Naming/MethodParameterName: 68 | Enabled: false 69 | 70 | Naming/PredicateName: 71 | Enabled: false 72 | 73 | Naming/VariableNumber: 74 | Enabled: false 75 | 76 | Style/AsciiComments: 77 | Enabled: false 78 | 79 | Style/BlockDelimiters: 80 | Enabled: false 81 | 82 | Style/CaseLikeIf: 83 | Enabled: false 84 | 85 | Style/ClassAndModuleChildren: 86 | Enabled: false 87 | 88 | Style/CommentAnnotation: 89 | Enabled: false 90 | 91 | Style/Documentation: 92 | Enabled: false 93 | 94 | Style/IfUnlessModifier: 95 | Enabled: false 96 | 97 | Style/Lambda: 98 | Enabled: false 99 | 100 | Style/ModuleFunction: 101 | Enabled: false 102 | 103 | Style/MultilineBlockChain: 104 | Enabled: false 105 | 106 | Style/NumericLiterals: 107 | Enabled: false 108 | 109 | Style/NumericPredicate: 110 | Enabled: false 111 | 112 | Style/ParallelAssignment: 113 | Enabled: false 114 | 115 | Style/PerlBackrefs: 116 | Enabled: false 117 | 118 | Style/QuotedSymbols: 119 | EnforcedStyle: double_quotes 120 | Enabled: false 121 | 122 | Style/RaiseArgs: 123 | Enabled: false 124 | 125 | Style/RescueStandardError: 126 | Enabled: false 127 | 128 | Style/SingleArgumentDig: 129 | Enabled: false 130 | 131 | Style/StringLiterals: 132 | EnforcedStyle: double_quotes 133 | Enabled: false 134 | 135 | Style/StringLiteralsInInterpolation: 136 | Enabled: false 137 | 138 | Style/SymbolArray: 139 | Enabled: false 140 | 141 | Style/TrailingCommaInArguments: 142 | Enabled: false 143 | 144 | Style/TrailingCommaInArrayLiteral: 145 | Enabled: false 146 | EnforcedStyleForMultiline: consistent_comma 147 | 148 | Style/TrailingCommaInHashLiteral: 149 | Enabled: false 150 | 151 | Style/TrailingUnderscoreVariable: 152 | Enabled: false 153 | 154 | Style/ZeroLengthPredicate: 155 | Enabled: false 156 | 157 | Style/DateTime: 158 | Enabled: true 159 | 160 | RSpec/ExpectChange: 161 | EnforcedStyle: block 162 | 163 | Gemspec/RequireMFA: 164 | # Our Gemspec files are internal, MFA isn't needed 165 | Enabled: false 166 | 167 | # Temporary Rubocop exclusions 168 | Style/OpenStructUse: 169 | Enabled: false 170 | 171 | # Ruby 3 migration exclusions 172 | Style/HashSyntax: 173 | Enabled: false 174 | 175 | Naming/BlockForwarding: 176 | Enabled: false 177 | 178 | Lint/RedundantDirGlobSort: 179 | Enabled: false 180 | 181 | RSpec/ExampleLength: 182 | Enabled: false 183 | 184 | RSpec/MultipleExpectations: 185 | Enabled: false 186 | 187 | RSpec/VerifiedDoubles: 188 | Enabled: false 189 | 190 | RSpec/ContextWording: 191 | Enabled: false 192 | 193 | RSpec/AnyInstance: 194 | Enabled: false 195 | 196 | RSpec/MessageSpies: 197 | Enabled: false 198 | 199 | RSpec/RepeatedDescription: 200 | Enabled: false 201 | 202 | RSpec/RepeatedExample: 203 | Enabled: false 204 | 205 | RSpec/HookArgument: 206 | Enabled: false 207 | 208 | RSpec/DescribeClass: 209 | Enabled: false 210 | 211 | RSpec/DescribedClass: 212 | Enabled: false 213 | 214 | RSpec/FilePath: 215 | Enabled: false 216 | 217 | RSpec/IdenticalEqualityAssertion: 218 | Enabled: false 219 | 220 | RSpec/InstanceVariable: 221 | Enabled: false 222 | 223 | RSpec/MissingExampleGroupArgument: 224 | Enabled: false 225 | 226 | RSpec/MultipleDescribes: 227 | Enabled: false 228 | 229 | RSpec/NestedGroups: 230 | Enabled: false 231 | 232 | RSpec/PredicateMatcher: 233 | Enabled: false 234 | 235 | RSpec/Rails/HttpStatus: 236 | Enabled: false 237 | 238 | RSpec/RepeatedExampleGroupDescription: 239 | Enabled: false 240 | 241 | RSpec/StubbedMock: 242 | Enabled: false 243 | 244 | Lint/UnusedMethodArgument: 245 | Enabled: false 246 | 247 | Lint/MissingSuper: 248 | Enabled: false 249 | 250 | RSpec/NoExpectationExample: 251 | Enabled: false 252 | 253 | Style/AccessorGrouping: 254 | Enabled: false 255 | 256 | Style/FormatStringToken: 257 | Enabled: false 258 | -------------------------------------------------------------------------------- /.ruby-version: -------------------------------------------------------------------------------- 1 | 3.3.6 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. 8 | 9 | ## Our Standards 10 | 11 | Examples of behavior that contributes to a positive environment for our community include: 12 | 13 | - Demonstrating empathy and kindness toward other people 14 | - Being respectful of differing opinions, viewpoints, and experiences 15 | - Giving and gracefully accepting constructive feedback 16 | - Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience 17 | - Focusing on what is best not just for us as individuals, but for the overall community 18 | 19 | Examples of unacceptable behavior include: 20 | 21 | - The use of sexualized language or imagery, and sexual attention or 22 | advances of any kind 23 | - Trolling, insulting or derogatory comments, and personal or political attacks 24 | - Public or private harassment 25 | - Publishing others' private information, such as a physical or email 26 | address, without their explicit permission 27 | - Other conduct which could reasonably be considered inappropriate in a 28 | professional setting 29 | 30 | ## Enforcement Responsibilities 31 | 32 | Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. 33 | 34 | Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. 35 | 36 | ## Scope 37 | 38 | This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. 39 | 40 | ## Enforcement 41 | 42 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at shayonj@gmail.com. All complaints will be reviewed and investigated promptly and fairly. 43 | 44 | All community leaders are obligated to respect the privacy and security of the reporter of any incident. 45 | 46 | ## Enforcement Guidelines 47 | 48 | Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: 49 | 50 | ### 1. Correction 51 | 52 | **Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. 53 | 54 | **Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. 55 | 56 | ### 2. Warning 57 | 58 | **Community Impact**: A violation through a single incident or series of actions. 59 | 60 | **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. 61 | 62 | ### 3. Temporary Ban 63 | 64 | **Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. 65 | 66 | **Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. 67 | 68 | ### 4. Permanent Ban 69 | 70 | **Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. 71 | 72 | **Consequence**: A permanent ban from any sort of public interaction within the community. 73 | 74 | ## Attribution 75 | 76 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, 77 | available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 78 | 79 | Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). 80 | 81 | [homepage]: https://www.contributor-covenant.org 82 | 83 | For answers to common questions about this code of conduct, see the FAQ at 84 | https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. 85 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:3.3.6-slim 2 | 3 | ARG VERSION 4 | 5 | RUN apt-get update && \ 6 | apt-get install -y --no-install-recommends \ 7 | wget \ 8 | gnupg2 \ 9 | lsb-release \ 10 | build-essential \ 11 | libpq-dev \ 12 | && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg && \ 13 | echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ 14 | apt-get update && \ 15 | apt-get install -y --no-install-recommends postgresql-client && \ 16 | gem install pg_easy_replicate -v $VERSION && \ 17 | apt-get remove -y wget gnupg2 lsb-release && \ 18 | apt-get autoremove -y && \ 19 | apt-get clean && \ 20 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 21 | 22 | RUN pg_dump --version 23 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | # Specify your gem's dependencies in pg_easy_replicate.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | pg_easy_replicate (0.3.8) 5 | ougai (~> 2.0.0) 6 | pg (~> 1.5.3) 7 | pg_query (~> 5.1.0) 8 | sequel (>= 5.69, < 5.87) 9 | thor (>= 1.2.2, < 1.4.0) 10 | 11 | GEM 12 | remote: https://rubygems.org/ 13 | specs: 14 | ast (2.4.2) 15 | bigdecimal (3.1.8) 16 | coderay (1.1.3) 17 | diff-lcs (1.5.1) 18 | google-protobuf (4.28.2-arm64-darwin) 19 | bigdecimal 20 | rake (>= 13) 21 | google-protobuf (4.28.2-x86_64-linux) 22 | bigdecimal 23 | rake (>= 13) 24 | haml (6.1.1) 25 | temple (>= 0.8.2) 26 | thor 27 | tilt 28 | json (2.8.2) 29 | language_server-protocol (3.17.0.3) 30 | method_source (1.1.0) 31 | oj (3.14.3) 32 | ougai (2.0.0) 33 | oj (~> 3.10) 34 | parallel (1.26.3) 35 | parser (3.3.6.0) 36 | ast (~> 2.4.1) 37 | racc 38 | pg (1.5.9) 39 | pg_query (5.1.0) 40 | google-protobuf (>= 3.22.3) 41 | prettier_print (1.2.1) 42 | pry (0.15.2) 43 | coderay (~> 1.1) 44 | method_source (~> 1.0) 45 | racc (1.8.1) 46 | rainbow (3.1.1) 47 | rake (13.2.1) 48 | rbs (3.1.0) 49 | regexp_parser (2.9.2) 50 | rexml (3.3.9) 51 | rspec (3.13.0) 52 | rspec-core (~> 3.13.0) 53 | rspec-expectations (~> 3.13.0) 54 | rspec-mocks (~> 3.13.0) 55 | rspec-core (3.13.0) 56 | rspec-support (~> 3.13.0) 57 | rspec-expectations (3.13.0) 58 | diff-lcs (>= 1.2.0, < 2.0) 59 | rspec-support (~> 3.13.0) 60 | rspec-mocks (3.13.0) 61 | diff-lcs (>= 1.2.0, < 2.0) 62 | rspec-support (~> 3.13.0) 63 | rspec-support (3.13.0) 64 | rubocop (1.64.1) 65 | json (~> 2.3) 66 | language_server-protocol (>= 3.17.0) 67 | parallel (~> 1.10) 68 | parser (>= 3.3.0.2) 69 | rainbow (>= 2.2.2, < 4.0) 70 | regexp_parser (>= 1.8, < 3.0) 71 | rexml (>= 3.2.5, < 4.0) 72 | rubocop-ast (>= 1.31.1, < 2.0) 73 | ruby-progressbar (~> 1.7) 74 | unicode-display_width (>= 2.4.0, < 3.0) 75 | rubocop-ast (1.36.1) 76 | parser (>= 3.3.1.0) 77 | rubocop-capybara (2.20.0) 78 | rubocop (~> 1.41) 79 | rubocop-factory_bot (2.25.1) 80 | rubocop (~> 1.41) 81 | rubocop-packaging (0.5.2) 82 | rubocop (>= 1.33, < 2.0) 83 | rubocop-performance (1.23.0) 84 | rubocop (>= 1.48.1, < 2.0) 85 | rubocop-ast (>= 1.31.1, < 2.0) 86 | rubocop-rake (0.6.0) 87 | rubocop (~> 1.0) 88 | rubocop-rspec (2.29.1) 89 | rubocop (~> 1.40) 90 | rubocop-capybara (~> 2.17) 91 | rubocop-factory_bot (~> 2.22) 92 | rubocop-rspec_rails (~> 2.28) 93 | rubocop-rspec_rails (2.28.2) 94 | rubocop (~> 1.40) 95 | ruby-progressbar (1.13.0) 96 | sequel (5.86.0) 97 | bigdecimal 98 | syntax_tree (6.2.0) 99 | prettier_print (>= 1.2.0) 100 | syntax_tree-haml (4.0.3) 101 | haml (>= 5.2) 102 | prettier_print (>= 1.2.1) 103 | syntax_tree (>= 6.0.0) 104 | syntax_tree-rbs (1.0.0) 105 | prettier_print 106 | rbs 107 | syntax_tree (>= 2.0.1) 108 | temple (0.10.1) 109 | thor (1.3.2) 110 | tilt (2.1.0) 111 | unicode-display_width (2.6.0) 112 | 113 | PLATFORMS 114 | arm64-darwin-22 115 | x86_64-linux 116 | 117 | DEPENDENCIES 118 | pg_easy_replicate! 119 | prettier_print 120 | pry 121 | rake 122 | rspec 123 | rubocop 124 | rubocop-packaging 125 | rubocop-performance 126 | rubocop-rake 127 | rubocop-rspec 128 | syntax_tree 129 | syntax_tree-haml 130 | syntax_tree-rbs 131 | 132 | BUNDLED WITH 133 | 2.4.12 134 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 Shayon Mukherjee 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pg_easy_replicate 2 | 3 | [![CI](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml) 4 | [![Smoke spec](https://github.com/shayonj/pg_easy_replicate/actions/workflows/smoke.yaml/badge.svg?branch=main)](https://github.com/shayonj/pg_easy_replicate/actions/workflows/ci.yaml) 5 | [![Gem Version](https://badge.fury.io/rb/pg_easy_replicate.svg?2)](https://badge.fury.io/rb/pg_easy_replicate) 6 | 7 | `pg_easy_replicate` is a CLI orchestrator tool that simplifies the process of setting up [logical replication](https://www.postgresql.org/docs/current/logical-replication.html) between two PostgreSQL databases. `pg_easy_replicate` also supports switchover. After the source (primary database) is fully replicated, `pg_easy_replicate` puts it into read-only mode and via logical replication flushes all data to the new target database. This ensures zero data loss and minimal downtime for the application. This method can be useful for performing minimal downtime (up to <1min, depending) major version upgrades between a Blue/Green PostgreSQL database setup, load testing and other similar use cases. 8 | 9 | Battle tested in production at [Tines](https://www.tines.com/) 🚀 10 | 11 | ![](./assets/mascot.png) 12 | 13 | - [Installation](#installation) 14 | - [Requirements](#requirements) 15 | - [Limits](#limits) 16 | - [Usage](#usage) 17 | - [CLI](#cli) 18 | - [Replicating all tables with a single group](#replicating-all-tables-with-a-single-group) 19 | - [Config check](#config-check) 20 | - [Bootstrap](#bootstrap) 21 | - [Bootstrap and Config Check with special user role in AWS or GCP](#bootstrap-and-config-check-with-special-user-role-in-aws-or-gcp) 22 | - [Config Check](#config-check-1) 23 | - [Bootstrap](#bootstrap-1) 24 | - [Start sync](#start-sync) 25 | - [DDL Changes Management](#ddl-changes-management) 26 | - [Listing DDL Changes](#listing-ddl-changes) 27 | - [Applying DDL Changes](#applying-ddl-changes) 28 | - [Stats](#stats) 29 | - [Notify](#notify) 30 | - [Performing switchover](#performing-switchover) 31 | - [Replicating single database with custom tables](#replicating-single-database-with-custom-tables) 32 | - [Exclude tables from replication](#exclude-tables-from-replication) 33 | - [Cleanup](#cleanup) 34 | - [Switchover strategies with minimal downtime](#switchover-strategies-with-minimal-downtime) 35 | - [Rolling restart strategy](#rolling-restart-strategy) 36 | - [DNS Failover strategy](#dns-failover-strategy) 37 | - [FAQ](#faq) 38 | - [Adding internal user to `pg_hba` or pgBouncer `userlist`](#adding-internal-user-to-pg_hba-or-pgbouncer-userlist) 39 | - [Contributing](#contributing) 40 | 41 | ## Installation 42 | 43 | Add this line to your application's Gemfile: 44 | 45 | ```ruby 46 | gem "pg_easy_replicate" 47 | ``` 48 | 49 | And then execute: 50 | 51 | $ bundle install 52 | 53 | Or install it yourself as: 54 | 55 | $ gem install pg_easy_replicate 56 | 57 | This will include all dependencies accordingly as well. Make sure the following requirements are satisfied. 58 | 59 | Or via Docker: 60 | 61 | docker pull shayonj/pg_easy_replicate:latest 62 | 63 | https://hub.docker.com/r/shayonj/pg_easy_replicate 64 | 65 | ## Requirements 66 | 67 | - PostgreSQL 10 and later 68 | - Ruby 3.0 and later 69 | - Database users should have `SUPERUSER` permissions, or pass in a special user with privileges to create the needed role, schema, publication and subscription on both databases. More on `--special-user-role` section below. 70 | - See more on [FAQ](#faq) below 71 | 72 | ## Limits 73 | 74 | All [Logical Replication Restrictions](https://www.postgresql.org/docs/current/logical-replication-restrictions.html) apply. 75 | 76 | ## Usage 77 | 78 | Ensure `SOURCE_DB_URL` and `TARGET_DB_URL` are present as environment variables in the runtime environment. 79 | 80 | - `SOURCE_DB_URL` = The database that you want to replicate FROM. 81 | - `TARGET_DB_URL` = The database that you want to replicate TO. 82 | 83 | The URL should be in postgres connection string format. Example: 84 | 85 | ```bash 86 | $ export SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME" 87 | $ export TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME" 88 | ``` 89 | 90 | **Optional** 91 | 92 | You can extend the default timeout by setting the following environment variable 93 | 94 | ```bash 95 | $ export PG_EASY_REPLICATE_STATEMENT_TIMEOUT="10s" # default 5s 96 | ``` 97 | 98 | You can get additional debug logging by adding the following environment variable 99 | 100 | ```bash 101 | $ export DEBUG="true" 102 | ``` 103 | 104 | Any `pg_easy_replicate` command can be run the same way with the docker image as well. As long the container is running in an environment where it has access to both the databases. Example 105 | 106 | ```bash 107 | docker run -e SOURCE_DB_URL="postgres://USERNAME:PASSWORD@localhost:5432/DATABASE_NAME" \ 108 | -e TARGET_DB_URL="postgres://USERNAME:PASSWORD@localhost:5433/DATABASE_NAME" \ 109 | -it --rm shayonj/pg_easy_replicate:latest \ 110 | pg_easy_replicate config_check 111 | ``` 112 | 113 | ## CLI 114 | 115 | ```bash 116 | $ pg_easy_replicate 117 | pg_easy_replicate commands: 118 | pg_easy_replicate bootstrap -g, --group-name=GROUP_NAME # Sets up temporary tables for information required during runtime 119 | pg_easy_replicate cleanup -g, --group-name=GROUP_NAME # Cleans up all bootstrapped data for the respective group 120 | pg_easy_replicate config_check # Prints if source and target database have the required config 121 | pg_easy_replicate help [COMMAND] # Describe available commands or one specific command 122 | pg_easy_replicate start_sync -g, --group-name=GROUP_NAME # Starts the logical replication from source database to target database provisioned in the group 123 | pg_easy_replicate stats -g, --group-name=GROUP_NAME # Prints the statistics in JSON for the group 124 | pg_easy_replicate notify -g, --group-name=GROUP_NAME, -u --url=URL_TO_NOTIFY # Sends notifications of all stats values for a group to a specified url 125 | pg_easy_replicate stop_sync -g, --group-name=GROUP_NAME # Stop the logical replication from source database to target database provisioned in the group 126 | pg_easy_replicate switchover -g, --group-name=GROUP_NAME # Puts the source database in read only mode after all the data is flushed and written 127 | pg_easy_replicate version # Prints the version 128 | 129 | ``` 130 | 131 | ## Replicating all tables with a single group 132 | 133 | You can create as many groups as you want for a single database. Groups are just a logical isolation of a single replication. 134 | 135 | ### Config check 136 | 137 | ```bash 138 | $ pg_easy_replicate config_check 139 | 140 | ✅ Config is looking good. 141 | ``` 142 | 143 | ### Bootstrap 144 | 145 | Every sync will need to be bootstrapped before you can set up the sync between two databases. Bootstrap creates a new super user to perform the orchestration required during the rest of the process. It also creates some internal metadata tables for record keeping. 146 | 147 | ```bash 148 | $ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema 149 | 150 | {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":21485,"level":30,"time":"2023-06-19T15:51:11.015-04:00","v":0,"msg":"Setting up schema","version":"0.1.0"} 151 | ... 152 | ``` 153 | 154 | ### Bootstrap and Config Check with special user role in AWS or GCP 155 | 156 | If you don't want your primary login user to have `superuser` privileges or you are on AWS or GCP, you will need to pass in the special user role that has the privileges to create role, schema, publication and subscription. This is required so `pg_easy_replicate` can create a dedicated user for replication which is granted the respective special user role to carry out its functionalities. 157 | 158 | For AWS the special user role is `rds_superuser`, and for GCP it is `cloudsqlsuperuser`. Please refer to docs for the most up to date information. 159 | 160 | **Note**: The user in the connection url must be part of the special user role being supplied. 161 | 162 | #### Config Check 163 | 164 | ```bash 165 | $ pg_easy_replicate config_check --special-user-role="rds_superuser" --copy-schema 166 | 167 | ✅ Config is looking good. 168 | ``` 169 | 170 | #### Bootstrap 171 | 172 | ```bash 173 | $ pg_easy_replicate bootstrap --group-name database-cluster-1 --special-user-role="rds_superuser" --copy-schema 174 | 175 | {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":21485,"level":30,"time":"2023-06-19T15:51:11.015-04:00","v":0,"msg":"Setting up schema","version":"0.1.0"} 176 | ... 177 | ``` 178 | 179 | ### Start sync 180 | 181 | Once the bootstrap is complete, you can start the sync. Starting the sync sets up the publication, subscription and performs other minor housekeeping things. 182 | 183 | **NOTE**: Start sync by default will drop all indices in the target database for performance reasons. And will automatically re-add the indices during `switchover`. It is turned on by default and you can opt out of this with `--no-recreate-indices-post-copy` 184 | 185 | ```bash 186 | $ pg_easy_replicate start_sync --group-name database-cluster-1 [-d ] 187 | 188 | {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":22113,"level":30,"time":"2023-06-19T15:54:54.874-04:00","v":0,"msg":"Setting up publication","publication_name":"pger_publication_database_cluster_1","version":"0.1.0"} 189 | ... 190 | ``` 191 | 192 | ### DDL Changes Management 193 | 194 | `pg_easy_replicate` now supports tracking and applying DDL (Data Definition Language) changes between the source and target databases. To track DDLs you can pass `--track-ddl` to `start_sync`. 195 | 196 | This feature ensures that most schema changes made to the source database tables that are being replicated during the replication process are tracked, so that you can apply them at your will before or after switchover. 197 | 198 | #### Listing DDL Changes 199 | 200 | To view the DDL changes that have been tracked: 201 | 202 | ```bash 203 | $ pg_easy_replicate list_ddl_changes -g [-l ] 204 | ``` 205 | 206 | This command will display a list of DDL changes in JSON format; 207 | 208 | ``` 209 | [ 210 | { 211 | "id": 1, 212 | "group_name": "cluster-1", 213 | "event_type": "ddl_command_end", 214 | "object_type": "table", 215 | "object_identity": "public.pgbench_accounts", 216 | "ddl_command": "ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)", 217 | "created_at": "2024-08-31 15:42:33 UTC" 218 | } 219 | ] 220 | ``` 221 | 222 | #### Applying DDL Changes 223 | 224 | `pg_easy_replicate` won't automatically apply the changes for you. To apply the tracked DDL changes to the target database: 225 | 226 | ```bash 227 | $ pg_easy_replicate apply_ddl_change -g [-i ] 228 | ``` 229 | 230 | If you specify a change ID with the `-i` option, only that specific change will be applied. If you don't specify an ID, you'll be prompted to apply all pending changes. 231 | 232 | ```bash 233 | $ pg_easy_replicate apply_ddl_change -g cluster-1 234 | The following DDL changes will be applied: 235 | ID: 1, Type: table, Command: ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)... 236 | 237 | Do you want to apply all these changes? (y/n): y 238 | ... 239 | All pending DDL changes applied successfully. 240 | ``` 241 | 242 | ### Stats 243 | 244 | You can inspect or watch stats any time during the sync process. The stats give you an idea of when the sync started, current flush/write lag, how many tables are in `replicating`, `copying` or other stages, and more. 245 | 246 | You can poll these stats to perform any other after the switchover is done. The stats include a `switchover_completed_at` which is updated once the switch over is complete. 247 | 248 | ```bash 249 | $ pg_easy_replicate stats --group-name database-cluster-1 250 | 251 | { 252 | "lag_stats": [ 253 | { 254 | "pid": 66, 255 | "client_addr": "192.168.128.2", 256 | "user_name": "jamesbond", 257 | "application_name": "pger_subscription_database_cluster_1", 258 | "state": "streaming", 259 | "sync_state": "async", 260 | "write_lag": "0.0", 261 | "flush_lag": "0.0", 262 | "replay_lag": "0.0" 263 | } 264 | ], 265 | "message_lsn_receipts": [ 266 | { 267 | "received_lsn": "0/1674688", 268 | "last_msg_send_time": "2023-06-19 19:56:35 UTC", 269 | "last_msg_receipt_time": "2023-06-19 19:56:35 UTC", 270 | "latest_end_lsn": "0/1674688", 271 | "latest_end_time": "2023-06-19 19:56:35 UTC" 272 | } 273 | ], 274 | "sync_started_at": "2023-06-19 19:54:54 UTC", 275 | "sync_failed_at": null, 276 | "switchover_completed_at": null 277 | 278 | .... 279 | ``` 280 | 281 | ### Notify 282 | 283 | You can send stats to an endpoint on an interval using notify. This can be configured to receieve the stats to this url on a frequency (default 10s). A timeout can also be configured for the request to the endpoint (default 10s). This gives you greater control over processing different events in the replication cycle in your workflow. 284 | 285 | ```bash 286 | $ pg_easy_replicate notify --group-name database-cluster-1 --url https://example.com/webhook --frequency 10 --timeout 10 287 | ``` 288 | 289 | ### Performing switchover 290 | 291 | `pg_easy_replicate` doesn't kick off the switchover on its own. When you start the sync via `start_sync`, it starts the replication between the two databases. Once you have had the time to monitor stats and any other key metrics, you can kick off the `switchover`. 292 | 293 | `switchover` will wait until all tables in the group are replicating and the delta for lag is <200kb (by calculating the `pg_wal_lsn_diff` between `sent_lsn` and `write_lsn`) and then perform the switch. 294 | 295 | Additionally, `switchover` will take care of re-adding the indices (it had removed in `start_sync`) in the target database before hand. Depending on the size of the tables, the recreation of indexes (which happens `CONCURRENTLY`) may take a while. See `start_sync` for more details. 296 | 297 | The switch is made by putting the user on the source database in `READ ONLY` mode, so that it is not accepting any more writes and waits for the flush lag to be `0`. It’s up to the user to kick off a rolling restart of their application containers or failover DNS (more on these below in strategies) after the switchover is complete, so that your application isn't sending any read + write requests to the old/source database. 298 | 299 | ```bash 300 | $ pg_easy_replicate switchover --group-name database-cluster-1 301 | 302 | {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Watching lag stats","version":"0.1.0"} 303 | ... 304 | ``` 305 | 306 | ## Replicating single database with custom tables 307 | 308 | By default all tables are added for replication but you can create multiple groups with custom tables for the same database. Example 309 | 310 | ```bash 311 | 312 | $ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema 313 | $ pg_easy_replicate start_sync --group-name database-cluster-1 --schema-name public --tables "users,posts,events" 314 | 315 | ... 316 | 317 | $ pg_easy_replicate bootstrap --group-name database-cluster-2 --copy-schema 318 | $ pg_easy_replicate start_sync --group-name database-cluster-2 --schema-name public --tables "comments,views" 319 | 320 | ... 321 | $ pg_easy_replicate switchover --group-name database-cluster-1 322 | $ pg_easy_replicate switchover --group-name database-cluster-2 323 | ... 324 | ``` 325 | 326 | ## Exclude tables from replication 327 | 328 | By default all tables are added for replication but you can exclude tables if necessary. Example 329 | 330 | ```bash 331 | ... 332 | $ pg_easy_replicate bootstrap --group-name database-cluster-1 --copy-schema 333 | $ pg_easy_replicate start_sync --group-name database-cluster-1 --schema-name public --exclude_tables "events" 334 | ... 335 | ``` 336 | 337 | ### Cleanup 338 | 339 | Use `cleanup` if you want to remove all bootstrapped data for the specified group. Additionally you can pass `-e` or `--everything` in order to clean up all schema changes for bootstrapped tables, users and any publication/subscription data. 340 | 341 | ```bash 342 | $ pg_easy_replicate cleanup --group-name database-cluster-1 --everything 343 | 344 | {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping groups table","version":"0.1.0"} 345 | 346 | {"name":"pg_easy_replicate","hostname":"PKHXQVK6DW","pid":24192,"level":30,"time":"2023-06-19T16:05:23.033-04:00","v":0,"msg":"Dropping schema","version":"0.1.0"} 347 | ... 348 | ``` 349 | 350 | ## Switchover strategies with minimal downtime 351 | 352 | For minimal downtime, it'd be best to watch/tail the stats and wait until `switchover_completed_at` is updated with a timestamp. Once that happens you can perform any of the following strategies. Note: These are just suggestions and `pg_easy_replicate` doesn't provide any functionalities for this. 353 | 354 | ### Rolling restart strategy 355 | 356 | In this strategy, you have a change ready to go which instructs your application to start connecting to the new database. Either using an environment variable or similar. Depending on the application type, it may or may not require a rolling restart. 357 | 358 | Next, you can set up a program that watches the `stats` and waits until `switchover_completed_at` is reporting as `true`. Once that happens it kicks off a rolling restart of your application containers so they can start making connections to the DNS of the new database. 359 | 360 | ### DNS Failover strategy 361 | 362 | In this strategy, you have a weighted based DNS system (example [AWS Route53 weighted records](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resource-record-sets-values-weighted.html)) where 100% of traffic goes to a primary origin and 0% to a secondary origin. The primary origin here is the DNS host for your source database and secondary origin is the DNS host for your target database. You can set up your application ahead of time to interact with the database using DNS from the weighted group. 363 | 364 | Next, you can set up a program that watches the `stats` and waits until `switchover_completed_at` is reporting as `true`. Once that happens it updates the weight in the DNS weighted group where 100% of the requests now go to the new/target database. Note: Keeping a low `ttl` is recommended. 365 | 366 | ## FAQ 367 | 368 | ### Adding internal user to `pg_hba` or pgBouncer `userlist` 369 | 370 | `pg_easy_replicate` sets up a designated user for managing the replication process. In case you handle user permissions through `pg_hba`, it's necessary to modify this list to permit sessions from `pger_su_h1a4fb`. Similarly, with pgBouncer, you'll need to authorize `pger_su_h1a4fb` for login access by including it in the `userlist`. 371 | 372 | ## Contributing 373 | 374 | PRs most welcome. You can get started locally by 375 | 376 | - `docker compose down -v && docker compose up --remove-orphans --build` 377 | - Install ruby `3.3.6` using RVM ([instruction](https://rvm.io/rvm/install#any-other-system)) 378 | - `bundle exec rspec` for specs 379 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | require "rspec/core/rake_task" 5 | require "standalone_migrations" 6 | 7 | RSpec::Core::RakeTask.new(:spec) 8 | 9 | require "rubocop/rake_task" 10 | 11 | RuboCop::RakeTask.new 12 | 13 | task default: [:spec, :rubocop] 14 | -------------------------------------------------------------------------------- /assets/mascot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/shayonj/pg_easy_replicate/25d1759ea241bbdf7cb14694a003c11bfaf52ed0/assets/mascot.png -------------------------------------------------------------------------------- /bin/pg_easy_replicate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "pg_easy_replicate" 5 | 6 | PgEasyReplicate::CLI.start(ARGV) 7 | -------------------------------------------------------------------------------- /bin/pg_easy_replicate_console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "bundler/setup" 5 | require "pg_easy_replicate" 6 | require "pry" 7 | 8 | Pry.start 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | source_db: 4 | image: postgres:14 5 | ports: 6 | - "5432:5432" 7 | environment: 8 | POSTGRES_USER: james-bond 9 | POSTGRES_PASSWORD: james-bond123@7!'3aaR 10 | POSTGRES_DB: postgres-db 11 | command: > 12 | -c max_connections=200 13 | -c wal_level=logical 14 | -c ssl=on 15 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 16 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 17 | networks: 18 | localnet: 19 | 20 | target_db: 21 | image: postgres:12 22 | ports: 23 | - "5433:5432" 24 | environment: 25 | POSTGRES_USER: james-bond 26 | POSTGRES_PASSWORD: james-bond123@7!'3aaR 27 | POSTGRES_DB: postgres-db 28 | command: > 29 | -c max_connections=200 30 | -c wal_level=logical 31 | -c ssl=on 32 | -c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem 33 | -c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key 34 | networks: 35 | localnet: 36 | 37 | networks: 38 | localnet: 39 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "json" 4 | require "ougai" 5 | require "pg" 6 | require "sequel" 7 | require "open3" 8 | require "English" 9 | require "pg_query" 10 | 11 | require "pg_easy_replicate/helper" 12 | require "pg_easy_replicate/version" 13 | require "pg_easy_replicate/query" 14 | require "pg_easy_replicate/index_manager" 15 | require "pg_easy_replicate/orchestrate" 16 | require "pg_easy_replicate/stats" 17 | require "pg_easy_replicate/group" 18 | require "pg_easy_replicate/cli" 19 | require "pg_easy_replicate/ddl_audit" 20 | require "pg_easy_replicate/ddl_manager" 21 | 22 | Sequel.default_timezone = :utc 23 | module PgEasyReplicate 24 | SCHEMA_FILE_LOCATION = "/tmp/pger_schema.sql" 25 | 26 | class Error < StandardError 27 | end 28 | 29 | extend Helper 30 | 31 | class << self 32 | def config( 33 | special_user_role: nil, 34 | copy_schema: false, 35 | tables: "", 36 | exclude_tables: "", 37 | schema_name: nil 38 | ) 39 | abort_with("SOURCE_DB_URL is missing") if source_db_url.nil? 40 | abort_with("TARGET_DB_URL is missing") if target_db_url.nil? 41 | 42 | if !tables.empty? && !exclude_tables.empty? 43 | abort_with( 44 | "Options --tables(-t) and --exclude-tables(-e) cannot be used together.", 45 | ) 46 | end 47 | 48 | system("which pg_dump") 49 | pg_dump_exists = $CHILD_STATUS.success? 50 | 51 | @config ||= 52 | begin 53 | q = 54 | "select name, setting from pg_settings where name in ('max_wal_senders', 'max_worker_processes', 'wal_level', 'max_replication_slots', 'max_logical_replication_workers');" 55 | 56 | { 57 | source_db_is_super_user: 58 | is_super_user?(source_db_url, special_user_role), 59 | target_db_is_super_user: 60 | is_super_user?(target_db_url, special_user_role), 61 | source_db: 62 | Query.run( 63 | query: q, 64 | connection_url: source_db_url, 65 | user: db_user(source_db_url), 66 | ), 67 | target_db: 68 | Query.run( 69 | query: q, 70 | connection_url: target_db_url, 71 | user: db_user(target_db_url), 72 | ), 73 | pg_dump_exists: pg_dump_exists, 74 | tables_have_replica_identity: 75 | tables_have_replica_identity?( 76 | conn_string: source_db_url, 77 | tables: tables, 78 | exclude_tables: exclude_tables, 79 | schema_name: schema_name, 80 | ), 81 | } 82 | rescue => e 83 | abort_with("Unable to check config: #{e.message}") 84 | end 85 | end 86 | 87 | def assert_config( 88 | special_user_role: nil, 89 | copy_schema: false, 90 | tables: "", 91 | exclude_tables: "", 92 | schema_name: nil 93 | ) 94 | config_hash = 95 | config( 96 | special_user_role: special_user_role, 97 | copy_schema: copy_schema, 98 | tables: tables, 99 | exclude_tables: exclude_tables, 100 | schema_name: schema_name, 101 | ) 102 | 103 | if copy_schema && !config_hash.dig(:pg_dump_exists) 104 | abort_with("pg_dump must exist if copy_schema (-c) is passed") 105 | end 106 | 107 | unless assert_wal_level_logical(config_hash.dig(:source_db)) 108 | abort_with("WAL_LEVEL should be LOGICAL on source DB") 109 | end 110 | 111 | unless assert_wal_level_logical(config_hash.dig(:target_db)) 112 | abort_with("WAL_LEVEL should be LOGICAL on target DB") 113 | end 114 | 115 | unless config_hash.dig(:source_db_is_super_user) 116 | abort_with("User on source database does not have super user privilege") 117 | end 118 | 119 | validate_table_lists(tables, exclude_tables, schema_name) 120 | 121 | unless config_hash.dig(:tables_have_replica_identity) 122 | abort_with( 123 | "Ensure all tables involved in logical replication have an appropriate replica identity set. This can be done using: 124 | 1. Default (Primary Key): `ALTER TABLE table_name REPLICA IDENTITY DEFAULT;` 125 | 2. Unique Index: `ALTER TABLE table_name REPLICA IDENTITY USING INDEX index_name;` 126 | 3. Full (All Columns): `ALTER TABLE table_name REPLICA IDENTITY FULL;`", 127 | ) 128 | end 129 | 130 | return if config_hash.dig(:target_db_is_super_user) 131 | abort_with("User on target database does not have super user privilege") 132 | end 133 | 134 | def bootstrap(options) 135 | logger.info("Setting up schema") 136 | setup_internal_schema 137 | 138 | if options[:copy_schema] 139 | logger.info("Setting up schema on target database") 140 | copy_schema( 141 | source_conn_string: source_db_url, 142 | target_conn_string: target_db_url, 143 | ) 144 | end 145 | 146 | logger.info("Setting up replication user on source database") 147 | create_user( 148 | conn_string: source_db_url, 149 | special_user_role: options[:special_user_role], 150 | grant_permissions_on_schema: true, 151 | ) 152 | 153 | logger.info("Setting up replication user on target database") 154 | create_user( 155 | conn_string: target_db_url, 156 | special_user_role: options[:special_user_role], 157 | ) 158 | 159 | logger.info("Setting up groups tables") 160 | Group.setup 161 | logger.info("Bootstrap completed successfully") 162 | rescue => e 163 | abort_with("Unable to bootstrap: #{e.message}") 164 | end 165 | 166 | def cleanup(options) 167 | cleanup_steps = [ 168 | -> do 169 | logger.info("Dropping groups table") 170 | Group.drop 171 | end, 172 | -> do 173 | if options[:restore_connection_on_source_db] 174 | restore_connections_on_source_db 175 | end 176 | end, 177 | -> do 178 | if options[:everything] 179 | logger.info("Dropping schema") 180 | drop_internal_schema 181 | end 182 | end, 183 | -> do 184 | if options[:everything] || options[:sync] 185 | logger.info("Dropping publication on source database") 186 | Orchestrate.drop_publication( 187 | group_name: options[:group_name], 188 | conn_string: source_db_url, 189 | ) 190 | end 191 | end, 192 | -> do 193 | if options[:everything] || options[:sync] 194 | logger.info("Dropping subscription on target database") 195 | Orchestrate.drop_subscription( 196 | group_name: options[:group_name], 197 | target_conn_string: target_db_url, 198 | ) 199 | end 200 | end, 201 | -> do 202 | if options[:everything] 203 | logger.info("Dropping replication user on source database") 204 | drop_user(conn_string: source_db_url) 205 | end 206 | end, 207 | -> do 208 | if options[:everything] 209 | logger.info("Dropping replication user on target database") 210 | drop_user(conn_string: target_db_url) 211 | end 212 | -> do 213 | if options[:everything] 214 | PgEasyReplicate::DDLManager.cleanup_ddl_tracking( 215 | conn_string: source_db_url, 216 | group_name: options[:group_name], 217 | ) 218 | end 219 | end 220 | end, 221 | ] 222 | 223 | cleanup_steps.each do |step| 224 | step.call 225 | rescue => e 226 | logger.warn( 227 | "Part of the cleanup step failed with #{e.message}. Continuing...", 228 | ) 229 | end 230 | 231 | logger.info("Cleanup process completed.") 232 | rescue => e 233 | abort_with("Unable to cleanup: #{e.message}") 234 | end 235 | 236 | def drop_internal_schema 237 | Query.run( 238 | query: 239 | "DROP SCHEMA IF EXISTS #{quote_ident(internal_schema_name)} CASCADE", 240 | connection_url: source_db_url, 241 | schema: internal_schema_name, 242 | user: db_user(source_db_url), 243 | ) 244 | rescue => e 245 | raise "Unable to drop schema: #{e.message}" 246 | end 247 | 248 | def setup_internal_schema 249 | sql = <<~SQL 250 | create schema if not exists #{quote_ident(internal_schema_name)}; 251 | grant usage on schema #{quote_ident(internal_schema_name)} to #{quote_ident(db_user(source_db_url))}; 252 | grant create on schema #{quote_ident(internal_schema_name)} to #{quote_ident(db_user(source_db_url))}; 253 | SQL 254 | 255 | Query.run( 256 | query: sql, 257 | connection_url: source_db_url, 258 | schema: internal_schema_name, 259 | user: db_user(source_db_url), 260 | ) 261 | rescue => e 262 | raise "Unable to setup schema: #{e.message}" 263 | end 264 | 265 | def logger 266 | @logger ||= 267 | begin 268 | logger = Ougai::Logger.new($stdout) 269 | logger.level = 270 | ENV["DEBUG"] ? Ougai::Logger::TRACE : Ougai::Logger::INFO 271 | logger.with_fields = { version: PgEasyReplicate::VERSION } 272 | logger 273 | end 274 | end 275 | 276 | def copy_schema(source_conn_string:, target_conn_string:) 277 | export_schema(conn_string: source_conn_string) 278 | import_schema(conn_string: target_conn_string) 279 | end 280 | 281 | def export_schema(conn_string:) 282 | logger.info("Exporting schema to #{SCHEMA_FILE_LOCATION}") 283 | _, stderr, status = 284 | Open3.capture3( 285 | "pg_dump", 286 | conn_string, 287 | "-f", 288 | SCHEMA_FILE_LOCATION, 289 | "--schema-only", 290 | ) 291 | 292 | success = status.success? 293 | raise stderr unless success 294 | rescue => e 295 | raise "Unable to export schema: #{e.message}" 296 | end 297 | 298 | def import_schema(conn_string:) 299 | logger.info("Importing schema from #{SCHEMA_FILE_LOCATION}") 300 | 301 | _, stderr, status = 302 | Open3.capture3("psql", "-f", SCHEMA_FILE_LOCATION, conn_string) 303 | 304 | success = status.success? 305 | raise stderr unless success 306 | rescue => e 307 | raise "Unable to import schema: #{e.message}" 308 | end 309 | 310 | def assert_wal_level_logical(db_config) 311 | db_config&.find do |r| 312 | r.dig(:name) == "wal_level" && r.dig(:setting) == "logical" 313 | end 314 | end 315 | 316 | def is_super_user?(url, special_user_role = nil) 317 | if special_user_role 318 | sql = <<~SQL 319 | SELECT r.rolname AS username, 320 | r1.rolname AS "role" 321 | FROM pg_catalog.pg_roles r 322 | LEFT JOIN pg_catalog.pg_auth_members m ON (m.member = r.oid) 323 | LEFT JOIN pg_roles r1 ON (m.roleid=r1.oid) 324 | WHERE r.rolname = '#{db_user(url)}' 325 | ORDER BY 1; 326 | SQL 327 | 328 | r = Query.run(query: sql, connection_url: url, user: db_user(url)) 329 | # If special_user_role is passed just ensure the url in conn_string has been granted 330 | # the special_user_role 331 | r.any? { |q| q[:role] == special_user_role } 332 | else 333 | r = 334 | Query.run( 335 | query: 336 | "SELECT rolname, rolsuper FROM pg_roles where rolname = '#{db_user(url)}';", 337 | connection_url: url, 338 | user: db_user(url), 339 | ) 340 | r.any? { |q| q[:rolsuper] } 341 | end 342 | rescue => e 343 | raise "Unable to check superuser conditions: #{e.message}" 344 | end 345 | 346 | def create_user( 347 | conn_string:, 348 | special_user_role: nil, 349 | grant_permissions_on_schema: false 350 | ) 351 | return if user_exists?(conn_string: conn_string, user: internal_user_name) 352 | 353 | password = connection_info(conn_string)[:password].gsub("'") { "''" } 354 | 355 | sql = <<~SQL 356 | create role #{quote_ident(internal_user_name)} with password '#{password}' login createdb createrole; 357 | grant all privileges on database #{quote_ident(db_name(conn_string))} TO #{quote_ident(internal_user_name)}; 358 | SQL 359 | 360 | Query.run( 361 | query: sql, 362 | connection_url: conn_string, 363 | user: db_user(conn_string), 364 | transaction: false, 365 | ) 366 | 367 | sql = 368 | if special_user_role 369 | "grant #{quote_ident(special_user_role)} to #{quote_ident(internal_user_name)};" 370 | else 371 | "alter user #{quote_ident(internal_user_name)} with superuser;" 372 | end 373 | 374 | Query.run( 375 | query: sql, 376 | connection_url: conn_string, 377 | user: db_user(conn_string), 378 | transaction: false, 379 | ) 380 | 381 | return unless grant_permissions_on_schema 382 | Query.run( 383 | query: 384 | "grant all on schema #{quote_ident(internal_schema_name)} to #{quote_ident(internal_user_name)}", 385 | connection_url: conn_string, 386 | user: db_user(conn_string), 387 | transaction: false, 388 | ) 389 | rescue => e 390 | raise "Unable to create user: #{e.message}" 391 | end 392 | 393 | def drop_user(conn_string:, user: internal_user_name) 394 | return unless user_exists?(conn_string: conn_string, user: user) 395 | 396 | sql = <<~SQL 397 | revoke all privileges on database #{quote_ident(db_name(conn_string))} from #{quote_ident(user)}; 398 | SQL 399 | 400 | Query.run( 401 | query: sql, 402 | connection_url: conn_string, 403 | user: db_user(conn_string), 404 | ) 405 | 406 | sql = <<~SQL 407 | drop role if exists #{quote_ident(user)}; 408 | SQL 409 | 410 | Query.run( 411 | query: sql, 412 | connection_url: conn_string, 413 | user: db_user(conn_string), 414 | ) 415 | rescue => e 416 | raise "Unable to drop user: #{e.message}" 417 | end 418 | 419 | def user_exists?(conn_string:, user: internal_user_name) 420 | sql = <<~SQL 421 | SELECT r.rolname AS username, 422 | r1.rolname AS "role" 423 | FROM pg_catalog.pg_roles r 424 | LEFT JOIN pg_catalog.pg_auth_members m ON (m.member = r.oid) 425 | LEFT JOIN pg_roles r1 ON (m.roleid=r1.oid) 426 | WHERE r.rolname = '#{user}' 427 | ORDER BY 1; 428 | SQL 429 | 430 | Query 431 | .run( 432 | query: sql, 433 | connection_url: conn_string, 434 | user: db_user(conn_string), 435 | ) 436 | .any? { |q| q[:username] == user } 437 | end 438 | 439 | def tables_have_replica_identity?( 440 | conn_string:, 441 | tables: "", 442 | exclude_tables: "", 443 | schema_name: nil 444 | ) 445 | schema_name ||= "public" 446 | 447 | table_list = 448 | determine_tables( 449 | schema: schema_name, 450 | conn_string: source_db_url, 451 | list: tables, 452 | exclude_list: exclude_tables, 453 | ) 454 | return false if table_list.empty? 455 | 456 | formatted_table_list = table_list.map { |table| "'#{table}'" }.join(", ") 457 | 458 | sql = <<~SQL 459 | SELECT t.relname AS table_name, 460 | CASE 461 | WHEN t.relreplident = 'd' THEN 'default' 462 | WHEN t.relreplident = 'n' THEN 'nothing' 463 | WHEN t.relreplident = 'i' THEN 'index' 464 | WHEN t.relreplident = 'f' THEN 'full' 465 | END AS replica_identity 466 | FROM pg_class t 467 | JOIN pg_namespace ns ON t.relnamespace = ns.oid 468 | WHERE ns.nspname = '#{schema_name}' 469 | AND t.relkind = 'r' 470 | AND t.relname IN (#{formatted_table_list}) 471 | SQL 472 | 473 | results = 474 | Query.run( 475 | query: sql, 476 | connection_url: conn_string, 477 | user: db_user(conn_string), 478 | ) 479 | 480 | results.all? { |r| r[:replica_identity] != "nothing" } 481 | end 482 | end 483 | end 484 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/cli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "thor" 4 | 5 | module PgEasyReplicate 6 | class CLI < Thor 7 | package_name "pg_easy_replicate" 8 | 9 | desc "config_check", 10 | "Prints if source and target database have the required config" 11 | method_option :special_user_role, 12 | aliases: "-s", 13 | desc: 14 | "Name of the role that has superuser permissions. Usually useful for AWS (rds_superuser) or GCP (cloudsqlsuperuser)." 15 | method_option :copy_schema, 16 | aliases: "-c", 17 | boolean: true, 18 | desc: "Copy schema to the new database" 19 | method_option :tables, 20 | aliases: "-t", 21 | default: "", 22 | desc: 23 | "Comma separated list of table names. Default: All tables" 24 | method_option :exclude_tables, 25 | aliases: "-e", 26 | default: "", 27 | desc: 28 | "Comma separated list of table names to exclude. Default: None" 29 | method_option :schema_name, 30 | aliases: "-s", 31 | desc: 32 | "Name of the schema tables are in, only required if passing list of tables" 33 | def config_check 34 | PgEasyReplicate.assert_config( 35 | special_user_role: options[:special_user_role], 36 | copy_schema: options[:copy_schema], 37 | tables: options[:tables], 38 | exclude_tables: options[:exclude_tables], 39 | schema_name: options[:schema_name], 40 | ) 41 | 42 | puts "✅ Config is looking good." 43 | end 44 | 45 | method_option :group_name, 46 | aliases: "-g", 47 | required: true, 48 | desc: "Name of the group to provision" 49 | method_option :special_user_role, 50 | aliases: "-s", 51 | desc: 52 | "Name of the role that has superuser permissions. Usually useful with AWS (rds_superuser) or GCP (cloudsqlsuperuser)." 53 | method_option :copy_schema, 54 | aliases: "-c", 55 | boolean: true, 56 | desc: "Copy schema to the new database" 57 | method_option :track_ddl, 58 | aliases: "-d", 59 | type: :boolean, 60 | default: false, 61 | desc: "Enable DDL tracking for the group" 62 | desc "bootstrap", 63 | "Sets up temporary tables for information required during runtime" 64 | def bootstrap 65 | PgEasyReplicate.bootstrap(options) 66 | end 67 | 68 | desc "cleanup", "Cleans up all bootstrapped data for the respective group" 69 | method_option :group_name, 70 | aliases: "-g", 71 | required: true, 72 | desc: "Name of the group previously provisioned" 73 | method_option :everything, 74 | aliases: "-e", 75 | desc: 76 | "Cleans up all bootstrap tables, users and any publication/subscription" 77 | method_option :sync, 78 | aliases: "-s", 79 | desc: 80 | "Cleans up the publication and subscription for the respective group" 81 | method_option :restore_connection_on_source_db, 82 | aliases: "-r", 83 | type: :boolean, 84 | default: false, 85 | desc: "Restore connection on source db after switchover" 86 | def cleanup 87 | PgEasyReplicate.cleanup(options) 88 | end 89 | 90 | desc "start_sync", 91 | "Starts the logical replication from source database to target database provisioned in the group" 92 | method_option :group_name, 93 | aliases: "-g", 94 | required: true, 95 | desc: "Name of the group to provision" 96 | method_option :schema_name, 97 | aliases: "-s", 98 | desc: 99 | "Name of the schema tables are in, only required if passing list of tables" 100 | method_option :tables, 101 | aliases: "-t", 102 | default: "", 103 | desc: 104 | "Comma separated list of table names. Default: All tables" 105 | method_option :exclude_tables, 106 | aliases: "-e", 107 | default: "", 108 | desc: 109 | "Comma separated list of table names to exclude. Default: None" 110 | method_option :recreate_indices_post_copy, 111 | type: :boolean, 112 | default: false, 113 | aliases: "-r", 114 | desc: 115 | "Drop all non-primary indices before copy and recreate them post-copy" 116 | method_option :track_ddl, 117 | type: :boolean, 118 | default: false, 119 | desc: "Enable DDL tracking for the group" 120 | def start_sync 121 | PgEasyReplicate::Orchestrate.start_sync(options) 122 | end 123 | 124 | desc "stop_sync", 125 | "Stop the logical replication from source database to target database provisioned in the group" 126 | method_option :group_name, 127 | aliases: "-g", 128 | required: true, 129 | desc: "Name of the group previously provisioned" 130 | def stop_sync 131 | PgEasyReplicate::Orchestrate.stop_sync(group_name: options[:group_name]) 132 | end 133 | 134 | desc "switchover", 135 | "Puts the source database in read only mode after all the data is flushed and written" 136 | method_option :group_name, 137 | aliases: "-g", 138 | required: true, 139 | desc: "Name of the group previously provisioned" 140 | method_option :lag_delta_size, 141 | aliases: "-l", 142 | desc: 143 | "The size of the lag to watch for before switchover. Default 200KB." 144 | method_option :skip_vacuum_analyze, 145 | type: :boolean, 146 | default: false, 147 | aliases: "-s", 148 | desc: "Skip vacuum analyzing tables before switchover." 149 | def switchover 150 | PgEasyReplicate::Orchestrate.switchover( 151 | group_name: options[:group_name], 152 | lag_delta_size: options[:lag_delta_size], 153 | skip_vacuum_analyze: options[:skip_vacuum_analyze], 154 | ) 155 | end 156 | 157 | desc "stats ", "Prints the statistics in JSON for the group" 158 | method_option :group_name, 159 | aliases: "-g", 160 | required: true, 161 | desc: "Name of the group previously provisioned" 162 | method_option :watch, aliases: "-w", desc: "Tail the stats" 163 | def stats 164 | if options[:watch] 165 | PgEasyReplicate::Stats.follow(options[:group_name]) 166 | else 167 | PgEasyReplicate::Stats.print(options[:group_name]) 168 | end 169 | end 170 | 171 | desc "notify", 172 | "Sends a notification with replication status to a specified url" 173 | method_option :group_name, 174 | aliases: "-g", 175 | required: true, 176 | desc: "Name of the group previously provisioned" 177 | method_option :url, 178 | aliases: "-u", 179 | required: true, 180 | desc: "URL for notification" 181 | method_option :frequency, 182 | aliases: "-f", 183 | type: :numeric, 184 | default: 10, 185 | desc: "Frequency for sending stats to the endpoint provided" 186 | method_option :timeout, 187 | aliases: "-t", 188 | type: :numeric, 189 | default: 10, 190 | desc: "Timeout for the notify request" 191 | 192 | def notify 193 | PgEasyReplicate::Stats.notify( 194 | options[:group_name], 195 | options[:url], 196 | options[:frequency], 197 | options[:timeout], 198 | ) 199 | end 200 | 201 | desc "list_ddl_changes", "Lists recent DDL changes in the source database" 202 | method_option :group_name, 203 | aliases: "-g", 204 | required: true, 205 | desc: "Name of the group" 206 | method_option :limit, 207 | aliases: "-l", 208 | type: :numeric, 209 | default: 100, 210 | desc: "Limit the number of DDL changes to display" 211 | def list_ddl_changes 212 | changes = 213 | PgEasyReplicate::DDLManager.list_ddl_changes( 214 | group_name: options[:group_name], 215 | limit: options[:limit], 216 | ) 217 | puts JSON.pretty_generate(changes) 218 | end 219 | 220 | desc "apply_ddl_change", "Applies DDL changes to the target database" 221 | method_option :group_name, 222 | aliases: "-g", 223 | required: true, 224 | desc: "Name of the group" 225 | method_option :id, 226 | aliases: "-i", 227 | type: :numeric, 228 | desc: 229 | "ID of the specific DDL change to apply. If not provided, all changes will be applied." 230 | def apply_ddl_change 231 | if options[:id] 232 | PgEasyReplicate::DDLManager.apply_ddl_change( 233 | group_name: options[:group_name], 234 | id: options[:id], 235 | ) 236 | puts "DDL change with ID #{options[:id]} applied successfully." 237 | else 238 | changes = 239 | PgEasyReplicate::DDLManager.list_ddl_changes( 240 | group_name: options[:group_name], 241 | ) 242 | if changes.empty? 243 | puts "No pending DDL changes to apply." 244 | return 245 | end 246 | 247 | puts "The following DDL changes will be applied:" 248 | changes.each do |change| 249 | puts "ID: #{change[:id]}, Type: #{change[:object_type]}, Command: #{change[:ddl_command]}" 250 | end 251 | puts "" 252 | print("Do you want to apply all these changes? (y/n): ") 253 | confirmation = $stdin.gets.chomp.downcase 254 | 255 | if confirmation == "y" 256 | PgEasyReplicate::DDLManager.apply_all_ddl_changes( 257 | group_name: options[:group_name], 258 | ) 259 | puts "All pending DDL changes applied successfully." 260 | else 261 | puts "Operation cancelled." 262 | end 263 | end 264 | end 265 | 266 | desc "version", "Prints the version" 267 | def version 268 | puts PgEasyReplicate::VERSION 269 | end 270 | 271 | def self.exit_on_failure? 272 | true 273 | end 274 | end 275 | end 276 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/ddl_audit.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "pg_query" 4 | 5 | module PgEasyReplicate 6 | class DDLAudit 7 | extend Helper 8 | 9 | class << self 10 | def setup(group_name) 11 | conn = connect_to_internal_schema 12 | return if conn.table_exists?(table_name) 13 | 14 | begin 15 | conn.create_table(table_name) do 16 | primary_key(:id) 17 | String(:group_name, null: false) 18 | String(:event_type, null: false) 19 | String(:object_type) 20 | String(:object_identity) 21 | String(:ddl_command, text: true) 22 | DateTime(:created_at, default: Sequel::CURRENT_TIMESTAMP) 23 | end 24 | 25 | create_trigger_function(conn, group_name) 26 | create_event_triggers(conn, group_name) 27 | rescue => e 28 | abort_with("Failed to set up DDL audit: #{e.message}") 29 | ensure 30 | conn&.disconnect 31 | end 32 | end 33 | 34 | def create( 35 | group_name, 36 | event_type, 37 | object_type, 38 | object_identity, 39 | ddl_command 40 | ) 41 | conn = connect_to_internal_schema 42 | begin 43 | conn[table_name].insert( 44 | group_name: group_name, 45 | event_type: event_type, 46 | object_type: object_type, 47 | object_identity: object_identity, 48 | ddl_command: ddl_command, 49 | created_at: Time.now.utc, 50 | ) 51 | rescue => e 52 | abort_with("Adding DDL audit entry failed: #{e.message}") 53 | ensure 54 | conn&.disconnect 55 | end 56 | end 57 | 58 | def list_changes(group_name, limit: 100) 59 | conn = connect_to_internal_schema 60 | begin 61 | conn[table_name] 62 | .where(group_name: group_name) 63 | .order(Sequel.desc(:id)) 64 | .limit(limit) 65 | .all 66 | rescue => e 67 | abort_with("Listing DDL changes failed: #{e.message}") 68 | ensure 69 | conn&.disconnect 70 | end 71 | end 72 | 73 | def apply_change(source_conn_string, target_conn_string, group_name, id) 74 | ddl_queries = fetch_ddl_query(source_conn_string, group_name, id: id) 75 | apply_ddl_changes(target_conn_string, ddl_queries) 76 | end 77 | 78 | def apply_all_changes(source_conn_string, target_conn_string, group_name) 79 | ddl_queries = fetch_ddl_query(source_conn_string, group_name) 80 | apply_ddl_changes(target_conn_string, ddl_queries) 81 | end 82 | 83 | def drop(group_name) 84 | conn = connect_to_internal_schema 85 | begin 86 | drop_event_triggers(conn, group_name) 87 | drop_trigger_function(conn, group_name) 88 | conn[table_name].where(group_name: group_name).delete 89 | rescue => e 90 | abort_with("Dropping DDL audit failed: #{e.message}") 91 | ensure 92 | conn&.disconnect 93 | end 94 | end 95 | 96 | private 97 | 98 | def table_name 99 | :pger_ddl_audits 100 | end 101 | 102 | def connect_to_internal_schema(conn_string = nil) 103 | Query.connect( 104 | connection_url: conn_string || source_db_url, 105 | schema: internal_schema_name, 106 | ) 107 | end 108 | 109 | def create_trigger_function(conn, group_name) 110 | group = PgEasyReplicate::Group.find(group_name) 111 | tables = group[:table_names].split(",").map(&:strip) 112 | schema_name = group[:schema_name] 113 | sanitized_group_name = sanitize_identifier(group_name) 114 | 115 | full_table_names = tables.map { |table| "#{schema_name}.#{table}" } 116 | table_pattern = full_table_names.join("|") 117 | 118 | conn.run(<<~SQL) 119 | CREATE OR REPLACE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}() RETURNS event_trigger AS $$ 120 | DECLARE 121 | obj record; 122 | ddl_command text; 123 | affected_table text; 124 | BEGIN 125 | SELECT current_query() INTO ddl_command; 126 | 127 | IF TG_EVENT = 'ddl_command_end' THEN 128 | FOR obj IN SELECT * FROM pg_event_trigger_ddl_commands() 129 | LOOP 130 | IF obj.object_identity ~ '^(#{table_pattern})' THEN 131 | INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command) 132 | VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command); 133 | ELSIF obj.object_type = 'index' THEN 134 | SELECT (regexp_match(ddl_command, 'ON\\s+(\\S+)'))[1] INTO affected_table; 135 | IF affected_table IN ('#{full_table_names.join("','")}') THEN 136 | INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command) 137 | VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command); 138 | END IF; 139 | END IF; 140 | END LOOP; 141 | ELSIF TG_EVENT = 'sql_drop' THEN 142 | FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() 143 | LOOP 144 | IF (obj.object_identity = ANY(ARRAY['#{full_table_names.join("','")}']) OR 145 | obj.object_identity ~ ('^' || '#{schema_name}' || '\\.(.*?)_.*$')) 146 | THEN 147 | INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command) 148 | VALUES ('#{group_name}', TG_EVENT, obj.object_type, obj.object_identity, ddl_command); 149 | END IF; 150 | END LOOP; 151 | ELSIF TG_EVENT = 'table_rewrite' THEN 152 | FOR obj IN SELECT * FROM pg_event_trigger_table_rewrite_oid() 153 | LOOP 154 | SELECT c.relname, n.nspname INTO affected_table 155 | FROM pg_class c 156 | JOIN pg_namespace n ON n.oid = c.relnamespace 157 | WHERE c.oid = obj.oid; 158 | 159 | IF affected_table IN ('#{full_table_names.join("','")}') THEN 160 | INSERT INTO #{internal_schema_name}.#{table_name} (group_name, event_type, object_type, object_identity, ddl_command) 161 | VALUES ('#{group_name}', TG_EVENT, 'table', affected_table, 'table_rewrite'); 162 | END IF; 163 | END LOOP; 164 | END IF; 165 | END; 166 | $$ LANGUAGE plpgsql; 167 | SQL 168 | rescue => e 169 | abort_with("Creating DDL trigger function failed: #{e.message}") 170 | end 171 | 172 | def create_event_triggers(conn, group_name) 173 | sanitized_group_name = sanitize_identifier(group_name) 174 | conn.run(<<~SQL) 175 | DROP EVENT TRIGGER IF EXISTS pger_ddl_trigger_#{sanitized_group_name}; 176 | CREATE EVENT TRIGGER pger_ddl_trigger_#{sanitized_group_name} ON ddl_command_end 177 | EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}(); 178 | 179 | DROP EVENT TRIGGER IF EXISTS pger_drop_trigger_#{sanitized_group_name}; 180 | CREATE EVENT TRIGGER pger_drop_trigger_#{sanitized_group_name} ON sql_drop 181 | EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}(); 182 | 183 | DROP EVENT TRIGGER IF EXISTS pger_table_rewrite_trigger_#{sanitized_group_name}; 184 | CREATE EVENT TRIGGER pger_table_rewrite_trigger_#{sanitized_group_name} ON table_rewrite 185 | EXECUTE FUNCTION #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}(); 186 | SQL 187 | rescue => e 188 | abort_with("Creating event triggers failed: #{e.message}") 189 | end 190 | 191 | def drop_event_triggers(conn, group_name) 192 | sanitized_group_name = sanitize_identifier(group_name) 193 | conn.run(<<~SQL) 194 | DROP EVENT TRIGGER IF EXISTS pger_ddl_trigger_#{sanitized_group_name}; 195 | DROP EVENT TRIGGER IF EXISTS pger_drop_trigger_#{sanitized_group_name}; 196 | DROP EVENT TRIGGER IF EXISTS pger_table_rewrite_trigger_#{sanitized_group_name}; 197 | SQL 198 | rescue => e 199 | abort_with("Dropping event triggers failed: #{e.message}") 200 | end 201 | 202 | def drop_trigger_function(conn, group_name) 203 | sanitized_group_name = sanitize_identifier(group_name) 204 | conn.run( 205 | "DROP FUNCTION IF EXISTS #{internal_schema_name}.pger_ddl_trigger_#{sanitized_group_name}();", 206 | ) 207 | rescue => e 208 | abort_with("Dropping trigger function failed: #{e.message}") 209 | end 210 | 211 | def self.extract_table_info(sql) 212 | parsed = PgQuery.parse(sql) 213 | stmt = parsed.tree.stmts.first.stmt 214 | 215 | case stmt 216 | when PgQuery::CreateStmt, PgQuery::IndexStmt, PgQuery::AlterTableStmt 217 | schema_name = stmt.relation.schemaname || "public" 218 | table_name = stmt.relation.relname 219 | "#{schema_name}.#{table_name}" 220 | end 221 | rescue PgQuery::ParseError 222 | nil 223 | end 224 | 225 | def sanitize_identifier(identifier) 226 | identifier.gsub(/[^a-zA-Z0-9_]/, "_") 227 | end 228 | 229 | def fetch_ddl_query(source_conn_string, group_name, id: nil) 230 | source_conn = connect_to_internal_schema(source_conn_string) 231 | begin 232 | query = source_conn[table_name].where(group_name: group_name) 233 | query = query.where(id: id) if id 234 | result = query.order(:id).select_map(:ddl_command) 235 | result.uniq 236 | rescue => e 237 | abort_with("Fetching DDL queries failed: #{e.message}") 238 | ensure 239 | source_conn&.disconnect 240 | end 241 | end 242 | 243 | def apply_ddl_changes(target_conn_string, ddl_queries) 244 | target_conn = Query.connect(connection_url: target_conn_string) 245 | begin 246 | ddl_queries.each do |query| 247 | target_conn.run(query) 248 | rescue => e 249 | abort_with( 250 | "Error executing DDL command: #{query}. Error: #{e.message}", 251 | ) 252 | end 253 | rescue => e 254 | abort_with("Applying DDL changes failed: #{e.message}") 255 | ensure 256 | target_conn&.disconnect 257 | end 258 | end 259 | end 260 | end 261 | end 262 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/ddl_manager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | module DDLManager 5 | extend Helper 6 | 7 | class << self 8 | def setup_ddl_tracking( 9 | group_name:, conn_string: source_db_url, 10 | schema: "public" 11 | ) 12 | DDLAudit.setup(group_name) 13 | end 14 | 15 | def cleanup_ddl_tracking( 16 | group_name:, conn_string: source_db_url, 17 | schema: "public" 18 | ) 19 | DDLAudit.drop(group_name) 20 | end 21 | 22 | def list_ddl_changes( 23 | group_name:, conn_string: source_db_url, 24 | schema: "public", 25 | limit: 100 26 | ) 27 | DDLAudit.list_changes(group_name, limit: limit) 28 | end 29 | 30 | def apply_ddl_change( 31 | group_name:, id:, source_conn_string: source_db_url, 32 | target_conn_string: target_db_url, 33 | schema: "public" 34 | ) 35 | DDLAudit.apply_change( 36 | source_conn_string, 37 | target_conn_string, 38 | group_name, 39 | id, 40 | ) 41 | end 42 | 43 | def apply_all_ddl_changes( 44 | group_name:, source_conn_string: source_db_url, 45 | target_conn_string: target_db_url, 46 | schema: "public" 47 | ) 48 | DDLAudit.apply_all_changes( 49 | source_conn_string, 50 | target_conn_string, 51 | group_name, 52 | ) 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/group.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | class Group 5 | extend Helper 6 | class << self 7 | def setup 8 | conn = 9 | Query.connect( 10 | connection_url: source_db_url, 11 | schema: internal_schema_name, 12 | ) 13 | return if conn.table_exists?("groups") 14 | conn.create_table("groups") do 15 | primary_key(:id) 16 | column(:name, String, null: false) 17 | column(:table_names, String, text: true) 18 | column(:schema_name, String) 19 | column(:created_at, Time, default: Sequel::CURRENT_TIMESTAMP) 20 | column(:updated_at, Time, default: Sequel::CURRENT_TIMESTAMP) 21 | column(:started_at, Time) 22 | column(:failed_at, Time) 23 | column(:recreate_indices_post_copy, TrueClass, default: false) 24 | column(:switchover_completed_at, Time) 25 | end 26 | ensure 27 | conn&.disconnect 28 | end 29 | 30 | def drop 31 | conn = 32 | Query.connect( 33 | connection_url: source_db_url, 34 | schema: internal_schema_name, 35 | ).drop_table?("groups") 36 | ensure 37 | conn&.disconnect 38 | end 39 | 40 | def create(options) 41 | groups.insert( 42 | name: options[:name], 43 | table_names: options[:table_names], 44 | schema_name: options[:schema_name], 45 | started_at: options[:started_at], 46 | failed_at: options[:failed_at], 47 | recreate_indices_post_copy: options[:recreate_indices_post_copy], 48 | ) 49 | rescue => e 50 | abort_with("Adding group entry failed: #{e.message}") 51 | end 52 | 53 | def update( 54 | group_name:, 55 | started_at: nil, 56 | switchover_completed_at: nil, 57 | failed_at: nil 58 | ) 59 | set = { 60 | started_at: started_at&.utc, 61 | switchover_completed_at: switchover_completed_at&.utc, 62 | failed_at: failed_at&.utc, 63 | updated_at: Time.now.utc, 64 | }.compact 65 | groups.where(name: group_name).update(set) 66 | rescue => e 67 | abort_with("Updating group entry failed: #{e.message}") 68 | end 69 | 70 | def find(group_name) 71 | groups.first(name: group_name) 72 | rescue => e 73 | abort_with("Finding group entry failed: #{e.message}") 74 | end 75 | 76 | def delete(group_name) 77 | groups.where(name: group_name).delete 78 | rescue => e 79 | abort_with("Deleting group entry failed: #{e.message}") 80 | end 81 | 82 | private 83 | 84 | def groups 85 | conn = 86 | Query.connect( 87 | connection_url: source_db_url, 88 | schema: internal_schema_name, 89 | ) 90 | conn[:groups] 91 | end 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | module Helper 5 | def source_db_url 6 | ENV.fetch("SOURCE_DB_URL", nil) 7 | end 8 | 9 | def secondary_source_db_url 10 | ENV.fetch("SECONDARY_SOURCE_DB_URL", nil) 11 | end 12 | 13 | def target_db_url 14 | ENV.fetch("TARGET_DB_URL", nil) 15 | end 16 | 17 | def logger 18 | PgEasyReplicate.logger 19 | end 20 | 21 | def internal_schema_name 22 | "pger" 23 | end 24 | 25 | def internal_user_name 26 | "pger_su_h1a4fb" 27 | end 28 | 29 | def publication_name(group_name) 30 | "pger_publication_#{underscore(group_name)}" 31 | end 32 | 33 | def subscription_name(group_name) 34 | "pger_subscription_#{underscore(group_name)}" 35 | end 36 | 37 | def underscore(str) 38 | str 39 | .gsub("::", "/") 40 | .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2') 41 | .gsub(/([a-z\d])([A-Z])/, '\1_\2') 42 | .tr("-", "_") 43 | .downcase 44 | end 45 | 46 | def quote_ident(sql_ident) 47 | PG::Connection.quote_ident(sql_ident) 48 | end 49 | 50 | def test_env? 51 | ENV.fetch("RACK_ENV", nil) == "test" 52 | end 53 | 54 | def connection_info(conn_string) 55 | PG::Connection 56 | .conninfo_parse(conn_string) 57 | .each_with_object({}) do |obj, hash| 58 | hash[obj[:keyword].to_sym] = obj[:val] 59 | end 60 | .compact 61 | end 62 | 63 | def db_user(url) 64 | connection_info(url)[:user] 65 | end 66 | 67 | def db_name(url) 68 | connection_info(url)[:dbname] 69 | end 70 | 71 | def abort_with(msg) 72 | raise(msg) if test_env? 73 | abort(msg) 74 | end 75 | 76 | def determine_tables(conn_string:, list: "", exclude_list: "", schema: nil) 77 | schema ||= "public" 78 | 79 | tables = convert_to_array(list) 80 | exclude_tables = convert_to_array(exclude_list) 81 | validate_table_lists(tables, exclude_tables, schema) 82 | 83 | if tables.empty? 84 | all_tables = list_all_tables(schema: schema, conn_string: conn_string) 85 | all_tables - (exclude_tables + %w[spatial_ref_sys]) 86 | else 87 | tables 88 | end 89 | end 90 | 91 | def list_all_tables(schema:, conn_string:) 92 | Query 93 | .run( 94 | query: 95 | "SELECT c.relname::information_schema.sql_identifier AS table_name 96 | FROM pg_namespace n 97 | JOIN pg_class c ON n.oid = c.relnamespace 98 | WHERE c.relkind = 'r' 99 | AND c.relpersistence = 'p' 100 | AND n.nspname::information_schema.sql_identifier = '#{schema}' 101 | ORDER BY table_name", 102 | connection_url: conn_string, 103 | user: db_user(conn_string), 104 | ) 105 | .map(&:values) 106 | .flatten 107 | end 108 | 109 | def convert_to_array(input) 110 | input.is_a?(Array) ? input : input&.split(",") || [] 111 | end 112 | 113 | def validate_table_lists(tables, exclude_tables, schema_name) 114 | table_list = convert_to_array(tables) 115 | exclude_table_list = convert_to_array(exclude_tables) 116 | 117 | if !table_list.empty? && !exclude_table_list.empty? 118 | abort_with( 119 | "Options --tables(-t) and --exclude-tables(-e) cannot be used together.", 120 | ) 121 | elsif !table_list.empty? 122 | if table_list.size > 0 && (schema_name.nil? || schema_name == "") 123 | abort_with("Schema name is required if tables are passed") 124 | end 125 | elsif exclude_table_list.size > 0 && 126 | (schema_name.nil? || schema_name == "") 127 | abort_with("Schema name is required if exclude tables are passed") 128 | end 129 | end 130 | 131 | def restore_connections_on_source_db 132 | logger.info("Restoring connections") 133 | 134 | alter_sql = 135 | "ALTER USER #{quote_ident(db_user(source_db_url))} set default_transaction_read_only = false" 136 | Query.run(query: alter_sql, connection_url: source_db_url) 137 | end 138 | end 139 | end 140 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/index_manager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | module IndexManager 5 | extend Helper 6 | 7 | def self.drop_indices( 8 | source_conn_string:, 9 | target_conn_string:, 10 | tables:, 11 | schema: 12 | ) 13 | logger.info("Dropping indices from target database") 14 | 15 | fetch_indices( 16 | conn_string: source_conn_string, 17 | tables: tables, 18 | schema: schema, 19 | ).each do |index| 20 | drop_sql = "DROP INDEX CONCURRENTLY #{schema}.#{index[:index_name]};" 21 | 22 | Query.run( 23 | query: drop_sql, 24 | connection_url: target_conn_string, 25 | schema: schema, 26 | transaction: false, 27 | ) 28 | end 29 | end 30 | 31 | def self.recreate_indices( 32 | source_conn_string:, 33 | target_conn_string:, 34 | tables:, 35 | schema: 36 | ) 37 | logger.info("Recreating indices on target database") 38 | 39 | indices = 40 | fetch_indices( 41 | conn_string: source_conn_string, 42 | tables: tables, 43 | schema: schema, 44 | ) 45 | indices.each do |index| 46 | create_sql = 47 | "#{index[:index_definition].gsub("CREATE INDEX", "CREATE INDEX CONCURRENTLY IF NOT EXISTS")};" 48 | 49 | Query.run( 50 | query: create_sql, 51 | connection_url: target_conn_string, 52 | schema: schema, 53 | transaction: false, 54 | ) 55 | end 56 | end 57 | 58 | def self.fetch_indices(conn_string:, tables:, schema:) 59 | return [] if tables.empty? 60 | table_list = tables.map { |table| "'#{table}'" }.join(",") 61 | 62 | sql = <<-SQL 63 | SELECT 64 | t.relname AS table_name, 65 | i.relname AS index_name, 66 | pg_get_indexdef(i.oid) AS index_definition 67 | FROM 68 | pg_class t, 69 | pg_class i, 70 | pg_index ix, 71 | pg_namespace n 72 | WHERE 73 | t.oid = ix.indrelid 74 | AND i.oid = ix.indexrelid 75 | AND n.oid = t.relnamespace 76 | AND t.relkind = 'r' -- only find indexes of tables 77 | AND ix.indisprimary = FALSE -- exclude primary keys 78 | AND ix.indisunique = FALSE -- exclude unique indexes 79 | AND n.nspname = '#{schema}' 80 | AND t.relname IN (#{table_list}) 81 | ORDER BY 82 | t.relname, 83 | i.relname; 84 | SQL 85 | Query.run(query: sql, connection_url: conn_string, schema: schema) 86 | end 87 | 88 | def self.wait_for_replication_completion(group_name:) 89 | loop do 90 | break if Stats.all_tables_replicating?(group_name) 91 | sleep(5) 92 | end 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/orchestrate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | class Orchestrate 5 | extend Helper 6 | 7 | class << self 8 | DEFAULT_LAG = 200_000 # 200kb 9 | DEFAULT_WAIT = 5 # seconds 10 | 11 | def start_sync(options) 12 | schema_name = options[:schema_name] || "public" 13 | tables = 14 | determine_tables( 15 | schema: schema_name, 16 | conn_string: source_db_url, 17 | list: options[:tables], 18 | exclude_list: options[:exclude_tables], 19 | ) 20 | 21 | if options[:recreate_indices_post_copy] 22 | IndexManager.drop_indices( 23 | source_conn_string: source_db_url, 24 | target_conn_string: target_db_url, 25 | tables: tables, 26 | schema: schema_name, 27 | ) 28 | end 29 | 30 | create_publication( 31 | group_name: options[:group_name], 32 | conn_string: source_db_url, 33 | ) 34 | 35 | add_tables_to_publication( 36 | group_name: options[:group_name], 37 | tables: tables, 38 | conn_string: source_db_url, 39 | schema: schema_name, 40 | ) 41 | 42 | create_subscription( 43 | group_name: options[:group_name], 44 | source_conn_string: secondary_source_db_url || source_db_url, 45 | target_conn_string: target_db_url, 46 | ) 47 | 48 | Group.create( 49 | name: options[:group_name], 50 | table_names: tables.join(","), 51 | schema_name: schema_name, 52 | started_at: Time.now.utc, 53 | recreate_indices_post_copy: options[:recreate_indices_post_copy], 54 | ) 55 | 56 | if options[:track_ddl] 57 | DDLManager.setup_ddl_tracking( 58 | conn_string: source_db_url, 59 | group_name: options[:group_name], 60 | schema: schema_name, 61 | ) 62 | end 63 | logger.info("Starting sync completed successfully") 64 | rescue => e 65 | stop_sync(group_name: options[:group_name]) 66 | if Group.find(options[:group_name]) 67 | Group.update(name: options[:group_name], failed_at: Time.now.utc) 68 | end 69 | abort_with("Starting sync failed: #{e.message}") 70 | end 71 | 72 | def create_publication(group_name:, conn_string:) 73 | logger.info( 74 | "Setting up publication", 75 | { publication_name: publication_name(group_name) }, 76 | ) 77 | 78 | Query.run( 79 | query: "create publication #{publication_name(group_name)}", 80 | connection_url: conn_string, 81 | user: db_user(conn_string), 82 | ) 83 | rescue => e 84 | raise "Unable to create publication: #{e.message}" 85 | end 86 | 87 | def add_tables_to_publication( 88 | schema:, 89 | group_name:, 90 | conn_string:, 91 | tables: [] 92 | ) 93 | logger.info( 94 | "Adding tables up publication", 95 | { publication_name: publication_name(group_name) }, 96 | ) 97 | 98 | tables.map do |table_name| 99 | Query.run( 100 | query: 101 | "ALTER PUBLICATION #{quote_ident(publication_name(group_name))} 102 | ADD TABLE #{quote_ident(table_name)}", 103 | connection_url: conn_string, 104 | schema: schema, 105 | user: db_user(conn_string), 106 | ) 107 | end 108 | rescue => e 109 | raise "Unable to add tables to publication: #{e.message}" 110 | end 111 | 112 | def drop_publication(group_name:, conn_string:) 113 | logger.info( 114 | "Dropping publication", 115 | { publication_name: publication_name(group_name) }, 116 | ) 117 | Query.run( 118 | query: 119 | "DROP PUBLICATION IF EXISTS #{quote_ident(publication_name(group_name))}", 120 | connection_url: conn_string, 121 | user: db_user(conn_string), 122 | ) 123 | rescue => e 124 | raise "Unable to drop publication: #{e.message}" 125 | end 126 | 127 | def create_subscription( 128 | group_name:, 129 | source_conn_string:, 130 | target_conn_string: 131 | ) 132 | logger.info( 133 | "Setting up subscription", 134 | { 135 | publication_name: publication_name(group_name), 136 | subscription_name: subscription_name(group_name), 137 | }, 138 | ) 139 | 140 | Query.run( 141 | query: 142 | "CREATE SUBSCRIPTION #{quote_ident(subscription_name(group_name))} 143 | CONNECTION '#{source_conn_string}' 144 | PUBLICATION #{quote_ident(publication_name(group_name))}", 145 | connection_url: target_conn_string, 146 | user: db_user(target_conn_string), 147 | transaction: false, 148 | ) 149 | rescue Sequel::DatabaseError => e 150 | if e.message.include?("canceling statement due to statement timeout") 151 | abort_with( 152 | "Subscription creation failed, please ensure both databases are in the same network region: #{e.message}", 153 | ) 154 | end 155 | 156 | raise "Unable to create subscription: #{e.message}" 157 | end 158 | 159 | def drop_subscription(group_name:, target_conn_string:) 160 | logger.info( 161 | "Dropping subscription", 162 | { 163 | publication_name: publication_name(group_name), 164 | subscription_name: subscription_name(group_name), 165 | }, 166 | ) 167 | Query.run( 168 | query: "DROP SUBSCRIPTION IF EXISTS #{subscription_name(group_name)}", 169 | connection_url: target_conn_string, 170 | user: db_user(target_conn_string), 171 | transaction: false, 172 | ) 173 | rescue => e 174 | raise "Unable to drop subscription: #{e.message}" 175 | end 176 | 177 | def stop_sync( 178 | group_name:, 179 | source_conn_string: nil, 180 | target_conn_string: nil 181 | ) 182 | logger.info( 183 | "Stopping sync", 184 | { 185 | publication_name: publication_name(group_name), 186 | subscription_name: subscription_name(group_name), 187 | }, 188 | ) 189 | drop_publication( 190 | group_name: group_name, 191 | conn_string: source_conn_string || source_db_url, 192 | ) 193 | drop_subscription( 194 | group_name: group_name, 195 | target_conn_string: target_conn_string || target_db_url, 196 | ) 197 | logger.info("Stopping sync completed successfully") 198 | rescue => e 199 | abort_with("Unable to stop sync: #{e.message}") 200 | end 201 | 202 | def switchover( 203 | group_name:, 204 | lag_delta_size: nil, 205 | skip_vacuum_analyze: false, 206 | source_conn_string: nil, 207 | target_conn_string: nil 208 | ) 209 | group = Group.find(group_name) 210 | abort_with("Group not found: #{group_name}") unless group 211 | 212 | tables_list = group[:table_names].split(",") 213 | 214 | source_conn = source_conn_string || source_db_url 215 | target_conn = target_conn_string || target_db_url 216 | 217 | unless skip_vacuum_analyze 218 | run_vacuum_analyze( 219 | conn_string: target_conn, 220 | tables: tables_list, 221 | schema: group[:schema_name], 222 | ) 223 | end 224 | 225 | watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG) 226 | 227 | if group[:recreate_indices_post_copy] 228 | IndexManager.wait_for_replication_completion(group_name: group_name) 229 | IndexManager.recreate_indices( 230 | source_conn_string: source_conn, 231 | target_conn_string: target_conn, 232 | tables: tables_list, 233 | schema: group[:schema_name], 234 | ) 235 | end 236 | 237 | watch_lag(group_name: group_name, lag: lag_delta_size || DEFAULT_LAG) 238 | 239 | revoke_connections_on_source_db(group_name) 240 | wait_for_remaining_catchup(group_name) 241 | refresh_sequences(conn_string: target_conn, schema: group[:schema_name]) 242 | 243 | drop_subscription( 244 | group_name: group_name, 245 | target_conn_string: target_conn, 246 | ) 247 | mark_switchover_complete(group_name) 248 | 249 | unless skip_vacuum_analyze 250 | run_vacuum_analyze( 251 | conn_string: target_conn, 252 | tables: tables_list, 253 | schema: group[:schema_name], 254 | ) 255 | end 256 | rescue => e 257 | restore_connections_on_source_db 258 | abort_with("Switchover failed: #{e.message}") 259 | end 260 | 261 | def watch_lag(group_name:, wait_time: DEFAULT_WAIT, lag: DEFAULT_LAG) 262 | logger.info("Watching lag stats") 263 | 264 | loop do 265 | sleep(wait_time) 266 | 267 | unless Stats.all_tables_replicating?(group_name) 268 | logger.debug( 269 | "All tables haven't reached replicating state, skipping check", 270 | ) 271 | next 272 | end 273 | 274 | lag_stat = Stats.lag_stats(group_name).first 275 | if lag_stat[:write_lag].nil? || lag_stat[:flush_lag].nil? || 276 | lag_stat[:replay_lag].nil? 277 | next 278 | end 279 | 280 | logger.debug("Current lag stats: #{lag_stat}") 281 | 282 | below_write_lag = lag_stat[:write_lag] <= lag 283 | below_flush_lag = lag_stat[:flush_lag] <= lag 284 | below_replay_lag = lag_stat[:replay_lag] <= lag 285 | 286 | break if below_write_lag && below_flush_lag && below_replay_lag 287 | end 288 | 289 | logger.info("Lag below #{DEFAULT_LAG} bytes. Continuing...") 290 | end 291 | 292 | def wait_for_remaining_catchup(group_name) 293 | logger.info("Waiting for remaining WAL to get flushed") 294 | 295 | watch_lag(group_name: group_name, lag: 0, wait_time: 0.2) 296 | 297 | logger.info("Caught up on remaining WAL lag") 298 | end 299 | 300 | def revoke_connections_on_source_db(group_name) 301 | logger.info( 302 | "Lag is now below #{DEFAULT_LAG}, marking source DB to read only", 303 | ) 304 | 305 | alter_sql = 306 | "ALTER USER #{quote_ident(db_user(source_db_url))} set default_transaction_read_only = true" 307 | Query.run(query: alter_sql, connection_url: source_db_url) 308 | 309 | kill_sql = 310 | "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE usename = '#{db_user(source_db_url)}';" 311 | 312 | Query.run(query: kill_sql, connection_url: source_db_url) 313 | rescue => e 314 | raise "Unable to revoke connections on source db: #{e.message}" 315 | end 316 | 317 | def refresh_sequences(conn_string:, schema: nil) 318 | logger.info("Refreshing sequences") 319 | sql = <<~SQL 320 | DO $$ 321 | DECLARE 322 | i TEXT; 323 | BEGIN 324 | FOR i IN ( 325 | SELECT 'SELECT SETVAL(' 326 | || quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) 327 | || ', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' 328 | || quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';' 329 | FROM pg_class AS S, 330 | pg_depend AS D, 331 | pg_class AS T, 332 | pg_attribute AS C, 333 | pg_tables AS PGT 334 | WHERE S.relkind = 'S' 335 | AND S.oid = D.objid 336 | AND D.refobjid = T.oid 337 | AND D.refobjid = C.attrelid 338 | AND D.refobjsubid = C.attnum 339 | AND T.relname = PGT.tablename 340 | ) LOOP 341 | EXECUTE i; 342 | END LOOP; 343 | END $$; 344 | SQL 345 | 346 | Query.run(query: sql, connection_url: conn_string, schema: schema) 347 | rescue => e 348 | raise "Unable to refresh sequences: #{e.message}" 349 | end 350 | 351 | def run_vacuum_analyze(conn_string:, tables:, schema:) 352 | tables.each do |t| 353 | logger.info( 354 | "Running vacuum analyze on #{t}", 355 | schema: schema, 356 | table: t, 357 | ) 358 | 359 | Query.run( 360 | query: "VACUUM VERBOSE ANALYZE #{quote_ident(t)};", 361 | connection_url: conn_string, 362 | schema: schema, 363 | transaction: false, 364 | using_vacuum_analyze: true, 365 | ) 366 | end 367 | rescue => e 368 | raise "Unable to run vacuum and analyze: #{e.message}" 369 | end 370 | 371 | def mark_switchover_complete(group_name) 372 | Group.update(group_name: group_name, switchover_completed_at: Time.now) 373 | end 374 | end 375 | end 376 | end 377 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/query.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | class Query 5 | extend Helper 6 | 7 | class << self 8 | def run( 9 | query:, 10 | connection_url:, 11 | user: internal_user_name, 12 | schema: nil, 13 | transaction: true, 14 | using_vacuum_analyze: false 15 | ) 16 | conn = 17 | connect(connection_url: connection_url, schema: schema, user: user) 18 | timeout ||= ENV["PG_EASY_REPLICATE_STATEMENT_TIMEOUT"] || "5s" 19 | if transaction 20 | r = 21 | conn.transaction do 22 | conn.run("SET search_path to #{quote_ident(schema)}") if schema 23 | conn.run("SET statement_timeout to '#{timeout}'") 24 | conn.fetch(query).to_a 25 | end 26 | else 27 | conn.run("SET search_path to #{quote_ident(schema)}") if schema 28 | if using_vacuum_analyze 29 | conn.run("SET statement_timeout=0") 30 | else 31 | conn.run("SET statement_timeout to '#{timeout}'") 32 | end 33 | r = conn.fetch(query).to_a 34 | end 35 | conn.disconnect 36 | r 37 | ensure 38 | conn&.fetch("RESET statement_timeout") 39 | conn&.disconnect 40 | end 41 | 42 | def connect(connection_url:, user: internal_user_name, schema: nil) 43 | c = 44 | Sequel.connect( 45 | connection_url, 46 | user: user, 47 | logger: ENV.fetch("DEBUG", nil) ? logger : nil, 48 | search_path: schema, 49 | ) 50 | logger.debug("Connection established") 51 | c 52 | end 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/stats.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "net/http" 4 | 5 | module PgEasyReplicate 6 | class Stats 7 | REPLICATION_STATE_MAP = { 8 | "i" => "initializing", 9 | "d" => "data_is_being_copied", 10 | "f" => "finished_table_copy", 11 | "s" => "synchronized", 12 | "r" => "replicating", 13 | }.freeze 14 | extend Helper 15 | 16 | class << self 17 | def object(group_name) 18 | stats = replication_stats(group_name) 19 | group = Group.find(group_name) 20 | { 21 | lag_stats: lag_stats(group_name), 22 | replication_slots: pg_replication_slots(group_name), 23 | replication_stats: stats, 24 | replication_stats_count_by_state: 25 | replication_stats_count_by_state(stats), 26 | message_lsn_receipts: message_lsn_receipts(group_name), 27 | sync_started_at: group[:started_at], 28 | sync_failed_at: group[:failed_at], 29 | switchover_completed_at: group[:switchover_completed_at], 30 | } 31 | end 32 | 33 | def print(group_name) 34 | puts JSON.pretty_generate(object(group_name)) 35 | end 36 | 37 | def follow(group_name) 38 | loop do 39 | print(group_name) 40 | sleep(1) 41 | end 42 | end 43 | 44 | def notify(group_name, url, frequency = 10, timeout = 10) 45 | loop do 46 | stats = object(group_name) 47 | uri = URI.parse(url) 48 | 49 | http = Net::HTTP.new(uri.host, uri.port) 50 | http.use_ssl = (uri.scheme == "https") 51 | http.open_timeout = timeout 52 | http.read_timeout = timeout 53 | 54 | request = Net::HTTP::Post.new(uri.request_uri) 55 | request.content_type = "application/json" 56 | request.body = stats.to_json 57 | 58 | response = http.request(request) 59 | 60 | puts "Notification sent: #{response.code} #{response.message}" 61 | 62 | sleep(frequency) 63 | end 64 | rescue StandardError => e 65 | abort_with("Notify failed with: #{e.message}") 66 | end 67 | 68 | # Get 69 | def lag_stats(group_name) 70 | sql = <<~SQL 71 | SELECT pid, 72 | client_addr, 73 | usename as user_name, 74 | application_name, 75 | state, 76 | sync_state, 77 | pg_wal_lsn_diff(sent_lsn, write_lsn) AS write_lag, 78 | pg_wal_lsn_diff(sent_lsn, flush_lsn) AS flush_lag, 79 | pg_wal_lsn_diff(sent_lsn, replay_lsn) AS replay_lag 80 | FROM pg_stat_replication 81 | WHERE application_name = '#{subscription_name(group_name)}'; 82 | SQL 83 | 84 | Query.run(query: sql, connection_url: source_db_url) 85 | end 86 | 87 | def pg_replication_slots(group_name) 88 | sql = <<~SQL 89 | select * from pg_replication_slots WHERE slot_name = '#{subscription_name(group_name)}'; 90 | SQL 91 | 92 | Query.run(query: sql, connection_url: source_db_url) 93 | end 94 | 95 | def replication_stats(group_name) 96 | sql = <<~SQL 97 | SELECT 98 | s.subname AS subscription_name, 99 | c.relnamespace :: regnamespace :: text as table_schema, 100 | c.relname as table_name, 101 | rel.srsubstate as replication_state 102 | FROM 103 | pg_catalog.pg_subscription s 104 | JOIN pg_catalog.pg_subscription_rel rel ON rel.srsubid = s.oid 105 | JOIN pg_catalog.pg_class c on c.oid = rel.srrelid 106 | WHERE s.subname = '#{subscription_name(group_name)}' 107 | SQL 108 | 109 | Query 110 | .run(query: sql, connection_url: target_db_url) 111 | .each do |obj| 112 | obj[:replication_state] = REPLICATION_STATE_MAP[ 113 | obj[:replication_state] 114 | ] 115 | end 116 | end 117 | 118 | def all_tables_replicating?(group_name) 119 | result = 120 | replication_stats(group_name) 121 | .each 122 | .with_object(Hash.new(0)) do |state, counts| 123 | counts[state[:replication_state]] += 1 124 | end 125 | result.keys.uniq.count == 1 && 126 | result.keys.first == REPLICATION_STATE_MAP["r"] 127 | end 128 | 129 | def replication_stats_count_by_state(stats) 130 | stats 131 | .each 132 | .with_object(Hash.new(0)) do |state, counts| 133 | counts[state[:replication_state]] += 1 134 | end 135 | end 136 | 137 | def message_lsn_receipts(group_name) 138 | sql = <<~SQL 139 | select 140 | received_lsn, 141 | last_msg_send_time, 142 | last_msg_receipt_time, 143 | latest_end_lsn, 144 | latest_end_time 145 | from 146 | pg_catalog.pg_stat_subscription 147 | WHERE subname = '#{subscription_name(group_name)}' 148 | SQL 149 | Query.run(query: sql, connection_url: target_db_url) 150 | end 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/pg_easy_replicate/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PgEasyReplicate 4 | VERSION = "0.3.8" 5 | end 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pg-osc", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "repository": "git@github.com:shayonj/pg-osc.git", 6 | "author": "Shayon Mukherjee ", 7 | "license": "MIT", 8 | "private": true, 9 | "dependencies": { 10 | "@prettier/plugin-ruby": "^3.2.2", 11 | "prettier": "^2.8.8" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /pg_easy_replicate.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "lib/pg_easy_replicate/version" 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "pg_easy_replicate" 7 | spec.version = PgEasyReplicate::VERSION 8 | spec.authors = ["Shayon Mukherjee"] 9 | spec.email = ["shayonj@gmail.com"] 10 | 11 | spec.description = 12 | "Easily setup logical replication and switchover to new database with minimal downtime" 13 | spec.summary = spec.description 14 | spec.homepage = "https://github.com/shayonj/pg_easy_replicate" 15 | spec.license = "MIT" 16 | spec.required_ruby_version = ">= 3.0.0" 17 | 18 | spec.metadata["homepage_uri"] = spec.homepage 19 | spec.metadata["source_code_uri"] = spec.homepage 20 | spec.metadata[ 21 | "changelog_uri" 22 | ] = "https://github.com/shayonj/pg_easy_replicate/blob/main/CODE_OF_CONDUCT.md" 23 | 24 | # Specify which files should be added to the gem when it is released. 25 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 26 | spec.files = 27 | Dir.chdir(File.expand_path(__dir__)) do 28 | `git ls-files -z`.split("\x0") 29 | .reject do |f| 30 | (f == __FILE__) || 31 | f.match( 32 | %r{\A(?:(?:test|spec|features)/|\.(?:git|travis|circleci)|appveyor)}, 33 | ) 34 | end 35 | end 36 | spec.bindir = "bin" 37 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 38 | spec.require_paths = ["lib"] 39 | spec.metadata = { "rubygems_mfa_required" => "true" } 40 | 41 | spec.add_runtime_dependency("ougai", "~> 2.0.0") 42 | spec.add_runtime_dependency("pg", "~> 1.5.3") 43 | spec.add_runtime_dependency("pg_query", "~> 5.1.0") 44 | spec.add_runtime_dependency("sequel", ">= 5.69", "< 5.87") 45 | spec.add_runtime_dependency("thor", ">= 1.2.2", "< 1.4.0") 46 | 47 | # rubocop:disable Gemspec/DevelopmentDependencies 48 | spec.add_development_dependency("prettier_print") 49 | spec.add_development_dependency("pry") 50 | spec.add_development_dependency("rake") 51 | spec.add_development_dependency("rspec") 52 | spec.add_development_dependency("rubocop") 53 | spec.add_development_dependency("rubocop-packaging") 54 | spec.add_development_dependency("rubocop-performance") 55 | spec.add_development_dependency("rubocop-rake") 56 | spec.add_development_dependency("rubocop-rspec") 57 | spec.add_development_dependency("syntax_tree") 58 | spec.add_development_dependency("syntax_tree-haml") 59 | spec.add_development_dependency("syntax_tree-rbs") 60 | # rubocop:enable Gemspec/DevelopmentDependencies 61 | end 62 | -------------------------------------------------------------------------------- /scripts/e2e-bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | if [[ -z ${GITHUB_WORKFLOW} ]]; then 6 | export SECONDARY_SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@source_db/postgres-db" 7 | fi 8 | 9 | export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5432/postgres-db" 10 | export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db" 11 | export PGPASSWORD='james-bond123@7!'"'"'3aaR' 12 | 13 | pgbench --initialize -s 5 --foreign-keys --host localhost -U james-bond -d postgres-db 14 | 15 | bundle exec bin/pg_easy_replicate config_check --copy-schema 16 | -------------------------------------------------------------------------------- /scripts/e2e-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | if [[ -z ${GITHUB_WORKFLOW} ]]; then 6 | export SECONDARY_SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@source_db/postgres-db" 7 | fi 8 | 9 | export SOURCE_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5432/postgres-db" 10 | export TARGET_DB_URL="postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db" 11 | export PGPASSWORD='james-bond123@7!'"'"''"'"'3aaR' 12 | 13 | echo "===== Performing Bootstrap and cleanup" 14 | bundle exec bin/pg_easy_replicate bootstrap -g cluster-1 --copy-schema 15 | bundle exec bin/pg_easy_replicate start_sync -g cluster-1 -s public --recreate-indices-post-copy --track-ddl 16 | bundle exec bin/pg_easy_replicate stats -g cluster-1 17 | 18 | echo "===== Applying DDL change" 19 | psql $SOURCE_DB_URL -c "ALTER TABLE public.pgbench_accounts ADD COLUMN test_column VARCHAR(255)" 20 | 21 | echo "===== Applying DDL changes" 22 | echo "Y" | bundle exec bin/pg_easy_replicate apply_ddl_change -g cluster-1 23 | 24 | # Switchover 25 | echo "===== Performing switchover" 26 | bundle exec bin/pg_easy_replicate switchover -g cluster-1 27 | -------------------------------------------------------------------------------- /scripts/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | export VERSION=$1 6 | echo "VERSION: ${VERSION}" 7 | 8 | echo "=== Building Gem ====" 9 | gem build pg_easy_replicate.gemspec 10 | 11 | echo "=== Pushing gem ====" 12 | gem push pg_easy_replicate-"$VERSION".gem 13 | 14 | echo "=== Sleeping for 15s ====" 15 | sleep 15 16 | 17 | echo "=== Pushing tags to github ====" 18 | git tag v"$VERSION" 19 | git push origin --tags 20 | 21 | echo "=== Cleaning up ====" 22 | rm pg_easy_replicate-"$VERSION".gem 23 | -------------------------------------------------------------------------------- /spec/database_helpers.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module DatabaseHelpers 4 | def test_schema 5 | ENV["POSTGRES_SCHEMA"] || "pger_test" 6 | end 7 | 8 | # We are use url encoded password below. 9 | # Original password is james-bond123@7!'3aaR 10 | def connection_url(user = "james-bond") 11 | "postgres://#{user}:james-bond123%407%21%273aaR@localhost:5432/postgres-db" 12 | end 13 | 14 | def target_connection_url(user = "james-bond") 15 | "postgres://#{user}:james-bond123%407%21%273aaR@localhost:5433/postgres-db" 16 | end 17 | 18 | def docker_compose_target_connection_url(user = "james-bond") 19 | "postgres://#{user}:james-bond123%407%21%273aaR@target_db/postgres-db" 20 | end 21 | 22 | def docker_compose_source_connection_url(user = "james-bond") 23 | return connection_url(user) if ENV["GITHUB_WORKFLOW"] # if running in CI/github actions 24 | "postgres://#{user}:james-bond123%407%21%273aaR@source_db/postgres-db" 25 | end 26 | 27 | def setup_tables(user = "james-bond", setup_target_db: true) 28 | setup(connection_url(user), user) 29 | setup(target_connection_url(user), user) if setup_target_db 30 | end 31 | 32 | def setup_roles 33 | [connection_url, target_connection_url].each do |url| 34 | PgEasyReplicate::Query.run( 35 | query: 36 | "drop role if exists #{PG::Connection.quote_ident("james-bond_sup")}; create user #{PG::Connection.quote_ident("james-bond_sup")} with login superuser password 'james-bond123@7!''3aaR';", 37 | connection_url: url, 38 | user: "james-bond", 39 | ) 40 | 41 | PgEasyReplicate::Query.run( 42 | query: 43 | "drop role if exists no_sup; create user no_sup with login password 'james-bond123@7!''3aaR';", 44 | connection_url: url, 45 | user: "james-bond", 46 | ) 47 | 48 | # setup role 49 | PgEasyReplicate::Query.run( 50 | query: 51 | "drop role if exists #{PG::Connection.quote_ident("james-bond_super_role")}; create role #{PG::Connection.quote_ident("james-bond_super_role")} with createdb createrole replication;", 52 | connection_url: url, 53 | user: "james-bond", 54 | ) 55 | 56 | # setup user with role 57 | sql = <<~SQL 58 | drop role if exists #{PG::Connection.quote_ident("james-bond_role_regular")}; 59 | create role #{PG::Connection.quote_ident("james-bond_role_regular")} WITH createrole createdb replication LOGIN PASSWORD 'james-bond123@7!''3aaR'; grant #{PG::Connection.quote_ident("james-bond_super_role")} to #{PG::Connection.quote_ident("james-bond_role_regular")}; 60 | grant all privileges on database #{PG::Connection.quote_ident("postgres-db")} TO #{PG::Connection.quote_ident("james-bond_role_regular")}; 61 | SQL 62 | PgEasyReplicate::Query.run( 63 | query: sql, 64 | connection_url: url, 65 | user: "james-bond", 66 | ) 67 | end 68 | end 69 | 70 | def cleanup_roles 71 | %w[ 72 | james-bond_sup 73 | no_sup 74 | james-bond_role_regular 75 | james-bond_super_role 76 | ].each do |role| 77 | if role == "james-bond_role_regular" 78 | PgEasyReplicate::Query.run( 79 | query: 80 | "revoke all privileges on database #{PG::Connection.quote_ident("postgres-db")} from #{PG::Connection.quote_ident("james-bond_role_regular")};", 81 | connection_url: connection_url, 82 | user: "james-bond", 83 | ) 84 | 85 | PgEasyReplicate::Query.run( 86 | query: 87 | "revoke all privileges on database #{PG::Connection.quote_ident("postgres-db")} from #{PG::Connection.quote_ident("james-bond_role_regular")};", 88 | connection_url: target_connection_url, 89 | user: "james-bond", 90 | ) 91 | end 92 | 93 | PgEasyReplicate::Query.run( 94 | query: "drop role if exists #{PG::Connection.quote_ident(role)};", 95 | connection_url: connection_url, 96 | user: "james-bond", 97 | ) 98 | 99 | PgEasyReplicate::Query.run( 100 | query: "drop role if exists #{PG::Connection.quote_ident(role)};", 101 | connection_url: target_connection_url, 102 | user: "james-bond", 103 | ) 104 | end 105 | end 106 | 107 | def setup(connection_url, user = "james-bond") 108 | PgEasyReplicate::Query.run( 109 | query: "DROP SCHEMA IF EXISTS #{test_schema} CASCADE;", 110 | connection_url: connection_url, 111 | user: user, 112 | ) 113 | 114 | conn = 115 | PgEasyReplicate::Query.connect(connection_url: connection_url, user: user) 116 | conn.run( 117 | "CREATE SCHEMA IF NOT EXISTS #{test_schema}; SET search_path TO #{test_schema};", 118 | ) 119 | 120 | unless conn.table_exists?("sellers") 121 | conn.create_table("sellers") do 122 | primary_key(:id) 123 | column(:name, String, unique: true) 124 | column(:last_login, Time) 125 | index(:id) 126 | index(:last_login) 127 | end 128 | end 129 | 130 | unless conn.table_exists?("items") 131 | conn.create_table("items") do 132 | primary_key(:id) 133 | column(:name, String) 134 | column(:last_purchase_at, Time) 135 | foreign_key(:seller_id, :sellers, on_delete: :cascade) 136 | index(:seller_id) 137 | index(:id) 138 | end 139 | end 140 | 141 | unless conn.table_exists?("spatial_ref_sys") 142 | conn.create_table("spatial_ref_sys") do 143 | primary_key(:id) 144 | column(:name, String) 145 | column(:last_purchase_at, Time) 146 | foreign_key(:seller_id, :sellers, on_delete: :cascade) 147 | index(:seller_id) 148 | index(:id) 149 | end 150 | end 151 | ensure 152 | conn&.disconnect 153 | end 154 | 155 | def teardown_tables 156 | PgEasyReplicate::Query.run( 157 | query: "DROP SCHEMA IF EXISTS #{test_schema} CASCADE;", 158 | connection_url: connection_url, 159 | user: "james-bond", 160 | ) 161 | 162 | PgEasyReplicate::Query.run( 163 | query: "DROP SCHEMA IF EXISTS #{test_schema} CASCADE;", 164 | connection_url: target_connection_url, 165 | user: "james-bond", 166 | ) 167 | end 168 | 169 | def get_schema 170 | PgEasyReplicate::Query.run( 171 | query: 172 | "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '#{PgEasyReplicate.internal_schema_name}';", 173 | connection_url: connection_url, 174 | schema: PgEasyReplicate.internal_schema_name, 175 | user: "james-bond", 176 | ) 177 | end 178 | 179 | def groups_table_exists? 180 | PgEasyReplicate::Query.run( 181 | query: 182 | "SELECT table_name FROM information_schema.tables WHERE table_name = 'groups'", 183 | connection_url: connection_url, 184 | schema: PgEasyReplicate.internal_schema_name, 185 | user: "james-bond", 186 | ) 187 | end 188 | 189 | def user_permissions(connection_url:, group_name:) 190 | PgEasyReplicate::Query.run( 191 | query: 192 | "select rolcreatedb, rolcreaterole, rolcanlogin, rolsuper from pg_authid where rolname = 'pger_su_h1a4fb';", 193 | connection_url: connection_url, 194 | user: "james-bond", 195 | ) 196 | end 197 | 198 | def pg_subscriptions(connection_url:) 199 | PgEasyReplicate::Query.run( 200 | query: 201 | "select subname, subpublications, subslotname, subenabled from pg_subscription;", 202 | connection_url: connection_url, 203 | user: "james-bond", 204 | ) 205 | end 206 | 207 | def pg_publication_tables(connection_url:) 208 | PgEasyReplicate::Query.run( 209 | query: "select * from pg_publication_tables;", 210 | connection_url: connection_url, 211 | user: "james-bond", 212 | ) 213 | end 214 | 215 | def pg_publications(connection_url:) 216 | PgEasyReplicate::Query.run( 217 | query: "select pubname from pg_catalog.pg_publication", 218 | connection_url: connection_url, 219 | user: "james-bond", 220 | ) 221 | end 222 | 223 | def vacuum_stats(url:, schema:) 224 | PgEasyReplicate::Query.run( 225 | connection_url: url, 226 | schema: schema, 227 | query: 228 | "SELECT last_vacuum, last_analyze, relname FROM pg_stat_all_tables WHERE schemaname = '#{schema}'", 229 | ) 230 | end 231 | 232 | def self.populate_env_vars 233 | ENV[ 234 | "SOURCE_DB_URL" 235 | ] = "postgres://james-bond:james-bond123%407%21%273aaR@localhost:5432/postgres-db" 236 | ENV[ 237 | "TARGET_DB_URL" 238 | ] = "postgres://james-bond:james-bond123%407%21%273aaR@localhost:5433/postgres-db" 239 | end 240 | 241 | def table_exists?(*args) 242 | if args.size == 1 243 | table_name = args.first 244 | schema = PgEasyReplicate::DDLAudit.send(:internal_schema_name) 245 | conn_url = connection_url 246 | elsif args.size == 3 247 | conn_url, schema, table_name = args 248 | else 249 | raise ArgumentError, 250 | "Wrong number of arguments (given #{args.size}, expected 1 or 3)" 251 | end 252 | 253 | PgEasyReplicate::Query.run( 254 | query: 255 | "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '#{schema}' AND table_name = '#{table_name}') AS exists", 256 | connection_url: conn_url, 257 | schema: schema, 258 | user: "james-bond", 259 | ).first[ 260 | :exists 261 | ] 262 | end 263 | 264 | def column_exists?(conn_url, schema, table, column) 265 | PgEasyReplicate::Query.run( 266 | query: 267 | "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_schema = '#{schema}' AND table_name = '#{table}' AND column_name = '#{column}') AS exists", 268 | connection_url: conn_url, 269 | schema: schema, 270 | user: "james-bond", 271 | ).first[ 272 | :exists 273 | ] 274 | end 275 | 276 | def function_exists?(function_name, conn_url = connection_url) 277 | PgEasyReplicate::Query.run( 278 | query: 279 | "SELECT EXISTS (SELECT 1 FROM pg_proc WHERE proname = '#{function_name}') AS exists", 280 | connection_url: conn_url, 281 | user: "james-bond", 282 | ).first[ 283 | :exists 284 | ] 285 | end 286 | 287 | def event_triggers_exist?(group_name, conn_url = nil) 288 | conn_url ||= connection_url # Use the default connection_url if not provided 289 | raise "No connection URL provided" if conn_url.nil? 290 | 291 | PgEasyReplicate::Query.run( 292 | query: 293 | "SELECT COUNT(*) FROM pg_event_trigger WHERE evtname IN ('pger_ddl_trigger_#{group_name}', 'pger_drop_trigger_#{group_name}', 'pger_table_rewrite_trigger_#{group_name}')", 294 | connection_url: conn_url, 295 | schema: PgEasyReplicate::DDLAudit.send(:internal_schema_name), 296 | ).first[ 297 | :count 298 | ] == 3 299 | end 300 | 301 | def execute_ddl(query, conn_url = connection_url) 302 | PgEasyReplicate::Query.run( 303 | query: query, 304 | connection_url: conn_url, 305 | schema: test_schema, 306 | user: "james-bond", 307 | ) 308 | end 309 | 310 | def ddl_audit_table_exists?(conn_url = nil, table_name = nil) 311 | schema = PgEasyReplicate::DDLAudit.send(:internal_schema_name) 312 | conn_url = connection_url if conn_url.nil? || conn_url.is_a?(Symbol) 313 | table_name ||= "pger_ddl_audits" 314 | 315 | PgEasyReplicate::Query.run( 316 | query: 317 | "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '#{schema}' AND table_name = '#{table_name}') AS exists", 318 | connection_url: conn_url, 319 | schema: schema, 320 | user: "james-bond", 321 | ).first[ 322 | :exists 323 | ] 324 | end 325 | 326 | def ddl_trigger_function_exists?(conn_url = connection_url) 327 | function_exists?("pger_ddl_trigger", conn_url) 328 | end 329 | 330 | def table_exists_in_schema?(conn_url, schema, table_name) 331 | PgEasyReplicate::Query.run( 332 | query: 333 | "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = '#{schema}' AND table_name = '#{table_name}') AS exists", 334 | connection_url: conn_url, 335 | schema: schema, 336 | user: "james-bond", 337 | ).first[ 338 | :exists 339 | ] 340 | end 341 | end 342 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate/ddl_audit_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe(PgEasyReplicate::DDLAudit) do 4 | let(:schema_name) { "pger_test" } 5 | let(:group_name) { "cluster1" } 6 | 7 | before do 8 | setup_tables 9 | PgEasyReplicate.bootstrap({ group_name: group_name }) 10 | PgEasyReplicate::Group.create( 11 | name: group_name, 12 | table_names: "sellers,items", 13 | schema_name: schema_name, 14 | started_at: Time.now.utc, 15 | ) 16 | end 17 | 18 | after do 19 | teardown_tables 20 | PgEasyReplicate.cleanup({ everything: true }) 21 | end 22 | 23 | describe ".setup" do 24 | it "creates the DDL audit table and triggers" do 25 | described_class.setup(group_name) 26 | 27 | trigger_status = 28 | PgEasyReplicate::Query.run( 29 | query: 30 | "SELECT evtenabled FROM pg_event_trigger WHERE evtname = 'pger_ddl_trigger_#{group_name}'", 31 | connection_url: connection_url, 32 | ).first 33 | puts "Debug: Trigger status - #{trigger_status.inspect}" 34 | 35 | table_exists = 36 | ddl_audit_table_exists?(nil, described_class.send(:table_name)) 37 | expect(table_exists).to be(true) 38 | 39 | trigger_function_exists = 40 | function_exists?("pger_ddl_trigger_#{group_name}") 41 | expect(trigger_function_exists).to be(true) 42 | 43 | event_triggers_exist = event_triggers_exist?(group_name) 44 | expect(event_triggers_exist).to be(true) 45 | end 46 | 47 | it "doesn't create the table if it already exists" do 48 | described_class.setup(group_name) 49 | expect { described_class.setup(group_name) }.not_to raise_error 50 | expect(table_exists?(described_class.send(:table_name))).to be(true) 51 | end 52 | end 53 | 54 | describe ".drop" do 55 | before { described_class.setup(group_name) } 56 | 57 | it "drops the DDL audit table, triggers, and function" do 58 | described_class.drop(group_name) 59 | 60 | table_exists = table_exists?(described_class.send(:table_name)) 61 | expect(table_exists).to be(true) # Table should still exist, only group-specific data is deleted 62 | 63 | trigger_function_exists = 64 | function_exists?("pger_ddl_trigger_#{group_name}") 65 | expect(trigger_function_exists).to be(false) 66 | 67 | event_triggers_exist = event_triggers_exist?(group_name) 68 | expect(event_triggers_exist).to be(false) 69 | end 70 | end 71 | 72 | describe "DDL change capture" do 73 | before { described_class.setup(group_name) } 74 | 75 | it "captures ALTER TABLE DDL for tables in the group" do 76 | execute_ddl( 77 | "ALTER TABLE #{schema_name}.sellers ADD COLUMN test_column VARCHAR(255)", 78 | ) 79 | 80 | changes = described_class.list_changes(group_name) 81 | expect(changes.size).to eq(1) 82 | expect(changes.first[:event_type]).to eq("ddl_command_end") 83 | expect(changes.first[:object_type]).to eq("table") 84 | expect(changes.first[:object_identity]).to eq("#{schema_name}.sellers") 85 | expect(changes.first[:ddl_command]).to include("ALTER TABLE") 86 | expect(changes.first[:ddl_command]).to include("ADD COLUMN test_column") 87 | end 88 | 89 | it "captures CREATE INDEX DDL for tables in the group" do 90 | execute_ddl( 91 | "CREATE INDEX idx_sellers_name ON #{schema_name}.sellers (name)", 92 | ) 93 | 94 | changes = described_class.list_changes(group_name) 95 | expect(changes.size).to eq(1) 96 | expect(changes.first[:event_type]).to eq("ddl_command_end") 97 | expect(changes.first[:object_type]).to eq("index") 98 | expect(changes.first[:object_identity]).to eq( 99 | "#{schema_name}.idx_sellers_name", 100 | ) 101 | expect(changes.first[:ddl_command]).to include("CREATE INDEX") 102 | expect(changes.first[:ddl_command]).to include( 103 | "ON #{schema_name}.sellers", 104 | ) 105 | end 106 | 107 | it "does not capture DDL for tables not in the group" do 108 | execute_ddl( 109 | "CREATE TABLE #{schema_name}.not_in_group (id serial PRIMARY KEY)", 110 | ) 111 | 112 | described_class.list_changes(group_name) 113 | 114 | execute_ddl( 115 | "ALTER TABLE #{schema_name}.not_in_group ADD COLUMN test_column VARCHAR(255)", 116 | ) 117 | 118 | changes = described_class.list_changes(group_name) 119 | expect(changes.size).to eq(0) 120 | 121 | execute_ddl("DROP TABLE #{schema_name}.not_in_group") 122 | end 123 | 124 | it "captures CREATE and DROP INDEX DDL for tables in the group" do 125 | execute_ddl( 126 | "CREATE INDEX idx_sellers_name ON #{schema_name}.sellers (name)", 127 | ) 128 | 129 | execute_ddl("DROP INDEX #{schema_name}.idx_sellers_name") 130 | 131 | changes = described_class.list_changes(group_name) 132 | 133 | expect(changes.size).to eq(2) 134 | 135 | create_index_change = 136 | changes.find { |c| c[:ddl_command].include?("CREATE INDEX") } 137 | drop_index_change = 138 | changes.find { |c| c[:ddl_command].include?("DROP INDEX") } 139 | 140 | expect(create_index_change).not_to be_nil 141 | expect(create_index_change[:event_type]).to eq("ddl_command_end") 142 | expect(create_index_change[:object_type]).to eq("index") 143 | expect(create_index_change[:object_identity]).to eq( 144 | "#{schema_name}.idx_sellers_name", 145 | ) 146 | expect(create_index_change[:ddl_command]).to include("CREATE INDEX") 147 | expect(create_index_change[:ddl_command]).to include( 148 | "ON #{schema_name}.sellers", 149 | ) 150 | 151 | expect(drop_index_change).not_to be_nil 152 | expect(drop_index_change[:event_type]).to eq("sql_drop") 153 | expect(drop_index_change[:object_type]).to eq("index") 154 | expect(drop_index_change[:object_identity]).to eq( 155 | "#{schema_name}.idx_sellers_name", 156 | ) 157 | expect(drop_index_change[:ddl_command]).to include("DROP INDEX") 158 | end 159 | 160 | it "captures ALTER TABLE DDL for adding and renaming a column" do 161 | execute_ddl( 162 | "ALTER TABLE #{schema_name}.sellers ADD COLUMN temp_email VARCHAR(255)", 163 | ) 164 | execute_ddl( 165 | "ALTER TABLE #{schema_name}.sellers RENAME COLUMN temp_email TO permanent_email", 166 | ) 167 | 168 | changes = described_class.list_changes(group_name) 169 | 170 | expect(changes.size).to eq(2) 171 | 172 | sorted_changes = changes.sort_by { |change| change[:created_at] } 173 | 174 | add_column_change = sorted_changes[0] 175 | rename_column_change = sorted_changes[1] 176 | 177 | expect(add_column_change[:event_type]).to eq("ddl_command_end") 178 | expect(add_column_change[:object_type]).to eq("table") 179 | expect(add_column_change[:object_identity]).to eq( 180 | "#{schema_name}.sellers", 181 | ) 182 | expect(add_column_change[:ddl_command]).to include("ALTER TABLE") 183 | expect(add_column_change[:ddl_command]).to include( 184 | "ADD COLUMN temp_email", 185 | ) 186 | 187 | expect(rename_column_change[:event_type]).to eq("ddl_command_end") 188 | expect(rename_column_change[:object_type]).to eq("table column") 189 | expect(rename_column_change[:object_identity]).to eq( 190 | "#{schema_name}.sellers.permanent_email", 191 | ) 192 | expect(rename_column_change[:ddl_command]).to include("ALTER TABLE") 193 | expect(rename_column_change[:ddl_command]).to include( 194 | "RENAME COLUMN temp_email TO permanent_email", 195 | ) 196 | end 197 | end 198 | 199 | describe ".list_changes" do 200 | before do 201 | described_class.setup(group_name) 202 | execute_ddl( 203 | "ALTER TABLE #{schema_name}.sellers ADD COLUMN email VARCHAR(255)", 204 | ) 205 | end 206 | 207 | it "lists DDL changes for the specific group" do 208 | changes = described_class.list_changes(group_name) 209 | 210 | expect(changes.size).to eq(1) 211 | expect(changes.first.keys).to match_array( 212 | %i[ 213 | id 214 | group_name 215 | created_at 216 | event_type 217 | object_type 218 | object_identity 219 | ddl_command 220 | ], 221 | ) 222 | expect(changes.first[:group_name]).to eq(group_name) 223 | expect(changes.first[:event_type]).to eq("ddl_command_end") 224 | expect(changes.first[:object_type]).to eq("table") 225 | expect(changes.first[:object_identity]).to include("sellers") 226 | expect(changes.first[:ddl_command]).to include("ALTER TABLE") 227 | expect(changes.first[:ddl_command]).to include("ADD COLUMN email") 228 | end 229 | end 230 | 231 | describe ".apply_change" do 232 | before { described_class.setup(group_name) } 233 | 234 | it "applies ALTER TABLE DDL change to the target database" do 235 | execute_ddl( 236 | "ALTER TABLE #{schema_name}.sellers ADD COLUMN email VARCHAR(255)", 237 | ) 238 | change_id = described_class.list_changes(group_name).first[:id] 239 | 240 | described_class.apply_change( 241 | connection_url, 242 | target_connection_url, 243 | group_name, 244 | change_id, 245 | ) 246 | 247 | column_exists = 248 | column_exists?(target_connection_url, schema_name, "sellers", "email") 249 | expect(column_exists).to be(true) 250 | end 251 | end 252 | 253 | describe ".apply_all_changes" do 254 | before { described_class.setup(group_name) } 255 | 256 | it "applies all DDL changes for the specific group to the target database" do 257 | execute_ddl( 258 | "ALTER TABLE #{schema_name}.sellers ADD COLUMN email VARCHAR(255)", 259 | ) 260 | execute_ddl( 261 | "ALTER TABLE #{schema_name}.items ADD COLUMN description TEXT", 262 | ) 263 | execute_ddl( 264 | "CREATE TABLE #{schema_name}.not_in_group (id serial PRIMARY KEY)", 265 | ) # This should not be applied 266 | 267 | described_class.apply_all_changes( 268 | connection_url, 269 | target_connection_url, 270 | group_name, 271 | ) 272 | 273 | sellers_column_exists = 274 | column_exists?(target_connection_url, schema_name, "sellers", "email") 275 | items_column_exists = 276 | column_exists?( 277 | target_connection_url, 278 | schema_name, 279 | "items", 280 | "description", 281 | ) 282 | not_in_group_exists = 283 | table_exists_in_schema?( 284 | target_connection_url, 285 | schema_name, 286 | "not_in_group", 287 | ) 288 | 289 | expect(sellers_column_exists).to be(true) 290 | expect(items_column_exists).to be(true) 291 | expect(not_in_group_exists).to be(false) 292 | end 293 | end 294 | end 295 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate/ddl_manager_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe(PgEasyReplicate::DDLManager) do 4 | let(:group_name) { "test_group" } 5 | let(:schema_name) { "public" } 6 | let(:conn_string) { "postgres://user:password@localhost:5432/testdb" } 7 | let(:source_conn_string) do 8 | "postgres://user:password@localhost:5432/sourcedb" 9 | end 10 | let(:target_conn_string) do 11 | "postgres://user:password@localhost:5432/targetdb" 12 | end 13 | 14 | describe ".setup_ddl_tracking" do 15 | it "calls DDLAudit.setup with the correct parameters" do 16 | expect(PgEasyReplicate::DDLAudit).to receive(:setup).with(group_name) 17 | 18 | described_class.setup_ddl_tracking( 19 | conn_string: conn_string, 20 | group_name: group_name, 21 | schema: schema_name, 22 | ) 23 | end 24 | end 25 | 26 | describe ".cleanup_ddl_tracking" do 27 | it "calls DDLAudit.drop with the correct parameters" do 28 | expect(PgEasyReplicate::DDLAudit).to receive(:drop).with(group_name) 29 | 30 | described_class.cleanup_ddl_tracking( 31 | conn_string: conn_string, 32 | group_name: group_name, 33 | schema: schema_name, 34 | ) 35 | end 36 | end 37 | 38 | describe ".list_ddl_changes" do 39 | it "calls DDLAudit.list_changes with the correct parameters" do 40 | limit = 50 41 | expect(PgEasyReplicate::DDLAudit).to receive(:list_changes).with( 42 | group_name, 43 | limit: limit, 44 | ) 45 | 46 | described_class.list_ddl_changes( 47 | conn_string: conn_string, 48 | group_name: group_name, 49 | schema: schema_name, 50 | limit: limit, 51 | ) 52 | end 53 | end 54 | 55 | describe ".apply_ddl_change" do 56 | it "calls DDLAudit.apply_change with the correct parameters" do 57 | id = 1 58 | expect(PgEasyReplicate::DDLAudit).to receive(:apply_change).with( 59 | source_conn_string, 60 | target_conn_string, 61 | group_name, 62 | id, 63 | ) 64 | 65 | described_class.apply_ddl_change( 66 | source_conn_string: source_conn_string, 67 | target_conn_string: target_conn_string, 68 | group_name: group_name, 69 | id: id, 70 | schema: schema_name, 71 | ) 72 | end 73 | end 74 | 75 | describe ".apply_all_ddl_changes" do 76 | it "calls DDLAudit.apply_all_changes with the correct parameters" do 77 | expect(PgEasyReplicate::DDLAudit).to receive(:apply_all_changes).with( 78 | source_conn_string, 79 | target_conn_string, 80 | group_name, 81 | ) 82 | 83 | described_class.apply_all_ddl_changes( 84 | source_conn_string: source_conn_string, 85 | target_conn_string: target_conn_string, 86 | group_name: group_name, 87 | schema: schema_name, 88 | ) 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate/group_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe(PgEasyReplicate::Group) do 4 | describe ".setup" do 5 | before do 6 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 7 | PgEasyReplicate.setup_internal_schema 8 | end 9 | 10 | after do 11 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 12 | end 13 | 14 | it "creates the table" do 15 | described_class.setup 16 | 17 | r = 18 | PgEasyReplicate::Query.run( 19 | query: "select * from groups", 20 | connection_url: connection_url, 21 | schema: PgEasyReplicate.internal_schema_name, 22 | ) 23 | expect(r).to eq([]) 24 | 25 | columns_sql = <<~SQL 26 | SELECT column_name, data_type 27 | FROM information_schema.columns 28 | WHERE table_schema = '#{PgEasyReplicate.internal_schema_name}' 29 | AND table_name = 'groups'; 30 | SQL 31 | columns = 32 | PgEasyReplicate::Query.run( 33 | query: columns_sql, 34 | connection_url: connection_url, 35 | schema: PgEasyReplicate.internal_schema_name, 36 | user: "james-bond", 37 | ) 38 | expect(columns).to eq( 39 | [ 40 | { column_name: "id", data_type: "integer" }, 41 | { column_name: "name", data_type: "text" }, 42 | { column_name: "table_names", data_type: "text" }, 43 | { column_name: "schema_name", data_type: "text" }, 44 | { 45 | column_name: "created_at", 46 | data_type: "timestamp without time zone", 47 | }, 48 | { 49 | column_name: "updated_at", 50 | data_type: "timestamp without time zone", 51 | }, 52 | { 53 | column_name: "started_at", 54 | data_type: "timestamp without time zone", 55 | }, 56 | { 57 | column_name: "failed_at", 58 | data_type: "timestamp without time zone", 59 | }, 60 | { column_name: "recreate_indices_post_copy", data_type: "boolean" }, 61 | { 62 | column_name: "switchover_completed_at", 63 | data_type: "timestamp without time zone", 64 | }, 65 | ], 66 | ) 67 | end 68 | end 69 | 70 | describe ".drop" do 71 | before { PgEasyReplicate.bootstrap({ group_name: "cluster1" }) } 72 | 73 | after do 74 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 75 | end 76 | 77 | it "drops the table" do 78 | described_class.setup 79 | described_class.drop 80 | 81 | sql = <<~SQL 82 | SELECT EXISTS ( 83 | SELECT FROM 84 | pg_tables 85 | WHERE 86 | schemaname = '#{PgEasyReplicate.internal_schema_name}' AND 87 | tablename = 'groups' 88 | ); 89 | SQL 90 | r = 91 | PgEasyReplicate::Query.run( 92 | query: sql, 93 | connection_url: connection_url, 94 | schema: PgEasyReplicate.internal_schema_name, 95 | ) 96 | expect(r).to eq([{ exists: false }]) 97 | end 98 | end 99 | 100 | describe ".create" do 101 | before do 102 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 103 | described_class.setup 104 | end 105 | 106 | after do 107 | described_class.drop 108 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 109 | end 110 | 111 | it "adds a row with just the required fields" do 112 | described_class.create({ name: "test" }) 113 | 114 | r = 115 | PgEasyReplicate::Query.run( 116 | query: "select * from groups", 117 | connection_url: connection_url, 118 | schema: PgEasyReplicate.internal_schema_name, 119 | ) 120 | expect(r.first[:name]).to eq("test") 121 | expect(r.first[:recreate_indices_post_copy]).to be_nil 122 | end 123 | 124 | it "adds a row with recreate_indices_post_copy" do 125 | described_class.create({ name: "test", recreate_indices_post_copy: true }) 126 | 127 | r = 128 | PgEasyReplicate::Query.run( 129 | query: "select * from groups", 130 | connection_url: connection_url, 131 | schema: PgEasyReplicate.internal_schema_name, 132 | ) 133 | expect(r.first[:recreate_indices_post_copy]).to be(true) 134 | end 135 | 136 | it "adds a row with table names and schema" do 137 | described_class.create( 138 | { name: "test", table_names: "table1, table2", schema_name: "foo" }, 139 | ) 140 | 141 | r = 142 | PgEasyReplicate::Query.run( 143 | query: "select * from groups", 144 | connection_url: connection_url, 145 | schema: PgEasyReplicate.internal_schema_name, 146 | ) 147 | expect(r.first[:name]).to eq("test") 148 | expect(r.first[:table_names]).to eq("table1, table2") 149 | expect(r.first[:schema_name]).to eq("foo") 150 | end 151 | 152 | it "captures the error" do 153 | expect { described_class.create({}) }.to raise_error( 154 | RuntimeError, 155 | /Adding group entry failed: PG::NotNullViolation: ERROR: null value in column "name"/, 156 | ) 157 | end 158 | end 159 | 160 | describe ".find" do 161 | before do 162 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 163 | described_class.setup 164 | end 165 | 166 | after do 167 | described_class.drop 168 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 169 | end 170 | 171 | it "returns a row" do 172 | described_class.create( 173 | { name: "test", table_names: "table1, table2", schema_name: "foo" }, 174 | ) 175 | expect(described_class.find("test")).to include( 176 | switchover_completed_at: nil, 177 | created_at: kind_of(Time), 178 | name: "test", 179 | schema_name: "foo", 180 | id: kind_of(Integer), 181 | started_at: nil, 182 | updated_at: kind_of(Time), 183 | failed_at: nil, 184 | table_names: "table1, table2", 185 | ) 186 | end 187 | end 188 | 189 | describe ".update" do 190 | before do 191 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 192 | described_class.setup 193 | end 194 | 195 | after do 196 | described_class.drop 197 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 198 | end 199 | 200 | it "updates the started_at and switchover_completed_at successfully" do 201 | described_class.create( 202 | { name: "test", table_names: "table1, table2", schema_name: "foo" }, 203 | ) 204 | 205 | described_class.update( 206 | group_name: "test", 207 | started_at: Time.now, 208 | switchover_completed_at: Time.now, 209 | ) 210 | 211 | expect(described_class.find("test")).to include( 212 | switchover_completed_at: kind_of(Time), 213 | created_at: kind_of(Time), 214 | name: "test", 215 | schema_name: "foo", 216 | id: kind_of(Integer), 217 | started_at: kind_of(Time), 218 | updated_at: kind_of(Time), 219 | table_names: "table1, table2", 220 | ) 221 | end 222 | end 223 | 224 | describe ".delete" do 225 | before do 226 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 227 | described_class.setup 228 | end 229 | 230 | after do 231 | described_class.drop 232 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 233 | end 234 | 235 | it "returns a row" do 236 | described_class.create( 237 | { name: "test", table_names: "table1, table2", schema_name: "foo" }, 238 | ) 239 | described_class.delete("test") 240 | 241 | expect(described_class.find("test")).to be_nil 242 | end 243 | end 244 | end 245 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate/index_manager_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe(PgEasyReplicate::IndexManager) do 6 | describe ".fetch_indices" do 7 | before do 8 | setup_tables 9 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 10 | end 11 | 12 | after do 13 | teardown_tables 14 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 15 | end 16 | 17 | it "fetches index information from the given connection string with no unique & PK indices" do 18 | result = 19 | described_class.fetch_indices( 20 | conn_string: connection_url, 21 | tables: %w[sellers items], 22 | schema: test_schema, 23 | ) 24 | 25 | expect(result).to eq( 26 | [ 27 | { 28 | table_name: "items", 29 | index_name: "items_id_index", 30 | index_definition: 31 | "CREATE INDEX items_id_index ON pger_test.items USING btree (id)", 32 | }, 33 | { 34 | table_name: "items", 35 | index_name: "items_seller_id_index", 36 | index_definition: 37 | "CREATE INDEX items_seller_id_index ON pger_test.items USING btree (seller_id)", 38 | }, 39 | { 40 | table_name: "sellers", 41 | index_name: "sellers_id_index", 42 | index_definition: 43 | "CREATE INDEX sellers_id_index ON pger_test.sellers USING btree (id)", 44 | }, 45 | { 46 | table_name: "sellers", 47 | index_name: "sellers_last_login_index", 48 | index_definition: 49 | "CREATE INDEX sellers_last_login_index ON pger_test.sellers USING btree (last_login)", 50 | }, 51 | ], 52 | ) 53 | end 54 | end 55 | 56 | describe ".drop_indices" do 57 | before do 58 | setup_tables 59 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 60 | end 61 | 62 | after do 63 | teardown_tables 64 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 65 | end 66 | 67 | it "drops non-primary + unique indices from the target database" do 68 | # Ensure index exists 69 | result = 70 | described_class.fetch_indices( 71 | conn_string: target_connection_url, 72 | tables: %w[sellers items], 73 | schema: test_schema, 74 | ) 75 | 76 | expect(result).to eq( 77 | [ 78 | { 79 | table_name: "items", 80 | index_name: "items_id_index", 81 | index_definition: 82 | "CREATE INDEX items_id_index ON pger_test.items USING btree (id)", 83 | }, 84 | { 85 | table_name: "items", 86 | index_name: "items_seller_id_index", 87 | index_definition: 88 | "CREATE INDEX items_seller_id_index ON pger_test.items USING btree (seller_id)", 89 | }, 90 | { 91 | table_name: "sellers", 92 | index_name: "sellers_id_index", 93 | index_definition: 94 | "CREATE INDEX sellers_id_index ON pger_test.sellers USING btree (id)", 95 | }, 96 | { 97 | table_name: "sellers", 98 | index_name: "sellers_last_login_index", 99 | index_definition: 100 | "CREATE INDEX sellers_last_login_index ON pger_test.sellers USING btree (last_login)", 101 | }, 102 | ], 103 | ) 104 | 105 | described_class.drop_indices( 106 | source_conn_string: connection_url, 107 | target_conn_string: target_connection_url, 108 | tables: %w[sellers items], 109 | schema: test_schema, 110 | ) 111 | 112 | result = 113 | described_class.fetch_indices( 114 | conn_string: target_connection_url, 115 | tables: %w[sellers items], 116 | schema: test_schema, 117 | ) 118 | 119 | expect(result).to eq([]) 120 | end 121 | end 122 | 123 | describe ".recreate_indices" do 124 | before do 125 | setup_tables 126 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 127 | end 128 | 129 | after do 130 | teardown_tables 131 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 132 | end 133 | 134 | it "recreates indices on the target database concurrently" do 135 | # Ensure index exists 136 | result = 137 | described_class.fetch_indices( 138 | conn_string: target_connection_url, 139 | tables: %w[sellers items], 140 | schema: test_schema, 141 | ) 142 | 143 | expect(result).to eq( 144 | [ 145 | { 146 | table_name: "items", 147 | index_name: "items_id_index", 148 | index_definition: 149 | "CREATE INDEX items_id_index ON pger_test.items USING btree (id)", 150 | }, 151 | { 152 | table_name: "items", 153 | index_name: "items_seller_id_index", 154 | index_definition: 155 | "CREATE INDEX items_seller_id_index ON pger_test.items USING btree (seller_id)", 156 | }, 157 | { 158 | table_name: "sellers", 159 | index_name: "sellers_id_index", 160 | index_definition: 161 | "CREATE INDEX sellers_id_index ON pger_test.sellers USING btree (id)", 162 | }, 163 | { 164 | table_name: "sellers", 165 | index_name: "sellers_last_login_index", 166 | index_definition: 167 | "CREATE INDEX sellers_last_login_index ON pger_test.sellers USING btree (last_login)", 168 | }, 169 | ], 170 | ) 171 | 172 | described_class.drop_indices( 173 | source_conn_string: connection_url, 174 | target_conn_string: target_connection_url, 175 | tables: %w[sellers items], 176 | schema: test_schema, 177 | ) 178 | 179 | # Ensure index is gone 180 | 181 | result = 182 | described_class.fetch_indices( 183 | conn_string: target_connection_url, 184 | tables: %w[sellers items], 185 | schema: test_schema, 186 | ) 187 | 188 | expect(result).to eq([]) 189 | 190 | described_class.recreate_indices( 191 | source_conn_string: connection_url, 192 | target_conn_string: target_connection_url, 193 | tables: %w[sellers items], 194 | schema: test_schema, 195 | ) 196 | 197 | # Ensure index exists 198 | result = 199 | described_class.fetch_indices( 200 | conn_string: target_connection_url, 201 | tables: %w[sellers items], 202 | schema: test_schema, 203 | ) 204 | 205 | expect(result).to eq( 206 | [ 207 | { 208 | table_name: "items", 209 | index_name: "items_id_index", 210 | index_definition: 211 | "CREATE INDEX items_id_index ON pger_test.items USING btree (id)", 212 | }, 213 | { 214 | table_name: "items", 215 | index_name: "items_seller_id_index", 216 | index_definition: 217 | "CREATE INDEX items_seller_id_index ON pger_test.items USING btree (seller_id)", 218 | }, 219 | { 220 | table_name: "sellers", 221 | index_name: "sellers_id_index", 222 | index_definition: 223 | "CREATE INDEX sellers_id_index ON pger_test.sellers USING btree (id)", 224 | }, 225 | { 226 | table_name: "sellers", 227 | index_name: "sellers_last_login_index", 228 | index_definition: 229 | "CREATE INDEX sellers_last_login_index ON pger_test.sellers USING btree (last_login)", 230 | }, 231 | ], 232 | ) 233 | end 234 | end 235 | 236 | describe ".wait_for_replication_completion" do 237 | it "waits until all tables are replicating" do 238 | allow(PgEasyReplicate::Stats).to receive( 239 | :all_tables_replicating?, 240 | ).and_return(false, false, true) 241 | 242 | expect(described_class).to receive(:sleep).with(5).twice 243 | described_class.wait_for_replication_completion(group_name: "group_name") 244 | 245 | expect(PgEasyReplicate::Stats).to have_received(:all_tables_replicating?) 246 | .with("group_name") 247 | .exactly(3) 248 | .times 249 | end 250 | end 251 | end 252 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate/query_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe(PgEasyReplicate::Query) do 4 | describe ".run" do 5 | before { setup_tables } 6 | 7 | it "runs the query successfully" do 8 | result = 9 | described_class.run( 10 | query: "SELECT 'FooBar' as result", 11 | connection_url: connection_url, 12 | user: "james-bond", 13 | ) 14 | 15 | expect(result).to eq([{ result: "FooBar" }]) 16 | end 17 | 18 | it "sets the statement_timeout" do 19 | result = 20 | described_class.run( 21 | query: "show statement_timeout", 22 | connection_url: connection_url, 23 | user: "james-bond", 24 | ) 25 | 26 | expect(result).to eq([{ statement_timeout: "5s" }]) 27 | end 28 | 29 | it "sets the statement_timeout using env var" do 30 | stub_const("ENV", { "PG_EASY_REPLICATE_STATEMENT_TIMEOUT" => "10s" }) 31 | 32 | result = 33 | described_class.run( 34 | query: "show statement_timeout", 35 | connection_url: connection_url, 36 | user: "james-bond", 37 | ) 38 | 39 | expect(result).to eq([{ statement_timeout: "10s" }]) 40 | end 41 | 42 | it "performs rollback successfully" do 43 | query = "ALTER TABLE sellers DROP COLUMN last_login;" 44 | allow_any_instance_of(Sequel::Postgres::Database).to receive( 45 | :fetch, 46 | ).and_raise(PG::DependentObjectsStillExist) 47 | 48 | expect { 49 | described_class.run( 50 | query: query, 51 | connection_url: connection_url, 52 | schema: "pger_test", 53 | user: "james-bond", 54 | ) 55 | }.to raise_error(PG::DependentObjectsStillExist) 56 | end 57 | 58 | it "performs query with supplied schema successfully" do 59 | expect( 60 | described_class.run( 61 | query: "select * from sellers;", 62 | connection_url: connection_url, 63 | schema: "pger_test", 64 | user: "james-bond", 65 | ).to_a, 66 | ).to eq([]) 67 | end 68 | 69 | it "runs the query successfully with custom user" do 70 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 71 | 72 | result = 73 | described_class.run( 74 | query: "SELECT session_user, current_user;", 75 | connection_url: connection_url, 76 | ) 77 | 78 | expect(result).to eq( 79 | [{ current_user: "pger_su_h1a4fb", session_user: "pger_su_h1a4fb" }], 80 | ) 81 | 82 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate/stats_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe(PgEasyReplicate::Stats) do 4 | describe ".lag_stats" do 5 | before do 6 | setup_tables 7 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 8 | 9 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 10 | PgEasyReplicate::Orchestrate.start_sync( 11 | { group_name: "cluster1", schema_name: test_schema }, 12 | ) 13 | end 14 | 15 | after do 16 | PgEasyReplicate::Orchestrate.stop_sync( 17 | group_name: "cluster1", 18 | source_conn_string: connection_url, 19 | target_conn_string: target_connection_url, 20 | ) 21 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 22 | teardown_tables 23 | end 24 | 25 | it "successfully" do 26 | result = nil 27 | count = 0 28 | # Wait for sync stats to be up 29 | loop do 30 | count += 1 31 | break if count > 5 32 | sleep 1 33 | result = described_class.lag_stats("cluster1").first 34 | end 35 | 36 | expect(result).to include( 37 | application_name: "pger_subscription_cluster1", 38 | client_addr: kind_of(String), 39 | flush_lag: kind_of(BigDecimal), 40 | pid: kind_of(Integer), 41 | replay_lag: kind_of(BigDecimal), 42 | state: kind_of(String), 43 | sync_state: kind_of(String), 44 | user_name: "james-bond", 45 | write_lag: kind_of(BigDecimal), 46 | ) 47 | end 48 | end 49 | 50 | describe ".pg_replication_slots" do 51 | before do 52 | setup_tables 53 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 54 | 55 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 56 | PgEasyReplicate::Orchestrate.start_sync( 57 | { group_name: "cluster1", schema_name: test_schema }, 58 | ) 59 | end 60 | 61 | after do 62 | PgEasyReplicate::Orchestrate.stop_sync( 63 | group_name: "cluster1", 64 | source_conn_string: connection_url, 65 | target_conn_string: target_connection_url, 66 | ) 67 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 68 | teardown_tables 69 | end 70 | 71 | it "successfully" do 72 | result = nil 73 | count = 0 74 | # Wait for sync stats to be up 75 | loop do 76 | count += 1 77 | break if count > 5 78 | sleep 1 79 | result = described_class.pg_replication_slots("cluster1").first 80 | end 81 | 82 | expect(result).to include( 83 | slot_name: "pger_subscription_cluster1", 84 | slot_type: "logical", 85 | ) 86 | end 87 | end 88 | 89 | describe ".replication_stats" do 90 | before do 91 | setup_tables 92 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 93 | 94 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 95 | PgEasyReplicate::Orchestrate.start_sync( 96 | { group_name: "cluster1", schema_name: test_schema }, 97 | ) 98 | end 99 | 100 | after do 101 | PgEasyReplicate::Orchestrate.stop_sync( 102 | group_name: "cluster1", 103 | source_conn_string: connection_url, 104 | target_conn_string: target_connection_url, 105 | ) 106 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 107 | teardown_tables 108 | end 109 | 110 | it "successfully" do 111 | ENV["TARGET_DB_URL"] = target_connection_url 112 | expect(described_class.replication_stats("cluster1")).to include( 113 | { 114 | replication_state: kind_of(String), 115 | subscription_name: "pger_subscription_cluster1", 116 | table_name: "items", 117 | table_schema: "pger_test", 118 | }, 119 | { 120 | replication_state: kind_of(String), 121 | subscription_name: "pger_subscription_cluster1", 122 | table_name: "sellers", 123 | table_schema: "pger_test", 124 | }, 125 | ) 126 | end 127 | end 128 | 129 | describe ".replication_stats_count_by_state" do 130 | before do 131 | setup_tables 132 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 133 | 134 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 135 | PgEasyReplicate::Orchestrate.start_sync( 136 | { group_name: "cluster1", schema_name: test_schema }, 137 | ) 138 | end 139 | 140 | after do 141 | PgEasyReplicate::Orchestrate.stop_sync( 142 | group_name: "cluster1", 143 | source_conn_string: connection_url, 144 | target_conn_string: target_connection_url, 145 | ) 146 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 147 | teardown_tables 148 | end 149 | 150 | it "successfully" do 151 | ENV["TARGET_DB_URL"] = target_connection_url 152 | expect( 153 | described_class.replication_stats_count_by_state( 154 | described_class.replication_stats("cluster1"), 155 | ), 156 | ).to be_a(Hash) 157 | end 158 | end 159 | 160 | describe ".message_lsn_receipts" do 161 | before do 162 | setup_tables 163 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 164 | 165 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 166 | PgEasyReplicate::Orchestrate.start_sync( 167 | { group_name: "cluster1", schema_name: test_schema }, 168 | ) 169 | end 170 | 171 | after do 172 | PgEasyReplicate::Orchestrate.stop_sync( 173 | group_name: "cluster1", 174 | source_conn_string: connection_url, 175 | target_conn_string: target_connection_url, 176 | ) 177 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 178 | teardown_tables 179 | end 180 | 181 | it "successfully" do 182 | ENV["TARGET_DB_URL"] = target_connection_url 183 | expect(described_class.message_lsn_receipts("cluster1").first.keys).to eq( 184 | %i[ 185 | received_lsn 186 | last_msg_send_time 187 | last_msg_receipt_time 188 | latest_end_lsn 189 | latest_end_time 190 | ], 191 | ) 192 | end 193 | end 194 | 195 | describe ".object" do 196 | before do 197 | setup_tables 198 | PgEasyReplicate.bootstrap({ group_name: "cluster1" }) 199 | 200 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 201 | PgEasyReplicate::Orchestrate.start_sync( 202 | { group_name: "cluster1", schema_name: test_schema }, 203 | ) 204 | end 205 | 206 | after do 207 | PgEasyReplicate::Orchestrate.stop_sync( 208 | group_name: "cluster1", 209 | source_conn_string: connection_url, 210 | target_conn_string: target_connection_url, 211 | ) 212 | PgEasyReplicate.cleanup({ everything: true, group_name: "cluster1" }) 213 | teardown_tables 214 | end 215 | 216 | it "successfully" do 217 | ENV["TARGET_DB_URL"] = target_connection_url 218 | expect(described_class.object("cluster1").keys).to eq( 219 | %i[ 220 | lag_stats 221 | replication_slots 222 | replication_stats 223 | replication_stats_count_by_state 224 | message_lsn_receipts 225 | sync_started_at 226 | sync_failed_at 227 | switchover_completed_at 228 | ], 229 | ) 230 | end 231 | end 232 | 233 | describe ".notify" do 234 | before do 235 | @mocked_stats = { 236 | lag_stats: [], 237 | replication_slots: [], 238 | replication_stats: [], 239 | replication_stats_count_by_state: {}, 240 | message_lsn_receipts: [], 241 | sync_started_at: Time.now - 5, 242 | sync_failed_at: nil, 243 | switchover_completed_at: Time.now, 244 | } 245 | allow(described_class).to receive(:object).with("cluster1").and_return(@mocked_stats) 246 | 247 | # # mocks the http request 248 | http_double = double('http', request: double('response', code: '200', message: 'OK')) 249 | allow(Net::HTTP).to receive(:new).and_return(http_double) 250 | allow(http_double).to receive(:use_ssl=) 251 | allow(http_double).to receive(:open_timeout=) 252 | allow(http_double).to receive(:read_timeout=) 253 | end 254 | 255 | it "logs notification success and indicates switchover completion" do 256 | thread = Thread.new do 257 | expect { described_class.notify("cluster1", "https://example.com/webhook", 1, 5) } 258 | .to output(/Notification sent: 200 OK/).to_stdout 259 | end 260 | 261 | sleep(0.1) 262 | thread.kill # Break out of the loop so the test doesnt hang 263 | 264 | expect { thread.join }.not_to raise_error 265 | end 266 | 267 | it "retries on failure" do 268 | allow(Net::HTTP).to receive(:new).and_raise(StandardError.new("network error")) 269 | 270 | expect { 271 | described_class.notify("cluster1", "https://example.com/webhook", 1, 1) 272 | }.to raise_error(StandardError, /Notify failed with: network error/) 273 | end 274 | end 275 | end 276 | -------------------------------------------------------------------------------- /spec/pg_easy_replicate_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe(PgEasyReplicate) do 4 | it "has a version number" do 5 | expect(PgEasyReplicate::VERSION).not_to be_nil 6 | end 7 | 8 | describe ".config" do 9 | before { setup_tables } 10 | 11 | after { teardown_tables } 12 | 13 | it "returns the config for both databases" do 14 | result = described_class.config(schema_name: test_schema) 15 | expect(result).to eq( 16 | { 17 | source_db_is_super_user: true, 18 | target_db_is_super_user: true, 19 | source_db: [ 20 | { name: "max_logical_replication_workers", setting: "4" }, 21 | { name: "max_replication_slots", setting: "10" }, 22 | { name: "max_wal_senders", setting: "10" }, 23 | { name: "max_worker_processes", setting: "8" }, 24 | { name: "wal_level", setting: "logical" }, 25 | ], 26 | tables_have_replica_identity: true, 27 | target_db: [ 28 | { name: "max_logical_replication_workers", setting: "4" }, 29 | { name: "max_replication_slots", setting: "10" }, 30 | { name: "max_wal_senders", setting: "10" }, 31 | { name: "max_worker_processes", setting: "8" }, 32 | { name: "wal_level", setting: "logical" }, 33 | ], 34 | pg_dump_exists: true, 35 | }, 36 | ) 37 | end 38 | 39 | describe ".assert_config" do 40 | let(:source_db_config_without_logical) do 41 | { 42 | source_db: [{ name: "wal_level", setting: "replication" }], 43 | target_db: [{ name: "wal_level", setting: "logical" }], 44 | } 45 | end 46 | 47 | let(:target_db_config_without_logical) do 48 | { 49 | source_db: [{ name: "wal_level", setting: "logical" }], 50 | target_db: [{ name: "wal_level", setting: "replication" }], 51 | } 52 | end 53 | 54 | it "raises wal level not being logical error for source db" do 55 | allow(described_class).to receive(:config).and_return( 56 | source_db_config_without_logical, 57 | ) 58 | expect { described_class.assert_config }.to raise_error( 59 | "WAL_LEVEL should be LOGICAL on source DB", 60 | ) 61 | end 62 | 63 | it "raises wal level not being logical error for target db" do 64 | allow(described_class).to receive(:config).and_return( 65 | target_db_config_without_logical, 66 | ) 67 | expect { described_class.assert_config }.to raise_error( 68 | "WAL_LEVEL should be LOGICAL on target DB", 69 | ) 70 | end 71 | 72 | it "raises when user is not superuser on source db" do 73 | allow(described_class).to receive(:config).and_return( 74 | { 75 | source_db_is_super_user: false, 76 | target_db: [{ name: "wal_level", setting: "logical" }], 77 | source_db: [{ name: "wal_level", setting: "logical" }], 78 | }, 79 | ) 80 | expect { described_class.assert_config }.to raise_error( 81 | "User on source database does not have super user privilege", 82 | ) 83 | end 84 | 85 | it "raises when user is not superuser on target db" do 86 | allow(described_class).to receive(:config).and_return( 87 | { 88 | source_db_is_super_user: true, 89 | target_db_is_super_user: false, 90 | target_db: [{ name: "wal_level", setting: "logical" }], 91 | source_db: [{ name: "wal_level", setting: "logical" }], 92 | tables_have_replica_identity: true, 93 | }, 94 | ) 95 | expect { described_class.assert_config }.to raise_error( 96 | "User on target database does not have super user privilege", 97 | ) 98 | end 99 | 100 | it "raises error when copy schema is present and pg_dump is not" do 101 | allow(described_class).to receive(:config).and_return( 102 | { 103 | source_db_is_super_user: true, 104 | target_db_is_super_user: true, 105 | target_db: [{ name: "wal_level", setting: "logical" }], 106 | source_db: [{ name: "wal_level", setting: "logical" }], 107 | pg_dump_exists: false, 108 | }, 109 | ) 110 | 111 | expect { 112 | described_class.assert_config(copy_schema: true) 113 | }.to raise_error("pg_dump must exist if copy_schema (-c) is passed") 114 | end 115 | 116 | it "raises error when tables don't have replicat identity" do 117 | allow(described_class).to receive(:config).and_return( 118 | { 119 | source_db_is_super_user: true, 120 | target_db_is_super_user: true, 121 | target_db: [{ name: "wal_level", setting: "logical" }], 122 | source_db: [{ name: "wal_level", setting: "logical" }], 123 | tables_have_replica_identity: false, 124 | }, 125 | ) 126 | expect { described_class.assert_config }.to raise_error( 127 | /Ensure all tables involved in logical replication have an appropriate replica identity/, 128 | ) 129 | end 130 | 131 | it "raises error when table is provided but schema isn't" do 132 | allow(described_class).to receive(:config).and_return( 133 | { 134 | source_db_is_super_user: true, 135 | target_db_is_super_user: true, 136 | target_db: [{ name: "wal_level", setting: "logical" }], 137 | source_db: [{ name: "wal_level", setting: "logical" }], 138 | tables_have_replica_identity: true, 139 | }, 140 | ) 141 | expect { 142 | described_class.assert_config(tables: "items") 143 | }.to raise_error(/Schema name is required if tables are passed/) 144 | end 145 | end 146 | 147 | describe ".is_super_user?" do 148 | before { setup_roles } 149 | after { cleanup_roles } 150 | 151 | it "returns true" do 152 | expect(described_class.send(:is_super_user?, connection_url)).to be( 153 | true, 154 | ) 155 | end 156 | 157 | it "returns true with non primary user" do 158 | expect( 159 | described_class.send( 160 | :is_super_user?, 161 | connection_url("james-bond_sup"), 162 | ), 163 | ).to be(true) 164 | end 165 | 166 | it "returns false" do 167 | expect( 168 | described_class.send(:is_super_user?, connection_url("no_sup")), 169 | ).to be(false) 170 | end 171 | 172 | it "returns true if user is part of the special user role" do 173 | expect( 174 | described_class.send( 175 | :is_super_user?, 176 | connection_url("james-bond_role_regular"), 177 | "james-bond_super_role", 178 | ), 179 | ).to be(true) 180 | end 181 | end 182 | 183 | describe ".setup_internal_schema" do 184 | it "sets up the schema" do 185 | described_class.setup_internal_schema 186 | 187 | expect(get_schema).to eq([{ schema_name: "pger" }]) 188 | end 189 | end 190 | 191 | describe ".drop_internal_schema" do 192 | before { described_class.bootstrap({ group_name: "cluster1" }) } 193 | 194 | after do 195 | described_class.cleanup({ everything: true, group_name: "cluster1" }) 196 | end 197 | 198 | it "drops up the schema" do 199 | described_class.setup_internal_schema 200 | described_class.drop_internal_schema 201 | 202 | expect(get_schema).to eq([]) 203 | end 204 | end 205 | 206 | describe ".drop_user" do 207 | it "drops the user" do 208 | described_class.create_user(conn_string: connection_url) 209 | 210 | expect(described_class.user_exists?(conn_string: connection_url)).to be( 211 | true, 212 | ) 213 | 214 | described_class.drop_user(conn_string: connection_url) 215 | 216 | expect(described_class.user_exists?(conn_string: connection_url)).to be( 217 | false, 218 | ) 219 | end 220 | end 221 | 222 | describe ".bootstrap" do 223 | before { setup_tables("james-bond", setup_target_db: false) } 224 | 225 | after do 226 | described_class.cleanup({ everything: true, group_name: "cluster1" }) 227 | teardown_tables 228 | `rm #{PgEasyReplicate::SCHEMA_FILE_LOCATION}` 229 | end 230 | 231 | it "successfully with everything" do 232 | described_class.bootstrap({ group_name: "cluster1" }) 233 | 234 | # Check schema exists 235 | expect(get_schema).to eq([{ schema_name: "pger" }]) 236 | 237 | # Check table exists 238 | expect(groups_table_exists?).to eq([{ table_name: "groups" }]) 239 | 240 | # Check user on source database 241 | expect( 242 | user_permissions( 243 | connection_url: connection_url, 244 | group_name: "cluster1", 245 | ), 246 | ).to eq( 247 | [ 248 | { 249 | rolcanlogin: true, 250 | rolcreatedb: true, 251 | rolcreaterole: true, 252 | rolsuper: true, 253 | }, 254 | ], 255 | ) 256 | 257 | # Check user exists on target database 258 | expect( 259 | user_permissions( 260 | connection_url: target_connection_url, 261 | group_name: "cluster1", 262 | ), 263 | ).to eq( 264 | [ 265 | { 266 | rolcanlogin: true, 267 | rolcreatedb: true, 268 | rolcreaterole: true, 269 | rolsuper: true, 270 | }, 271 | ], 272 | ) 273 | end 274 | 275 | it "successfully with copy_schema" do 276 | described_class.bootstrap({ group_name: "cluster1", copy_schema: true }) 277 | 278 | # Check schema exists 279 | expect(get_schema).to eq([{ schema_name: "pger" }]) 280 | 281 | # Check table exists 282 | expect(groups_table_exists?).to eq([{ table_name: "groups" }]) 283 | 284 | # Check user on source database 285 | expect( 286 | user_permissions( 287 | connection_url: connection_url, 288 | group_name: "cluster1", 289 | ), 290 | ).to eq( 291 | [ 292 | { 293 | rolcanlogin: true, 294 | rolcreatedb: true, 295 | rolcreaterole: true, 296 | rolsuper: true, 297 | }, 298 | ], 299 | ) 300 | 301 | # Check user exists on target database 302 | expect( 303 | user_permissions( 304 | connection_url: target_connection_url, 305 | group_name: "cluster1", 306 | ), 307 | ).to eq( 308 | [ 309 | { 310 | rolcanlogin: true, 311 | rolcreatedb: true, 312 | rolcreaterole: true, 313 | rolsuper: true, 314 | }, 315 | ], 316 | ) 317 | 318 | conn = 319 | PgEasyReplicate::Query.connect( 320 | connection_url: target_connection_url, 321 | schema: test_schema, 322 | user: "james-bond", 323 | ) 324 | expect(conn.fetch("SELECT * FROM items").to_a).to eq([]) 325 | end 326 | 327 | it "is idempotent and doesn't provision groups and user again" do 328 | described_class.bootstrap({ group_name: "cluster1" }) 329 | 330 | # Capture initial state 331 | initial_schema = get_schema 332 | initial_groups_table = groups_table_exists? 333 | initial_source_user_permissions = 334 | user_permissions( 335 | connection_url: connection_url, 336 | group_name: "cluster1", 337 | ) 338 | initial_target_user_permissions = 339 | user_permissions( 340 | connection_url: target_connection_url, 341 | group_name: "cluster1", 342 | ) 343 | 344 | # Second bootstrap 345 | described_class.bootstrap({ group_name: "cluster1" }) 346 | 347 | # Check that nothing has changed 348 | expect(get_schema).to eq(initial_schema) 349 | expect(groups_table_exists?).to eq(initial_groups_table) 350 | expect( 351 | user_permissions( 352 | connection_url: connection_url, 353 | group_name: "cluster1", 354 | ), 355 | ).to eq(initial_source_user_permissions) 356 | expect( 357 | user_permissions( 358 | connection_url: target_connection_url, 359 | group_name: "cluster1", 360 | ), 361 | ).to eq(initial_target_user_permissions) 362 | end 363 | end 364 | 365 | describe ".cleanup" do 366 | it "successfully with everything" do 367 | described_class.bootstrap({ group_name: "cluster1" }) 368 | ENV["SECONDARY_SOURCE_DB_URL"] = docker_compose_source_connection_url 369 | PgEasyReplicate::Orchestrate.start_sync({ group_name: "cluster1" }) 370 | described_class.cleanup({ everything: true, group_name: "cluster1" }) 371 | 372 | # Check schema exists 373 | expect(get_schema).to eq([]) 374 | 375 | # Check table exists 376 | expect(groups_table_exists?).to eq([]) 377 | 378 | # Check user on source database 379 | expect( 380 | user_permissions( 381 | connection_url: connection_url, 382 | group_name: "cluster1", 383 | ), 384 | ).to eq([]) 385 | 386 | # Check user exists on target database 387 | expect( 388 | user_permissions( 389 | connection_url: target_connection_url, 390 | group_name: "cluster1", 391 | ), 392 | ).to eq([]) 393 | 394 | expect(pg_publications(connection_url: connection_url)).to eq([]) 395 | expect(pg_subscriptions(connection_url: target_connection_url)).to eq( 396 | [], 397 | ) 398 | end 399 | end 400 | 401 | describe ".export_schema" do 402 | before { setup_tables } 403 | 404 | after do 405 | teardown_tables 406 | `rm #{PgEasyReplicate::SCHEMA_FILE_LOCATION}` 407 | end 408 | 409 | it "succesfully" do 410 | described_class.export_schema(conn_string: connection_url) 411 | file_contents = File.read(PgEasyReplicate::SCHEMA_FILE_LOCATION) 412 | expect(file_contents).to match(/PostgreSQL database dump complete/) 413 | end 414 | 415 | it "raises error" do 416 | expect { 417 | described_class.export_schema(conn_string: "postgres://foo@bar") 418 | }.to raise_error( 419 | /Unable to export schema: pg_dump: error: could not translate host name "bar"/, 420 | ) 421 | end 422 | end 423 | 424 | describe ".import_schema" do 425 | before { setup_tables("james-bond", setup_target_db: false) } 426 | 427 | after do 428 | teardown_tables 429 | `rm #{PgEasyReplicate::SCHEMA_FILE_LOCATION}` 430 | end 431 | 432 | it "succesfully" do 433 | described_class.export_schema(conn_string: connection_url) 434 | described_class.import_schema(conn_string: target_connection_url) 435 | 436 | conn = 437 | PgEasyReplicate::Query.connect( 438 | connection_url: target_connection_url, 439 | schema: test_schema, 440 | user: "james-bond", 441 | ) 442 | expect(conn.fetch("SELECT * FROM items").to_a).to eq([]) 443 | end 444 | 445 | it "raises error" do 446 | expect { 447 | described_class.import_schema(conn_string: "postgres://foo@bar") 448 | }.to raise_error( 449 | /Unable to import schema: psql: error: could not translate host name "bar"/, 450 | ) 451 | end 452 | end 453 | 454 | describe ".excluding tables" do 455 | before { setup_tables } 456 | after { teardown_tables } 457 | 458 | tables = "items" 459 | 460 | it "returns error if tables and exclude_tables specified tables are both specified" do 461 | expect { 462 | described_class.config( 463 | tables: tables, 464 | exclude_tables: tables, 465 | schema_name: test_schema, 466 | ) 467 | }.to raise_error(RuntimeError) 468 | expect { 469 | described_class.assert_config( 470 | tables: tables, 471 | exclude_tables: tables, 472 | schema_name: test_schema, 473 | ) 474 | }.to raise_error(RuntimeError) 475 | end 476 | 477 | it "doesnt return error if only exclude_tables specified tables are both specified" do 478 | allow(described_class).to receive(:config).and_return( 479 | { 480 | source_db_is_super_user: true, 481 | target_db_is_super_user: true, 482 | target_db: [{ name: "wal_level", setting: "logical" }], 483 | source_db: [{ name: "wal_level", setting: "logical" }], 484 | tables_have_replica_identity: true, 485 | }, 486 | ) 487 | expect { 488 | described_class.config( 489 | exclude_tables: tables, 490 | schema_name: test_schema, 491 | ) 492 | }.not_to raise_error 493 | expect { 494 | described_class.assert_config( 495 | exclude_tables: tables, 496 | schema_name: test_schema, 497 | ) 498 | }.not_to raise_error 499 | end 500 | end 501 | end 502 | end 503 | -------------------------------------------------------------------------------- /spec/smoke_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe("SmokeSpec") do 4 | describe "dataset" do 5 | it "matches after switchover and applies DDL changes" do 6 | # Bootstrap 7 | system("./scripts/e2e-bootstrap.sh") 8 | expect($CHILD_STATUS.success?).to be(true) 9 | 10 | r = 11 | PgEasyReplicate::Query.run( 12 | query: "select count(*) from pgbench_accounts", 13 | connection_url: connection_url, 14 | user: "james-bond", 15 | ) 16 | expect(r).to eq([{ count: 500_000 }]) 17 | last_count = r.last[:count] 18 | 19 | pid = 20 | fork do 21 | puts("Running insertions") 22 | 3000.times do |i| 23 | new_aid = last_count + (i + 1) 24 | sql = <<~SQL 25 | INSERT INTO "public"."pgbench_accounts"("aid", "bid", "abalance", "filler") VALUES(#{new_aid}, 1, 0, '0') RETURNING "aid", "bid", "abalance", "filler"; 26 | SQL 27 | PgEasyReplicate::Query.run( 28 | query: sql, 29 | connection_url: connection_url, 30 | user: "james-bond", 31 | ) 32 | rescue => e 33 | if e.message.include?( 34 | "cannot execute INSERT in a read-only transaction", 35 | ) || e.message.include?("terminating connection") 36 | PgEasyReplicate::Query.run( 37 | query: sql, 38 | connection_url: target_connection_url, 39 | user: "james-bond", 40 | ) 41 | else 42 | raise 43 | end 44 | end 45 | end 46 | Process.detach(pid) 47 | 48 | system("./scripts/e2e-start.sh") 49 | expect($CHILD_STATUS.success?).to be(true) 50 | 51 | begin 52 | Process.wait(pid) 53 | rescue Errno::ECHILD #rubocop:disable Lint/SuppressedException 54 | end 55 | 56 | r = 57 | PgEasyReplicate::Query.run( 58 | query: "select count(*) from pgbench_accounts", 59 | connection_url: target_connection_url, 60 | user: "james-bond", 61 | ) 62 | expect(r).to eq([{ count: 503_000 }]) 63 | 64 | columns = 65 | PgEasyReplicate::Query.run( 66 | query: 67 | "SELECT column_name FROM information_schema.columns WHERE table_name = 'pgbench_accounts' AND column_name = 'test_column'", 68 | connection_url: target_connection_url, 69 | user: "james-bond", 70 | ) 71 | expect(columns).to eq([{ column_name: "test_column" }]) 72 | 73 | expect( 74 | vacuum_stats(url: target_connection_url, schema: "public"), 75 | ).to include( 76 | { 77 | last_analyze: kind_of(Time), 78 | last_vacuum: kind_of(Time), 79 | relname: "pgbench_tellers", 80 | }, 81 | { 82 | last_analyze: kind_of(Time), 83 | last_vacuum: kind_of(Time), 84 | relname: "pgbench_history", 85 | }, 86 | { 87 | last_analyze: kind_of(Time), 88 | last_vacuum: kind_of(Time), 89 | relname: "pgbench_branches", 90 | }, 91 | { 92 | last_analyze: kind_of(Time), 93 | last_vacuum: kind_of(Time), 94 | relname: "pgbench_accounts", 95 | }, 96 | ) 97 | ensure 98 | begin 99 | Process.kill("KILL", pid) if pid 100 | rescue Errno::ESRCH 101 | puts("proc closed") 102 | end 103 | end 104 | end 105 | end 106 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "pg_easy_replicate" 4 | require "./spec/database_helpers" 5 | require "pry" 6 | 7 | ENV["RACK_ENV"] = "test" 8 | ENV["DEBUG"] = "true" 9 | 10 | RSpec.configure do |config| 11 | # Enable flags like --only-failures and --next-failure 12 | config.example_status_persistence_file_path = ".rspec_status" 13 | 14 | # Disable RSpec exposing methods globally on `Module` and `main` 15 | config.disable_monkey_patching! 16 | 17 | config.expect_with(:rspec) { |c| c.syntax = :expect } 18 | 19 | config.include(DatabaseHelpers) 20 | config.before(:suite) { DatabaseHelpers.populate_env_vars } 21 | config.after(:suite) do 22 | PgEasyReplicate.drop_internal_schema 23 | rescue StandardError # rubocop:disable Lint/SuppressedException 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@prettier/plugin-ruby@^3.2.2": 6 | version "3.2.2" 7 | resolved "https://registry.yarnpkg.com/@prettier/plugin-ruby/-/plugin-ruby-3.2.2.tgz#43c9d85349032f74d34c4f57e6a77487d5c5bdc1" 8 | integrity sha512-Vc7jVE39Fgswl517ET4kPtpnoRWE6XTi1Sivd84rZyomYnHYUmvUsEeoOf6tVhzTuIXE5XVQB1YCG2hulrwR3Q== 9 | dependencies: 10 | prettier ">=2.3.0" 11 | 12 | prettier@>=2.3.0, prettier@^2.8.8: 13 | version "2.8.8" 14 | resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" 15 | integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== 16 | --------------------------------------------------------------------------------