├── .github ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── .readthedocs.yaml ├── CHANGELOG.rst ├── CODE_OF_CONDUCT.md ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── RELEASE_NOTES.rst ├── docs ├── Makefile ├── _static │ └── .placeholder ├── changelog.rst ├── conf.py ├── configuration_file.rst ├── global_lib.rst ├── index.rst ├── make.bat ├── mysql_lib.rst ├── pg_lib.rst ├── readme.rst ├── release_notes.rst ├── requirements.txt ├── sql_util.rst ├── upgrade_procedure.rst └── usage.rst ├── images ├── pgchameleon.png └── pgchameleon.svg ├── parse.py ├── pg_chameleon ├── __init__.py ├── configuration │ └── config-example.yml ├── lib │ ├── __init__.py │ ├── global_lib.py │ ├── mysql_lib.py │ ├── pg_lib.py │ └── sql_util.py └── sql │ ├── create_schema.sql │ ├── dev │ ├── custom_aggregate.sql │ ├── fn_parse_json.sql │ ├── fn_parse_json_2.sql │ ├── fn_process_batch.sql │ ├── fn_replay_mysql.sql │ ├── fn_replay_mysql_v3.sql │ └── fn_replay_mysql_v3_stream.sql │ ├── drop_schema.sql │ ├── fn_process_batch.sql │ ├── fn_replay_data.sql │ ├── get_fkeys.sql │ ├── scratch.sql │ ├── scratch3.sql │ ├── tests │ ├── 01_create.sql │ ├── 02_insert.sql │ ├── 03_update.sql │ ├── 04_delete.sql │ ├── 05_drop.sql │ ├── 06_create_broken.sql │ ├── 07_insert_broken.sql │ ├── 08_drop_broken.sql │ ├── 09_alter.sql │ ├── 10_insert_expanded.sql │ ├── fn_load_data.sql │ ├── get_default_val.sql │ ├── test.sql │ ├── test_pkeyless.sql │ └── wrong_log_position.sql │ └── upgrade │ ├── .placeholder │ ├── 200_to_201.sql │ ├── 201_to_202.sql │ ├── 202_to_203.sql │ ├── 203_to_204.sql │ ├── 204_to_205.sql │ ├── 205_to_206.sql │ ├── 206_to_207.sql │ ├── 207_to_208.sql │ ├── 208_to_209.sql │ └── 209_to_2010.sql ├── scripts ├── chameleon └── chameleon.py ├── setup.cfg ├── setup.py ├── test_logical_decoding.py └── tests ├── install_mysql.sh ├── my5.5.cnf ├── my5.6.cnf ├── my5.7.cnf ├── setup_db.sh ├── setup_mysql.sql ├── test.sh ├── test.yml └── test_tokenizer.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: the4thdoctor 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: the4thdoctor 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Screenshots** 20 | If applicable, add screenshots to help explain your problem. 21 | 22 | **Environment(please complete the following information):** 23 | - OS: [e.g. Devuan GNU/Linux] 24 | - MySQL Version[e.g. 5.7.27] 25 | - PostgreSQL Version[e.g. 11] 26 | - Python Version [e.g. 3.5] 27 | - Cloud hosted database [e.g PostgreSQL on AWS RDS and MySQL on Google Cloud] 28 | 29 | **Additional context** 30 | Add any other context about the problem here. 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | lib64 19 | 20 | # Installer logs 21 | pip-log.txt 22 | 23 | # Unit test / coverage reports 24 | .coverage 25 | .tox 26 | nosetests.xml 27 | 28 | # Translations 29 | *.mo 30 | 31 | # Mr Developer 32 | .mr.developer.cfg 33 | .project 34 | .pydevproject 35 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.9" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | 17 | # Build documentation in the "docs/" directory with Sphinx 18 | sphinx: 19 | configuration: docs/conf.py 20 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 21 | # builder: "dirhtml" 22 | # Fail on all warnings to avoid broken references 23 | # fail_on_warning: true 24 | 25 | # Optionally build your docs in additional formats such as PDF and ePub 26 | formats: 27 | - pdf 28 | - epub 29 | 30 | # Optional but recommended, declare the Python requirements required 31 | # to build your documentation 32 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 33 | python: 34 | install: 35 | - requirements: docs/requirements.txt 36 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | changelog 2 | ************************* 3 | 4 | 2.0.21 - 21 January 2025 5 | .......................................................... 6 | * PR #163 provided by @bukem providing an optimization of the procedure for applying changes to Postgresql 7 | * Issue #170 add check for existing replica schema and display an hint instead of an exception 8 | * Fix incorrect placement of the new parameter net_read_timeout. Now it's set as an instance variable from global_lib.py 9 | 10 | 2.0.20 - 01 January 2025 11 | .......................................................... 12 | * Merge the SQL library improvements built by @nikochiko for the `Google Summer of Code 2023 `_ 13 | * Fix setup.py for newer python versions as per patch provided by @darix in Issue #172 14 | * Merge PR #169 provided by @Jamal-B Fix read and replay daemons death detection 15 | * Merge PR #171 provided by @JasonLiu1567 fix issue #111 16 | * Merge PR #173 provided by @acarapetis Ignore MySQL indices with prefix key parts 17 | * DEPRECATION of rollbar support 18 | 19 | 2.0.19 - 25 March 2023 20 | .......................................................... 21 | * Merge pull request #144, mysql-replication support for PyMySQL>0.10.0 was introduced in v0.22 22 | * add support for fillfactor when running init_replica 23 | * improve logging on discarded rows 24 | * add distinct on group concat when collecting foreign keys 25 | * use mysql-replication>=0.31, fix for crash when replicating from MariaDB 26 | 27 | 2.0.18 - 31 March 2022 28 | .......................................................... 29 | * Support the ON DELETE and ON UPDATE clause when creating the foreign keys in PostgreSQL 30 | * change logic for index and foreign key names by managing only duplicates within same schema 31 | * use mysql-replication<0.27 as new versions crash when receiving queries 32 | * add copy_schema method for copying only the schema without data (EXPERIMENTAL) 33 | * change type for identifiers in replica schema to varchar(64) 34 | 35 | 2.0.17 - 30 January 2022 36 | .......................................................... 37 | * Remove argparse from the requirements 38 | * Add the collect for unique constraints when keep_existing_schema is **Yes** 39 | * Fix wrong order in copy data/create indices when keep_existing_schema is **No** 40 | * Remove check for log_bin we are replicating from Aurora MySQL 41 | * Manage different the different behaviour in pyyaml to allow pg_chameleon to be installed as rpm in centos 7 42 | 43 | 2.0.16 - 23 September 2020 44 | .......................................................... 45 | * Fix for issue #126 init_replica failure with tables on transactional engine and invalid data 46 | 47 | 2.0.15 - 20 September 2020 48 | .......................................................... 49 | * Support for reduced lock if MySQL engine is transactional, thanks to @rascalDan 50 | * setup.py now requires python-mysql-replication to version 0.22 which adds support for PyMySQL >=0.10.0 51 | * removed PyMySQL requirement <0.10.0 from setup.py 52 | * prevent pg_chameleon to run as root 53 | 54 | 2.0.14 - 26 July 2020 55 | .......................................................... 56 | * Add support for spatial data types (requires postgis installed on the target database) 57 | * When ``keep_existing_schema`` is set to ``yes`` now drops and recreates indices, and constraints during the ``init_replica`` process 58 | * Fix for issue #115 thanks to @porshkevich 59 | * setup.py now forces PyMySQL to version <0.10.0 because it breaks the python-mysql-replication library (issue #117) 60 | 61 | 2.0.13 - 05 July 2020 62 | .......................................................... 63 | * **EXPERIMENTAL** support for Point datatype - @jovankricka-everon 64 | * Add ``keep_existing_schema`` in MySQL source type to keep the existing scema in place instead of rebuilding it from the mysql source 65 | * Change tabs to spaces in code 66 | 67 | 2.0.12 - 11 Dec 2019 68 | .......................................................... 69 | * Fixes for issue #96 thanks to @daniel-qcode 70 | * Change for configuration and SQL files location 71 | * Package can build now as source and wheel 72 | * The minimum python requirements now is 3.5 73 | 74 | 2.0.11 - 25 Oct 2019 75 | .......................................................... 76 | * Fix wrong formatting for yaml example files. @rebtoor 77 | * Make start_replica run in foreground when log_file == stdout . @clifff 78 | * Travis seems to break down constantly, Disable the CI until a fix is found. Evaluate to use a different CI. 79 | * Add the add loader to yaml.load as required by the new PyYAML version. 80 | 81 | 2.0.10 - 01 Sep 2018 82 | .......................................................... 83 | * Fix regression in new replay function with PostgreSQL 10 84 | * Convert to string the dictionary entries pulled from a json field 85 | * Let ``enable_replica`` to disable any leftover maintenance flag 86 | * Add capture in CHANGE for tables in the form schema.table 87 | 88 | 2.0.9 - 19 Aug 2018 89 | .......................................................... 90 | * Fix wrong check for the next auto maintenance run if the maintenance wasn't run before 91 | * Improve the replay function's speed 92 | * Remove blocking from the GTID operational mode 93 | 94 | 95 | 2.0.8 - 14 Jul 2018 96 | .......................................................... 97 | * Add support for skip events as requested in issue #76. Is now possible to skip events (insert,delete,update) for single tables or for entire schemas. 98 | * **EXPERIMENTAL** support for the GTID. When configured on MySQL or Percona server pg_chameleon will use the GTID to auto position the replica stream. Mariadb is not supported by this change. 99 | * ALTER TABLE RENAME is now correctly parsed and executed 100 | * Add horrible hack to ALTER TABLE MODIFY. Previously modify with default values would parse wrongly and fail when translating to PostgreSQL dialect 101 | * Disable erroring the source when running with ``--debug`` switch enabled 102 | * Add cleanup for logged events when refreshing schema and syncing tables. previously spurious logged events could lead to primary key violations when syncing single tables or refreshing single schemas. 103 | 104 | 105 | 2.0.7 - 19 May 2018 106 | .......................................................... 107 | * Fix for issue #71, make the multiprocess logging safe. Now each replica process logs in a separate file 108 | * Fix the ``--full`` option to store true instead of false. Previously the option had no effect. 109 | * Add `auto_maintenance` optional parameter to trigger a vacuum over the log tables after a specific timeout 110 | * Fix for issue #75, avoid the wrong conversion to string for None keys when cleaning up malformed rows during the init replica and replica process 111 | * Fix for issue #73, fix for wrong data type tokenisation when an alter table adds a column with options (e.g. ``ADD COLUMN foo DEFAULT NULL``) 112 | * Fix wrong TRUNCATE TABLE tokenisation if the statement specifies the table with the schema. 113 | 114 | 2.0.6 - 29 April 2018 115 | .......................................................... 116 | * fix for issue #69 add source's optional parameter ``on_error_read:`` to allow the read process to continue in case of connection issues with the source database (e.g. MySQL in maintenance) 117 | * remove the detach partition during the maintenance process as this proved to be a very fragile approach 118 | * add switch ``--full`` to run a ``VACUUM FULL`` during the maintenance 119 | * when running the maintentenance execute a ``VACUUM`` instead of a ``VACUUM FULL`` 120 | * fix for issue #68. fallback to ``binlog_row_image=FULL`` if the parameter is missing in mysql 5.5. 121 | * add cleanup for default value ``NOW()`` when adding a new column with ``ALTER TABLE`` 122 | * allow ``enable_replica`` to reset the source status in the case of a catalogue version mismatch 123 | 124 | 2.0.5 - 25 March 2018 125 | .......................................................... 126 | * fix wrong exclusion when running sync_tables with limit_tables set 127 | * add `run_maintenance` command to perform a VACUUM FULL on the source's log tables 128 | * add `stop_all_replicas` command to stop all the running sources within the target postgresql database 129 | 130 | 2.0.4 - 04 March 2018 131 | .......................................................... 132 | * Fix regression added in 2.0.3 when handling MODIFY DDL 133 | * Improved handling of dropped columns during the replica 134 | 135 | 136 | 2.0.3 - 11 February 2018 137 | .......................................................... 138 | 139 | * fix regression added by commit 8c09ccb. when ALTER TABLE ADD COLUMN is in the form datatype DEFAULT (NOT) NULL the parser captures two words instead of one 140 | * Improve the speed of the cleanup on startup deleting only for the source's log tables instead of the parent table 141 | * fix for issue #63. change the field i_binlog_position to bigint in order to avoid an integer overflow error when the binlog is largher than 2 GB. 142 | * change to psycopg2-binary in install_requires. This change will ensure the psycopg2 will install using the wheel package when available. 143 | * add upgrade_catalogue_v20 for minor schema upgrades 144 | 145 | 2.0.2 - 21 January 2018 146 | .......................................................... 147 | * Fix for issue #61, missing post replay cleanup for processed batches. 148 | * add private method ``_swap_enums`` to the class ``pg_engine`` which moves the enumerated types from the loading to the destination schema. 149 | 150 | 2.0.1 - 14 January 2018 151 | .......................................................... 152 | * Fix for issue #58. Improve the read replica performance by filtering the row images when ``limit_tables/skip_tables`` are set. 153 | * Make the ``read_replica_stream`` method private. 154 | * Fix read replica crash if in alter table a column was defined as ``character varying`` 155 | 156 | 2.0.0 - 01 January 2018 157 | .......................................................... 158 | * Add option ``--rollbar-level`` to set the maximum level for the messages to be sent to rollbar. Accepted values: "critical", "error", "warning", "info". The Default is "info". 159 | * Add command ``enable_replica`` used to reset the replica status in case of error or unespected crash 160 | * Add script alias ``chameleon`` along with ``chameleon.py`` 161 | 162 | 2.0.0.rc1 - 24 December 2017 163 | .......................................................... 164 | * Fix for issue #52, When adding a unique key the table's creation fails because of the NULLable field 165 | * Add check for the MySQL configuration when initialising or refreshing replicated entities 166 | * Add class rollbar_notifier for simpler message management 167 | * Add end of init_replica,refresh_schema,sync_tables notification to rollbar 168 | * Allow ``--tables disabled`` when syncing the tables to re synchronise all the tables excluded from the replica 169 | 170 | 2.0.0.beta1 - 10 December 2017 171 | .......................................................... 172 | * fix a race condition where an unrelated DDL can cause the collected binlog rows to be added several times to the log_table 173 | * fix regression in write ddl caused by the change of private method 174 | * fix wrong ddl parsing when a column definition is surrounded by parentheses e.g. ``ALTER TABLE foo ADD COLUMN(bar varchar(30));`` 175 | * error handling for wrong table names, wrong schema names, wrong source name and wrong commands 176 | * init_replica for source pgsql now can read from an hot standby but the copy is not consistent 177 | * init_replica for source pgsql adds "replicated tables" for better show_status display 178 | * check if the source is registered when running commands that require a source name 179 | 180 | 2.0.0.alpha3 - 03 December 2017 181 | .......................................................... 182 | * Remove limit_tables from binlogreader initialisation, as we can read from multiple schemas we should only exclude the tables not limit 183 | * Fix wrong formatting for default value when altering a field 184 | * Add upgrade procedure from version 1.8.2 to 2.0 185 | * Improve error logging and table exclusion in replay function 186 | * Add stack trace capture to the rollbar and log message when one of the replica daemon crash 187 | * Add ``on_error_replay`` to set whether the replay process should skip the tables or exit on error 188 | * Add init_replica support for source type pgsql (EXPERIMENTAL) 189 | 190 | 191 | 2.0.0.alpha2 - 18 November 2017 192 | .......................................................... 193 | * Fix wrong position when determining the destination schema in read_replica_stream 194 | * Fix wrong log position stored in the source's high watermark 195 | * Fix wrong table inclusion/exclusion in read_replica_steam 196 | * Add source parameter ``replay_max_rows`` to set the amount of rows to replay. Previously the value was set by ``replica_batch_size`` 197 | * Fix crash when an alter table affected a table not replicated 198 | * Fixed issue with alter table during the drop/set default for the column (thanks to psycopg2's sql.Identifier) 199 | * add type display to source status 200 | * Add fix for issue #33 cleanup NUL markers from the rows before trying to insert them in PostgreSQL 201 | * Fix broken save_discarded_row 202 | * Add more detail to show_status when specifying the source with --source 203 | * Changed some methods to private 204 | * ensure the match for the alter table's commands are enclosed by word boundaries 205 | * add if exists when trying to drop the table in swap tables. previously adding a new table failed because the table wasn't there 206 | * fix wrong drop enum type when adding a new field 207 | * add log error for storing the errors generated during the replay 208 | * add not functional class pgsql_source for source type pgsql 209 | * allow ``type_override`` to be empty 210 | * add show_status command for displaying the log error entries 211 | * add separate logs for per source 212 | * change log line formatting inspired by the super clean look in pgbackrest (thanks you guys) 213 | 214 | 2.0.0.alpha1 - 11 November 2017 215 | .......................................................... 216 | 217 | * Python 3 only development 218 | * Add support for reading from multiple MySQL schemas and restore them it into a target PostgreSQL database. The source and target schema names can be different. 219 | * Conservative approach to the replica. Tables which generate errors are automatically excluded from the replica. 220 | * Daemonised init_replica process. 221 | * Daemonised replica process with two separated subprocess, one for the read and one for the replay. 222 | * Soft replica initialisation. The tables are locked when needed and stored with their log coordinates. The replica damon will put the database in a consistent status gradually. 223 | * Rollbar integration for a simpler error detection. 224 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at 4thdoctor.gallifrey@gmail.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2025 Federico Campoli 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include pg_chameleon/configuration/config-example.yml 2 | include pg_chameleon/sql/create_schema.sql 3 | include pg_chameleon/sql/drop_schema.sql 4 | include pg_chameleon/sql/upgrade/*.sql 5 | include LICENSE.txt 6 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://img.shields.io/github/issues/the4thdoctor/pg_chameleon.svg 2 | :target: https://github.com/the4thdoctor/pg_chameleon/issues 3 | 4 | .. image:: https://img.shields.io/github/forks/the4thdoctor/pg_chameleon.svg 5 | :target: https://github.com/the4thdoctor/pg_chameleon/network 6 | 7 | .. image:: https://img.shields.io/github/stars/the4thdoctor/pg_chameleon.svg 8 | :target: https://github.com/the4thdoctor/pg_chameleon/stargazers 9 | 10 | .. image:: https://img.shields.io/badge/license-BSD-blue.svg 11 | :target: https://raw.githubusercontent.com/the4thdoctor/pg_chameleon/main/LICENSE.txt 12 | 13 | .. image:: https://img.shields.io/github/release/the4thdoctor/pg_chameleon 14 | :target: https://github.com/the4thdoctor/pg_chameleon/releases 15 | 16 | .. image:: https://img.shields.io/pypi/dm/pg_chameleon.svg 17 | :target: https://pypi.org/project/pg_chameleon 18 | 19 | 20 | pg_chameleon is a MySQL to PostgreSQL replica system written in Python 3. 21 | The system use the library mysql-replication to pull the row images from MySQL which are stored into PostgreSQL as JSONB. 22 | A pl/pgsql function decodes the jsonb values and replays the changes against the PostgreSQL database. 23 | 24 | pg_chameleon 2.0 `is available on pypi `_ 25 | 26 | The documentation `is available on read the docs `_ 27 | 28 | Please submit your `bug reports on GitHub `_. 29 | 30 | 31 | Requirements 32 | ****************** 33 | 34 | Replica host 35 | .............................. 36 | 37 | Operating system: Linux, FreeBSD, OpenBSD 38 | Python: CPython 3.7+ 39 | 40 | * `PyMySQL `_ 41 | * `mysql-replication `_ 42 | * `psycopg2 `_ 43 | * `PyYAML `_ 44 | * `tabulate `_ 45 | * `rollbar `_ 46 | * `daemonize `_ 47 | * `sphinx `_ 48 | * `sphinx-autobuild `_ 49 | 50 | 51 | Origin database 52 | ................................. 53 | 54 | MySQL 5.5+ 55 | 56 | Aurora MySQL 5.7+ 57 | 58 | Destination database 59 | .............................. 60 | 61 | PostgreSQL 9.5+ 62 | 63 | Example scenarios 64 | .............................. 65 | 66 | * Analytics 67 | * Migrations 68 | * Data aggregation from multiple MySQL databases 69 | 70 | Features 71 | .............................. 72 | 73 | * Read from multiple MySQL schemas and restore them it into a target PostgreSQL database. The source and target schema names can be different. 74 | * Setup PostgreSQL to act as a MySQL replica. 75 | * Support for enumerated and binary data types. 76 | * Basic DDL Support (CREATE/DROP/ALTER TABLE, DROP PRIMARY KEY/TRUNCATE, RENAME). 77 | * Discard of rubbish data coming from the replica. 78 | * Conservative approach to the replica. Tables which generate errors are automatically excluded from the replica. 79 | * Possibilty to refresh single tables or single schemas. 80 | * Basic replica monitoring. 81 | * Detach replica from MySQL for migration support. 82 | * Data type override (e.g. tinyint(1) to boolean) 83 | * Daemonised init_replica process. 84 | * Daemonised replica process with two separated subprocess, one for the read and one for the replay. 85 | * **DEPRECATED** Rollbar integration 86 | * Support for geometrical data. **Requires PostGIS on the target database.** 87 | * Minimal locking during init_replica for transactional engines (e.g. innodb) 88 | 89 | 90 | 91 | 92 | 93 | Caveats 94 | .............................. 95 | The replica requires the tables to have a primary or unique key. Tables without primary/unique key are initialised during the init_replica process but not replicated. 96 | 97 | The copy_max_memory is just an estimate. The average rows size is extracted from mysql's informations schema and can be outdated. 98 | If the copy process fails for memory error check the failing table's row length and the number of rows for each slice. 99 | 100 | Python 3 is supported only from version 3.7 as required by the parser library *parsy*. 101 | 102 | The lag is determined using the last received event timestamp and the postgresql timestamp. If the mysql is read only the lag will increase because 103 | no replica event is coming in. 104 | 105 | The detach replica process resets the sequences in postgres to let the database work standalone. The foreign keys from the source MySQL schema are extracted and created initially as NOT VALID. The foreign keys are created without the ON DELETE or ON UPDATE clauses. 106 | A second run tries to validate the foreign keys. If an error occurs it gets logged out according to the source configuration. 107 | 108 | 109 | 110 | Setup 111 | ***************** 112 | 113 | RPM PGDG 114 | .............................. 115 | 116 | pg_chameleon is included in the PGDG RMP repository thanks to Devrim. 117 | 118 | Please follow the instructions on `https://www.postgresql.org/download/linux/redhat/ `_ 119 | 120 | openSUSE Build Service 121 | .............................. 122 | 123 | pg_chameleon is available on the `openSUSE build Service `_ 124 | 125 | Currently all releases are supported except SLE_12_SP5 because of unresolved dependencies. 126 | 127 | Virtual env setup 128 | .............................. 129 | 130 | * Create a virtual environment (e.g. python3 -m venv venv) 131 | * Activate the virtual environment (e.g. source venv/bin/activate) 132 | * Upgrade pip with **pip install pip --upgrade** 133 | * Install pg_chameleon with **pip install pg_chameleon**. 134 | * Create a user on mysql for the replica (e.g. usr_replica) 135 | * Grant access to usr on the replicated database (e.g. GRANT ALL ON sakila.* TO 'usr_replica';) 136 | * Grant RELOAD privilege to the user (e.g. GRANT RELOAD ON \*.\* to 'usr_replica';) 137 | * Grant REPLICATION CLIENT privilege to the user (e.g. GRANT REPLICATION CLIENT ON \*.\* to 'usr_replica';) 138 | * Grant REPLICATION SLAVE privilege to the user (e.g. GRANT REPLICATION SLAVE ON \*.\* to 'usr_replica';) 139 | 140 | 141 | 142 | Configuration directory 143 | ******************************** 144 | The system wide install is now supported correctly. 145 | 146 | The configuration is set with the command ``chameleon set_configuration_files`` in $HOME/.pg_chameleon . 147 | Inside the directory there are three subdirectories. 148 | 149 | 150 | * configuration is where the configuration files are stored. 151 | * pid is where the replica pid file is created. it can be changed in the configuration file 152 | * logs is where the replica logs are saved if log_dest is file. It can be changed in the configuration file 153 | 154 | You should use config-example.yaml as template for the other configuration files. 155 | Check the `configuration file reference `_ for an overview. 156 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @echo " html to make standalone HTML files" 21 | @echo " dirhtml to make HTML files named index.html in directories" 22 | @echo " singlehtml to make a single large HTML file" 23 | @echo " pickle to make pickle files" 24 | @echo " json to make JSON files" 25 | @echo " htmlhelp to make HTML files and a HTML help project" 26 | @echo " qthelp to make HTML files and a qthelp project" 27 | @echo " applehelp to make an Apple Help Book" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " epub3 to make an epub3" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " xml to make Docutils-native XML files" 41 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | @echo " coverage to run coverage check of the documentation (if enabled)" 45 | @echo " dummy to check syntax errors of document sources" 46 | 47 | .PHONY: clean 48 | clean: 49 | rm -rf $(BUILDDIR)/* 50 | 51 | .PHONY: html 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | .PHONY: dirhtml 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | .PHONY: singlehtml 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | .PHONY: pickle 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | .PHONY: json 76 | json: 77 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 78 | @echo 79 | @echo "Build finished; now you can process the JSON files." 80 | 81 | .PHONY: htmlhelp 82 | htmlhelp: 83 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 84 | @echo 85 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 86 | ".hhp project file in $(BUILDDIR)/htmlhelp." 87 | 88 | .PHONY: qthelp 89 | qthelp: 90 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 91 | @echo 92 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 93 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 94 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pg_chameleon.qhcp" 95 | @echo "To view the help file:" 96 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pg_chameleon.qhc" 97 | 98 | .PHONY: applehelp 99 | applehelp: 100 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 101 | @echo 102 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 103 | @echo "N.B. You won't be able to view it unless you put it in" \ 104 | "~/Library/Documentation/Help or install it in your application" \ 105 | "bundle." 106 | 107 | .PHONY: devhelp 108 | devhelp: 109 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 110 | @echo 111 | @echo "Build finished." 112 | @echo "To view the help file:" 113 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pg_chameleon" 114 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pg_chameleon" 115 | @echo "# devhelp" 116 | 117 | .PHONY: epub 118 | epub: 119 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 120 | @echo 121 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 122 | 123 | .PHONY: epub3 124 | epub3: 125 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 126 | @echo 127 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 128 | 129 | .PHONY: latex 130 | latex: 131 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 132 | @echo 133 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 134 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 135 | "(use \`make latexpdf' here to do that automatically)." 136 | 137 | .PHONY: latexpdf 138 | latexpdf: 139 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 140 | @echo "Running LaTeX files through pdflatex..." 141 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 142 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 143 | 144 | .PHONY: latexpdfja 145 | latexpdfja: 146 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 147 | @echo "Running LaTeX files through platex and dvipdfmx..." 148 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 149 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 150 | 151 | .PHONY: text 152 | text: 153 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 154 | @echo 155 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 156 | 157 | .PHONY: man 158 | man: 159 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 160 | @echo 161 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 162 | 163 | .PHONY: texinfo 164 | texinfo: 165 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 166 | @echo 167 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 168 | @echo "Run \`make' in that directory to run these through makeinfo" \ 169 | "(use \`make info' here to do that automatically)." 170 | 171 | .PHONY: info 172 | info: 173 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 174 | @echo "Running Texinfo files through makeinfo..." 175 | make -C $(BUILDDIR)/texinfo info 176 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 177 | 178 | .PHONY: gettext 179 | gettext: 180 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 181 | @echo 182 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 183 | 184 | .PHONY: changes 185 | changes: 186 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 187 | @echo 188 | @echo "The overview file is in $(BUILDDIR)/changes." 189 | 190 | .PHONY: linkcheck 191 | linkcheck: 192 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 193 | @echo 194 | @echo "Link check complete; look for any errors in the above output " \ 195 | "or in $(BUILDDIR)/linkcheck/output.txt." 196 | 197 | .PHONY: doctest 198 | doctest: 199 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 200 | @echo "Testing of doctests in the sources finished, look at the " \ 201 | "results in $(BUILDDIR)/doctest/output.txt." 202 | 203 | .PHONY: coverage 204 | coverage: 205 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 206 | @echo "Testing of coverage in the sources finished, look at the " \ 207 | "results in $(BUILDDIR)/coverage/python.txt." 208 | 209 | .PHONY: xml 210 | xml: 211 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 212 | @echo 213 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 214 | 215 | .PHONY: pseudoxml 216 | pseudoxml: 217 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 218 | @echo 219 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 220 | 221 | .PHONY: dummy 222 | dummy: 223 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 224 | @echo 225 | @echo "Build finished. Dummy builder generates no files." 226 | -------------------------------------------------------------------------------- /docs/_static/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/the4thdoctor/pg_chameleon/5458575165565b4593d33af912e7fdbeb4db49f8/docs/_static/.placeholder -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CHANGELOG.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # pg_chameleon documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Oct 9 17:08:30 2016. 5 | # 6 | # This file is execfile()d with the current directory set to its 7 | # containing dir. 8 | # 9 | # Note that not all possible configuration values are present in this 10 | # autogenerated file. 11 | # 12 | # All configuration values have a default; values that are commented out 13 | # serve to show the default. 14 | 15 | # If extensions (or modules to document with autodoc) are in another directory, 16 | # add these directories to sys.path here. If the directory is relative to the 17 | # documentation root, use os.path.abspath to make it absolute, like shown here. 18 | # 19 | import os 20 | import sys 21 | sys.path.insert(0, os.path.abspath('.')) 22 | sys.path.insert(0, os.path.abspath('.')) 23 | sys.path.insert(0, os.path.abspath('..')) 24 | sys.path.insert(0, os.path.abspath('../pg_chameleon')) 25 | sys.path.insert(0, os.path.abspath('../pg_chameleon/lib')) 26 | 27 | # -- General configuration ------------------------------------------------ 28 | 29 | # If your documentation needs a minimal Sphinx version, state it here. 30 | # 31 | # needs_sphinx = '1.0' 32 | 33 | # Add any Sphinx extension module names here, as strings. They can be 34 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 35 | # ones. 36 | extensions = [ 37 | 'sphinx.ext.autodoc', 38 | 'sphinx.ext.doctest', 39 | 'sphinx.ext.viewcode', 40 | ] 41 | 42 | # Add any paths that contain templates here, relative to this directory. 43 | templates_path = ['_templates'] 44 | 45 | # The suffix(es) of source filenames. 46 | # You can specify multiple suffix as a list of string: 47 | # 48 | # source_suffix = ['.rst', '.md'] 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | # 53 | # source_encoding = 'utf-8-sig' 54 | 55 | # The master toctree document. 56 | master_doc = 'index' 57 | 58 | # General information about the project. 59 | project = u'pg_chameleon' 60 | copyright = u'2016-2025 Federico Campoli' 61 | author = u'Federico Campoli' 62 | 63 | # The version info for the project you're documenting, acts as replacement for 64 | # |version| and |release|, also used in various other places throughout the 65 | # built documents. 66 | # 67 | # The short X.Y version. 68 | version = u'2.0' 69 | # The full version, including alpha/beta/rc tags. 70 | release = u'v2.0.21' 71 | 72 | # The language for content autgenerated by Sphinx. Refer to documentation 73 | # for a list of supported languages. 74 | # 75 | # This is also used if you do content translation via gettext catalogs. 76 | # Usually you set "language" from the command line for these cases. 77 | language = "en" 78 | 79 | # There are two options for replacing |today|: either, you set today to some 80 | # non-false value, then it is used: 81 | # 82 | # today = '' 83 | # 84 | # Else, today_fmt is used as the format for a strftime call. 85 | # 86 | # today_fmt = '%B %d, %Y' 87 | 88 | # List of patterns, relative to source directory, that match files and 89 | # directories to ignore when looking for source files. 90 | # This patterns also effect to html_static_path and html_extra_path 91 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 92 | 93 | # The reST default role (used for this markup: `text`) to use for all 94 | # documents. 95 | # 96 | # default_role = None 97 | 98 | # If true, '()' will be appended to :func: etc. cross-reference text. 99 | # 100 | # add_function_parentheses = True 101 | 102 | # If true, the current module name will be prepended to all description 103 | # unit titles (such as .. function::). 104 | # 105 | # add_module_names = True 106 | 107 | # If true, sectionauthor and moduleauthor directives will be shown in the 108 | # output. They are ignored by default. 109 | # 110 | # show_authors = False 111 | 112 | # The name of the Pygments (syntax highlighting) style to use. 113 | pygments_style = 'sphinx' 114 | 115 | # A list of ignored prefixes for module index sorting. 116 | # modindex_common_prefix = [] 117 | 118 | # If true, keep warnings as "system message" paragraphs in the built documents. 119 | # keep_warnings = False 120 | 121 | # If true, `todo` and `todoList` produce output, else they produce nothing. 122 | todo_include_todos = False 123 | 124 | 125 | # -- Options for HTML output ---------------------------------------------- 126 | 127 | # The theme to use for HTML and HTML Help pages. See the documentation for 128 | # a list of builtin themes. 129 | # 130 | html_theme = 'classic' 131 | 132 | # Theme options are theme-specific and customize the look and feel of a theme 133 | # further. For a list of options available for each theme, see the 134 | # documentation. 135 | # 136 | # html_theme_options = {} 137 | 138 | # Add any paths that contain custom themes here, relative to this directory. 139 | # html_theme_path = [] 140 | 141 | # The name for this set of Sphinx documents. 142 | # " v documentation" by default. 143 | # 144 | # html_title = u'pg_chameleon v0.1 DEVEL' 145 | 146 | # A shorter title for the navigation bar. Default is the same as html_title. 147 | # 148 | # html_short_title = None 149 | 150 | # The name of an image file (relative to this directory) to place at the top 151 | # of the sidebar. 152 | # 153 | # html_logo = None 154 | 155 | # The name of an image file (relative to this directory) to use as a favicon of 156 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 157 | # pixels large. 158 | # 159 | # html_favicon = None 160 | 161 | # Add any paths that contain custom static files (such as style sheets) here, 162 | # relative to this directory. They are copied after the builtin static files, 163 | # so a file named "default.css" will overwrite the builtin "default.css". 164 | html_static_path = ['_static'] 165 | 166 | # Add any extra paths that contain custom files (such as robots.txt or 167 | # .htaccess) here, relative to this directory. These files are copied 168 | # directly to the root of the documentation. 169 | # 170 | # html_extra_path = [] 171 | 172 | # If not None, a 'Last updated on:' timestamp is inserted at every page 173 | # bottom, using the given strftime format. 174 | # The empty string is equivalent to '%b %d, %Y'. 175 | # 176 | # html_last_updated_fmt = None 177 | 178 | # If true, SmartyPants will be used to convert quotes and dashes to 179 | # typographically correct entities. 180 | # 181 | # html_use_smartypants = True 182 | 183 | # Custom sidebar templates, maps document names to template names. 184 | # 185 | # html_sidebars = {} 186 | 187 | # Additional templates that should be rendered to pages, maps page names to 188 | # template names. 189 | # 190 | # html_additional_pages = {} 191 | 192 | # If false, no module index is generated. 193 | # 194 | # html_domain_indices = True 195 | 196 | # If false, no index is generated. 197 | # 198 | # html_use_index = True 199 | 200 | # If true, the index is split into individual pages for each letter. 201 | # 202 | # html_split_index = False 203 | 204 | # If true, links to the reST sources are added to the pages. 205 | # 206 | # html_show_sourcelink = True 207 | 208 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 209 | # 210 | # html_show_sphinx = True 211 | 212 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 213 | # 214 | # html_show_copyright = True 215 | 216 | # If true, an OpenSearch description file will be output, and all pages will 217 | # contain a tag referring to it. The value of this option must be the 218 | # base URL from which the finished HTML is served. 219 | # 220 | # html_use_opensearch = '' 221 | 222 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 223 | # html_file_suffix = None 224 | 225 | # Language to be used for generating the HTML full-text search index. 226 | # Sphinx supports the following languages: 227 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' 228 | # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' 229 | # 230 | # html_search_language = 'en' 231 | 232 | # A dictionary with options for the search language support, empty by default. 233 | # 'ja' uses this config value. 234 | # 'zh' user can custom change `jieba` dictionary path. 235 | # 236 | # html_search_options = {'type': 'default'} 237 | 238 | # The name of a javascript file (relative to the configuration directory) that 239 | # implements a search results scorer. If empty, the default will be used. 240 | # 241 | # html_search_scorer = 'scorer.js' 242 | 243 | # Output file base name for HTML help builder. 244 | htmlhelp_basename = 'pg_chameleondoc' 245 | 246 | # -- Options for LaTeX output --------------------------------------------- 247 | 248 | latex_elements = { 249 | # The paper size ('letterpaper' or 'a4paper'). 250 | # 251 | # 'papersize': 'letterpaper', 252 | 253 | # The font size ('10pt', '11pt' or '12pt'). 254 | # 255 | # 'pointsize': '10pt', 256 | 257 | # Additional stuff for the LaTeX preamble. 258 | # 259 | # 'preamble': '', 260 | 261 | # Latex figure (float) alignment 262 | # 263 | # 'figure_align': 'htbp', 264 | } 265 | 266 | # Grouping the document tree into LaTeX files. List of tuples 267 | # (source start file, target name, title, 268 | # author, documentclass [howto, manual, or own class]). 269 | latex_documents = [ 270 | (master_doc, 'pg_chameleon.tex', u'pg\\_chameleon Documentation', 271 | u'Federico Campoli', 'manual'), 272 | ] 273 | 274 | # The name of an image file (relative to this directory) to place at the top of 275 | # the title page. 276 | # 277 | # latex_logo = None 278 | 279 | # For "manual" documents, if this is true, then toplevel headings are parts, 280 | # not chapters. 281 | # 282 | # latex_use_parts = False 283 | 284 | # If true, show page references after internal links. 285 | # 286 | # latex_show_pagerefs = False 287 | 288 | # If true, show URL addresses after external links. 289 | # 290 | # latex_show_urls = False 291 | 292 | # Documents to append as an appendix to all manuals. 293 | # 294 | # latex_appendices = [] 295 | 296 | # It false, will not define \strong, \code, itleref, \crossref ... but only 297 | # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added 298 | # packages. 299 | # 300 | # latex_keep_old_macro_names = True 301 | 302 | # If false, no module index is generated. 303 | # 304 | # latex_domain_indices = True 305 | 306 | 307 | # -- Options for manual page output --------------------------------------- 308 | 309 | # One entry per manual page. List of tuples 310 | # (source start file, name, description, authors, manual section). 311 | man_pages = [ 312 | (master_doc, 'pg_chameleon', u'pg_chameleon Documentation', 313 | [author], 1) 314 | ] 315 | 316 | # If true, show URL addresses after external links. 317 | # 318 | # man_show_urls = False 319 | 320 | 321 | # -- Options for Texinfo output ------------------------------------------- 322 | 323 | # Grouping the document tree into Texinfo files. List of tuples 324 | # (source start file, target name, title, author, 325 | # dir menu entry, description, category) 326 | texinfo_documents = [ 327 | (master_doc, 'pg_chameleon', u'pg_chameleon Documentation', 328 | author, 'pg_chameleon', 'MySQL to PostgreSQL replica', 329 | 'Database'), 330 | ] 331 | 332 | # Documents to append as an appendix to all manuals. 333 | # 334 | # texinfo_appendices = [] 335 | 336 | # If false, no module index is generated. 337 | # 338 | # texinfo_domain_indices = True 339 | 340 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 341 | # 342 | # texinfo_show_urls = 'footnote' 343 | 344 | # If true, do not generate a @detailmenu in the "Top" node's menu. 345 | # 346 | # texinfo_no_detailmenu = False 347 | 348 | 349 | # -- Options for Epub output ---------------------------------------------- 350 | 351 | # Bibliographic Dublin Core info. 352 | epub_title = project 353 | epub_author = author 354 | epub_publisher = author 355 | epub_copyright = copyright 356 | 357 | # The basename for the epub file. It defaults to the project name. 358 | # epub_basename = project 359 | 360 | # The HTML theme for the epub output. Since the default themes are not 361 | # optimized for small screen space, using the same theme for HTML and epub 362 | # output is usually not wise. This defaults to 'epub', a theme designed to save 363 | # visual space. 364 | # 365 | # epub_theme = 'epub' 366 | 367 | # The language of the text. It defaults to the language option 368 | # or 'en' if the language is not set. 369 | # 370 | # epub_language = '' 371 | 372 | # The scheme of the identifier. Typical schemes are ISBN or URL. 373 | # epub_scheme = '' 374 | 375 | # The unique identifier of the text. This can be a ISBN number 376 | # or the project homepage. 377 | # 378 | # epub_identifier = '' 379 | 380 | # A unique identification for the text. 381 | # 382 | # epub_uid = '' 383 | 384 | # A tuple containing the cover image and cover page html template filenames. 385 | # 386 | # epub_cover = () 387 | 388 | # A sequence of (type, uri, title) tuples for the guide element of content.opf. 389 | # 390 | # epub_guide = () 391 | 392 | # HTML files that should be inserted before the pages created by sphinx. 393 | # The format is a list of tuples containing the path and title. 394 | # 395 | # epub_pre_files = [] 396 | 397 | # HTML files that should be inserted after the pages created by sphinx. 398 | # The format is a list of tuples containing the path and title. 399 | # 400 | # epub_post_files = [] 401 | 402 | # A list of files that should not be packed into the epub file. 403 | epub_exclude_files = ['search.html'] 404 | 405 | # The depth of the table of contents in toc.ncx. 406 | # 407 | # epub_tocdepth = 3 408 | 409 | # Allow duplicate toc entries. 410 | # 411 | # epub_tocdup = True 412 | 413 | # Choose between 'default' and 'includehidden'. 414 | # 415 | # epub_tocscope = 'default' 416 | 417 | # Fix unsupported image types using the Pillow. 418 | # 419 | # epub_fix_images = False 420 | 421 | # Scale large images. 422 | # 423 | # epub_max_image_width = 0 424 | 425 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 426 | # 427 | # epub_show_urls = 'inline' 428 | 429 | # If false, no index is generated. 430 | # 431 | # epub_use_index = True 432 | -------------------------------------------------------------------------------- /docs/configuration_file.rst: -------------------------------------------------------------------------------- 1 | The configuration file 2 | ******************************** 3 | 4 | The file config-example.yaml is stored in **~/.pg_chameleon/configuration** and should be used as template for the other configuration files. 5 | The configuration consists of three configuration groups. 6 | 7 | Global settings 8 | .............................. 9 | 10 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 11 | :language: yaml 12 | :lines: 2-9 13 | :linenos: 14 | 15 | * pid_dir directory where the process pids are saved. 16 | * log_dir directory where the logs are stored. 17 | * log_dest log destination. stdout for debugging purposes, file for the normal activity. 18 | * log_level logging verbosity. allowed values are debug, info, warning, error. 19 | * log_days_keep configure the retention in days for the daily rotate replica logs. 20 | * rollbar_key: the optional rollbar key 21 | * rollbar_env: the optional rollbar environment 22 | 23 | If both rollbar_key and rollbar_env are configured some messages are sent to the rollbar conf 24 | 25 | type override 26 | ............................................... 27 | 28 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 29 | :language: yaml 30 | :lines: 11-16 31 | :linenos: 32 | 33 | The type_override allows the user to override the default type conversion into a different one. 34 | Each type key should be named exactly like the mysql type to override including the dimensions. 35 | Each type key needs two subkeys. 36 | 37 | * override_to specifies the destination type which must be a postgresql type and the type cast should be possible 38 | * override_tables is a yaml list which specifies to which tables the override applies. If the first list item is set to "*" then the override is applied to all tables in the replicated schemas. 39 | 40 | The override is applied when running the init_replica,refresh_schema andsync_tables process. 41 | The override is also applied for each matching DDL (create table/alter table) if the table name matches the override_tables values. 42 | 43 | 44 | PostgreSQL target connection 45 | ............................................... 46 | 47 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 48 | :language: yaml 49 | :lines: 21-28 50 | :linenos: 51 | 52 | The pg_conn key maps the target database connection string. 53 | 54 | 55 | sources configuration 56 | ............................................... 57 | 58 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 59 | :language: yaml 60 | :lines: 30-95 61 | :linenos: 62 | 63 | The key sources allow to setup multiple replica sources writing on the same postgresql database. 64 | The key name myst be unique within the replica configuration. 65 | 66 | 67 | The following remarks apply only to the mysql source type. 68 | 69 | For the postgresql source type. See the last section for the description and the limitations. 70 | 71 | Database connection 72 | ============================= 73 | 74 | 75 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 76 | :language: yaml 77 | :lines: 30-68 78 | :emphasize-lines: 3-9 79 | :linenos: 80 | 81 | The db_conn key maps the target database connection string. Within the connection is possible to configure the connect_timeout which is 10 seconds by default. 82 | Larger values could help the tool working better on slow networks. Low values can cause the connection to fail before any action is performed. 83 | 84 | Schema mappings 85 | ============================= 86 | 87 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 88 | :language: yaml 89 | :lines: 30-68 90 | :emphasize-lines: 10-11 91 | :linenos: 92 | 93 | The key schema mappings is a dictionary. Each key is a MySQL database that needs to be replicated in PostgreSQL. Each value is the destination schema in the PostgreSQL database. 94 | In the example provided the MySQL database ``delphis_mediterranea`` is replicated into the schema ``loxodonta_africana`` stored in the database specified in the pg_conn key (db_replica). 95 | 96 | Limit and skip tables 97 | ============================= 98 | 99 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 100 | :language: yaml 101 | :lines: 30-68 102 | :emphasize-lines: 12-15 103 | :linenos: 104 | 105 | * limit_tables list with the tables to replicate. If the list is empty then the entire mysql database is replicated. 106 | * skip_tables list with the tables to exclude from the replica. 107 | 108 | The table's names should be in the form SCHEMA_NAME.TABLE_NAME. 109 | 110 | Grant select to option 111 | ============================================================= 112 | 113 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 114 | :language: yaml 115 | :lines: 30-68 116 | :emphasize-lines: 16-17 117 | :linenos: 118 | 119 | This key allows to specify a list of database roles which will get select access on the replicate tables. 120 | 121 | 122 | 123 | 124 | 125 | 126 | Source configuration parameters 127 | ==================================== 128 | 129 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 130 | :language: yaml 131 | :lines: 30-68 132 | :emphasize-lines: 18-31 133 | :linenos: 134 | 135 | * lock_timeout the max time in seconds that the target postgresql connections should wait for acquiring a lock. This parameter applies to init_replica,refresh_schema and sync_tables when performing the relation's swap. 136 | * my_server_id the server id for the mysql replica. must be unique within the replica cluster 137 | * replica_batch_size the max number of rows that are pulled from the mysql replica before a write on the postgresql database is performed. See caveats in README for a complete explanation. 138 | * batch_retention the max retention for the replayed batches rows in t_replica_batch. The field accepts any valid interval accepted by PostgreSQL 139 | * copy_max_memory the max amount of memory to use when copying the table in PostgreSQL. Is possible to specify the value in (k)ilobytes, (M)egabytes, (G)igabytes adding the suffix (e.g. 300M). 140 | * copy_mode the allowed values are ‘file’ and ‘direct’. With direct the copy happens on the fly. With file the table is first dumped in a csv file then reloaded in PostgreSQL. 141 | * out_dir the directory where the csv files are dumped during the init_replica process if the copy mode is file. 142 | * sleep_loop seconds between a two replica batches. 143 | * on_error_replay specifies whether the replay process should ``exit`` or ``continue`` if any error during the replay happens. If ``continue`` is specified the offending tables are removed from the replica. 144 | * on_error_read specifies whether the read process should ``exit`` or ``continue`` if a connection error during the read process happens. If ``continue`` is specified the process emits a warning and waits for the connection to come back. If the parameter is omitted the default is ``exit`` which cause the replica process to stop with error. 145 | * auto_maintenance specifies the timeout after an automatic maintenance is triggered. The parameter accepts values valid for the `PostgreSQL interval data type `_ (e.g. ``1 day``). If the value is set to ``disabled`` the automatic maintenance doesn't run. If the parameter is omitted the default is ``disabled``. 146 | * gtid_enable **(EXPERIMENTAL)** Specifies whether to use the gtid to auto position the replica stream. This parameter have effect only on MySQL and only if the server is configured with the GTID. 147 | * type specifies the source database type. The system supports ``mysql`` or ``pgsql``. See below for the pgsql limitations. 148 | 149 | 150 | Skip events configuration 151 | ==================================== 152 | 153 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 154 | :language: yaml 155 | :lines: 30-68 156 | :emphasize-lines: 32-37 157 | :linenos: 158 | 159 | 160 | The ``skip_events`` variable allows to tell pg_chameleon to skip events for tables or entire schemas. 161 | The example provided with configuration-example.ym disables the inserts on the table ``delphis_mediterranea.foo`` and disables the deletes on the entire schema ``delphis_mediterranea``. 162 | 163 | Keep existing schema 164 | ==================================== 165 | 166 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 167 | :language: yaml 168 | :lines: 30-68 169 | :emphasize-lines: 38-38 170 | :linenos: 171 | 172 | 173 | When set to ``Yes`` init_replica,refresh_schema and 174 | sync_tables do not recreate the affected tables using the data from the MySQL source. 175 | 176 | Instead the existing tables are truncated and the data is reloaded. 177 | A REINDEX TABLE is executed in order to have the indices in good shape after the reload. 178 | 179 | When ``keep_existing_schema`` is set to Yes the parameter ``grant_select_to`` have no effect. 180 | 181 | net_read_timeout 182 | ==================================== 183 | 184 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 185 | :language: yaml 186 | :lines: 30-68 187 | :emphasize-lines: 39-39 188 | :linenos: 189 | 190 | Configures for the session the net_read_timeout. 191 | Useful if the table copy during init replica fails on slow networks. 192 | 193 | It defaults to 600 seconds. 194 | 195 | PostgreSQL source type (EXPERIMENTAL) 196 | ================================================================ 197 | 198 | pg_chameleon 2.0 has an experimental support for the postgresql source type. 199 | When set to ``pgsql`` the system expects a postgresql source database rather a mysql. 200 | The following limitations apply. 201 | 202 | * There is no support for real time replica 203 | * The data copy happens always with file method 204 | * The copy_max_memory doesn't apply 205 | * The type override doesn't apply 206 | * Only ``init_replica`` is currently supported 207 | * The source connection string requires a database name 208 | * In the ``show_status`` detailed command the replicated tables counters are always zero 209 | 210 | 211 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 212 | :language: yaml 213 | :lines: 69-96 214 | :emphasize-lines: 7,16,25,27 215 | :linenos: 216 | 217 | Fillfactor 218 | ================================================================ 219 | The dictionary fillfactor is used to set the fillfactor for tables that are expected to work with large updates. 220 | The key name defines the fillfactor level (The allowed values range is 10 to 100). 221 | If key name is set to "*" then the fillfactor applies to all tables in the replicated schema. 222 | If the table appears multiple times, then only the last matched value will be applied 223 | 224 | .. literalinclude:: ../pg_chameleon/configuration/config-example.yml 225 | :language: yaml 226 | :lines: 101-108 227 | :linenos: -------------------------------------------------------------------------------- /docs/global_lib.rst: -------------------------------------------------------------------------------- 1 | global_lib api documentation 2 | ====================================================== 3 | 4 | .. automodule:: global_lib 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | :show-inheritance: 9 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pg_chameleon documentation master file, created by 2 | sphinx-quickstart on Wed Sep 14 22:19:28 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | pg_chameleon MySQL to PostgreSQL replica 7 | ========================================================= 8 | 9 | .. image:: ../images/pgchameleon.png 10 | :align: right 11 | 12 | 13 | 14 | 15 | 16 | pg_chameleon is a replication tool from MySQL to PostgreSQL developed in Python 3.5+ 17 | The system use the library mysql-replication to pull the row images from MySQL which are transformed into a jsonb object. 18 | A pl/pgsql function decodes the jsonb and replays the changes into the PostgreSQL database. 19 | 20 | The tool requires an initial replica setup which pulls the data from MySQL in read only mode. 21 | 22 | pg_chameleon can pull the data from a cascading replica when the MySQL slave is configured with log-slave-updates. 23 | 24 | 25 | 26 | `Documentation available at pgchameleon.org `_ 27 | 28 | `Release available via pypi `_ 29 | 30 | 31 | FEATURES 32 | ************************* 33 | * Replicates multiple MySQL schemas within the same MySQL cluster into a target PostgreSQL database. The source and target schema names can be different. 34 | * Conservative approach to the replica. Tables which generate errors are automatically excluded from the replica. 35 | * Daemonised init_replica,refresh_schema,sync_tables processes. 36 | * Daemonised replica process with two separated subprocess, one for the read and one for the replay. 37 | * Soft replica initialisation. The tables are locked when needed and stored with their log coordinates. The replica damon will put the database in a consistent status gradually. 38 | * Rollbar integration for a simpler error detection and alerting. 39 | 40 | 41 | CHANGELOG 42 | ******************** 43 | 44 | .. toctree:: 45 | :maxdepth: 2 46 | 47 | <./changelog.rst> 48 | 49 | 50 | 51 | 52 | 53 | RELEASE NOTES 54 | ******************** 55 | 56 | .. toctree:: 57 | :maxdepth: 2 58 | 59 | <./release_notes.rst> 60 | 61 | 62 | Upgrade procedure 63 | ******************** 64 | 65 | .. toctree:: 66 | :maxdepth: 2 67 | 68 | <./upgrade_procedure.rst> 69 | 70 | 71 | README 72 | ******************** 73 | 74 | .. toctree:: 75 | :maxdepth: 2 76 | 77 | <./readme.rst> 78 | 79 | The configuration file 80 | *********************************** 81 | 82 | .. toctree:: 83 | :maxdepth: 2 84 | 85 | <./configuration_file.rst> 86 | 87 | 88 | 89 | Usage instructions 90 | *********************************** 91 | 92 | .. toctree:: 93 | :maxdepth: 2 94 | 95 | <./usage.rst> 96 | 97 | Module reference 98 | *********************************** 99 | 100 | .. toctree:: 101 | :maxdepth: 2 102 | 103 | Module global_lib <./global_lib.rst> 104 | Module mysql_lib <./mysql_lib.rst> 105 | Module pg_lib <./pg_lib.rst> 106 | Module sql_util <./sql_util.rst> 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. epub3 to make an epub3 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. xml to make Docutils-native XML files 38 | echo. pseudoxml to make pseudoxml-XML files for display purposes 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | echo. coverage to run coverage check of the documentation if enabled 42 | echo. dummy to check syntax errors of document sources 43 | goto end 44 | ) 45 | 46 | if "%1" == "clean" ( 47 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 48 | del /q /s %BUILDDIR%\* 49 | goto end 50 | ) 51 | 52 | 53 | REM Check if sphinx-build is available and fallback to Python version if any 54 | %SPHINXBUILD% 1>NUL 2>NUL 55 | if errorlevel 9009 goto sphinx_python 56 | goto sphinx_ok 57 | 58 | :sphinx_python 59 | 60 | set SPHINXBUILD=python -m sphinx.__init__ 61 | %SPHINXBUILD% 2> nul 62 | if errorlevel 9009 ( 63 | echo. 64 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 65 | echo.installed, then set the SPHINXBUILD environment variable to point 66 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 67 | echo.may add the Sphinx directory to PATH. 68 | echo. 69 | echo.If you don't have Sphinx installed, grab it from 70 | echo.http://sphinx-doc.org/ 71 | exit /b 1 72 | ) 73 | 74 | :sphinx_ok 75 | 76 | 77 | if "%1" == "html" ( 78 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 79 | if errorlevel 1 exit /b 1 80 | echo. 81 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 82 | goto end 83 | ) 84 | 85 | if "%1" == "dirhtml" ( 86 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 87 | if errorlevel 1 exit /b 1 88 | echo. 89 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 90 | goto end 91 | ) 92 | 93 | if "%1" == "singlehtml" ( 94 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 95 | if errorlevel 1 exit /b 1 96 | echo. 97 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 98 | goto end 99 | ) 100 | 101 | if "%1" == "pickle" ( 102 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 103 | if errorlevel 1 exit /b 1 104 | echo. 105 | echo.Build finished; now you can process the pickle files. 106 | goto end 107 | ) 108 | 109 | if "%1" == "json" ( 110 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 111 | if errorlevel 1 exit /b 1 112 | echo. 113 | echo.Build finished; now you can process the JSON files. 114 | goto end 115 | ) 116 | 117 | if "%1" == "htmlhelp" ( 118 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 119 | if errorlevel 1 exit /b 1 120 | echo. 121 | echo.Build finished; now you can run HTML Help Workshop with the ^ 122 | .hhp project file in %BUILDDIR%/htmlhelp. 123 | goto end 124 | ) 125 | 126 | if "%1" == "qthelp" ( 127 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 128 | if errorlevel 1 exit /b 1 129 | echo. 130 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 131 | .qhcp project file in %BUILDDIR%/qthelp, like this: 132 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pg_chameleon.qhcp 133 | echo.To view the help file: 134 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pg_chameleon.ghc 135 | goto end 136 | ) 137 | 138 | if "%1" == "devhelp" ( 139 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 140 | if errorlevel 1 exit /b 1 141 | echo. 142 | echo.Build finished. 143 | goto end 144 | ) 145 | 146 | if "%1" == "epub" ( 147 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 148 | if errorlevel 1 exit /b 1 149 | echo. 150 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 151 | goto end 152 | ) 153 | 154 | if "%1" == "epub3" ( 155 | %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 156 | if errorlevel 1 exit /b 1 157 | echo. 158 | echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. 159 | goto end 160 | ) 161 | 162 | if "%1" == "latex" ( 163 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 164 | if errorlevel 1 exit /b 1 165 | echo. 166 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdf" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "latexpdfja" ( 181 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 182 | cd %BUILDDIR%/latex 183 | make all-pdf-ja 184 | cd %~dp0 185 | echo. 186 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 187 | goto end 188 | ) 189 | 190 | if "%1" == "text" ( 191 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 192 | if errorlevel 1 exit /b 1 193 | echo. 194 | echo.Build finished. The text files are in %BUILDDIR%/text. 195 | goto end 196 | ) 197 | 198 | if "%1" == "man" ( 199 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 200 | if errorlevel 1 exit /b 1 201 | echo. 202 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 203 | goto end 204 | ) 205 | 206 | if "%1" == "texinfo" ( 207 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 208 | if errorlevel 1 exit /b 1 209 | echo. 210 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 211 | goto end 212 | ) 213 | 214 | if "%1" == "gettext" ( 215 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 216 | if errorlevel 1 exit /b 1 217 | echo. 218 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 219 | goto end 220 | ) 221 | 222 | if "%1" == "changes" ( 223 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 224 | if errorlevel 1 exit /b 1 225 | echo. 226 | echo.The overview file is in %BUILDDIR%/changes. 227 | goto end 228 | ) 229 | 230 | if "%1" == "linkcheck" ( 231 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 232 | if errorlevel 1 exit /b 1 233 | echo. 234 | echo.Link check complete; look for any errors in the above output ^ 235 | or in %BUILDDIR%/linkcheck/output.txt. 236 | goto end 237 | ) 238 | 239 | if "%1" == "doctest" ( 240 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 241 | if errorlevel 1 exit /b 1 242 | echo. 243 | echo.Testing of doctests in the sources finished, look at the ^ 244 | results in %BUILDDIR%/doctest/output.txt. 245 | goto end 246 | ) 247 | 248 | if "%1" == "coverage" ( 249 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 250 | if errorlevel 1 exit /b 1 251 | echo. 252 | echo.Testing of coverage in the sources finished, look at the ^ 253 | results in %BUILDDIR%/coverage/python.txt. 254 | goto end 255 | ) 256 | 257 | if "%1" == "xml" ( 258 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 259 | if errorlevel 1 exit /b 1 260 | echo. 261 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 262 | goto end 263 | ) 264 | 265 | if "%1" == "pseudoxml" ( 266 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 267 | if errorlevel 1 exit /b 1 268 | echo. 269 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 270 | goto end 271 | ) 272 | 273 | if "%1" == "dummy" ( 274 | %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy 275 | if errorlevel 1 exit /b 1 276 | echo. 277 | echo.Build finished. Dummy builder generates no files. 278 | goto end 279 | ) 280 | 281 | :end 282 | -------------------------------------------------------------------------------- /docs/mysql_lib.rst: -------------------------------------------------------------------------------- 1 | mysql_lib api documentation 2 | ====================================================== 3 | 4 | .. automodule:: mysql_lib 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | :show-inheritance: 9 | -------------------------------------------------------------------------------- /docs/pg_lib.rst: -------------------------------------------------------------------------------- 1 | pg_lib api documentation 2 | ====================================================== 3 | 4 | .. automodule:: pg_lib 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | :show-inheritance: 9 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | 3 | -------------------------------------------------------------------------------- /docs/release_notes.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../RELEASE_NOTES.rst 2 | 3 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | PyMySQL>=0.10.0 2 | mysql-replication>=0.31 3 | psycopg2-binary>=2.8.3 4 | PyYAML>=3.13 5 | tabulate>=0.8.1 6 | daemonize>=2.4.7 7 | rollbar>=0.13.17 8 | parsy>=2.1 9 | Sphinx>=7.4.7 10 | -------------------------------------------------------------------------------- /docs/sql_util.rst: -------------------------------------------------------------------------------- 1 | sql_util api documentation 2 | ====================================================== 3 | 4 | .. automodule:: sql_util 5 | :members: 6 | :undoc-members: 7 | :inherited-members: 8 | :show-inheritance: 9 | 10 | 11 | -------------------------------------------------------------------------------- /docs/upgrade_procedure.rst: -------------------------------------------------------------------------------- 1 | Maintenance release upgrade 2 | ******************************** 3 | Upgrading a maintenance release is in general very simple but requires some attention. 4 | 5 | Always check the `release notes `_. 6 | If there is no schema upgrade the procedure is straightforward 7 | 8 | * Stop all the replica processes with ``chameleon stop_all_replicas --config `` 9 | * Install the upgrade with ``pip install pg_chameleon --upgrade`` 10 | * Check if the version is upgraded with ``chameleon --version`` 11 | * Start all the replicas. 12 | 13 | 14 | If the release comes with a schema upgrade, **after stopping the replicas** take a backup of the schema ``sch_chameleon`` with pg_dump for good measure. 15 | 16 | ``pg_dump -h -n sch_chameleon -Fc -f sch_chameleon.dmp -U -d `` 17 | 18 | * If working via ssh is suggested to use screen or tmux for the upgrade 19 | * Upgrade the pg_chameleon package with ``pip install pg_chameleon --upgrade`` 20 | * Upgrade the replica schema with the command ``chameleon upgrade_replica_schema --config `` 21 | * Start the replica processes 22 | 23 | If the upgrade procedure refuses to upgrade the catalogue because of running or errored replicas is possible to reset the statuses using the command ``chameleon enable_replica --source ``. 24 | 25 | If the catalogue upgrade is still not possible downgrading pgchameleon to the previous version. E.g. ``pip install pg_chameleon==2.0.7``. 26 | 27 | 28 | 29 | 30 | 31 | Version 1.8 to 2.0 upgrade 32 | ******************************** 33 | pg_chameleon 2.0 can upgrade an existing 1.8 replica catalogue using the command ``upgrade_replica_schema``. 34 | As the new version supports different schema mappings within the same source the parameter ``schema_mappings`` must match all the pairs 35 | ``my_database destination_schema`` for the source database that we are configuring. 36 | Any discrepancy will abort the upgrade procedure. 37 | 38 | Preparation 39 | .............................. 40 | * Check the pg_chameleon version you are upgrading is 1.8.2. If not upgrade it and **start the replica for each source present in the old catalogue**. 41 | This step is required in order to have the destination and target schema's updated from the configuration files. 42 | * Check the replica catalogue version is 1.7 with ``SELECT * FROM sch_chameleon.v_version;``. 43 | * Check the field t_source_schema have a schema name set ``SELECT t_source_schema FROM sch_chameleon.t_sources;`` 44 | * Take a backup of the existing schema sch_chameleon with ``pg_dump`` 45 | * Install pg_chameleon version 2 and create the configuration files executing ``chameleon set_configuration_files``. 46 | * cd in ``~/.pg_chameleon/configuration/`` and copy the file ``config-example.yml`` in a different file ``e.g. cp config-example.yml upgraded.yml`` 47 | * Edit the file and set the target and source's database connections. You may want to change the source name as well 48 | 49 | 50 | For each configuration file in the old setup ``~/.pg_chameleon/config/`` using the MySQL database configured in the source you should get the values stored in 51 | ``my_database`` and ``destination_schema`` and add it to the new source's schema_mappings. 52 | 53 | For example, if there are two sources ``source_01.yaml`` and ``source_02.yaml`` with the following configuration: 54 | 55 | Both sources are pointing the same MySQL database 56 | 57 | .. code-block:: yaml 58 | 59 | mysql_conn: 60 | host: my_host.foo.bar 61 | port: 3306 62 | user: my_replica 63 | passwd: foo_bar 64 | 65 | source_01.yaml have the following schema setup 66 | 67 | .. code-block:: yaml 68 | 69 | my_database: my_schema_01 70 | destination_schema: db_schema_01 71 | 72 | 73 | source_02.yaml have the following schema setup 74 | 75 | .. code-block:: yaml 76 | 77 | my_database: my_schema_02 78 | destination_schema: db_schema_02 79 | 80 | The new source's database configuration should be 81 | 82 | .. code-block:: yaml 83 | 84 | mysql: 85 | db_conn: 86 | host: "my_host.foo.bar" 87 | port: "3306" 88 | user: "my_replica" 89 | password: "foo_bar" 90 | charset: 'utf8' 91 | connect_timeout: 10 92 | schema_mappings: 93 | my_schema_01: db_schema_01 94 | my_schema_02: db_schema_02 95 | 96 | 97 | Upgrade 98 | .............................. 99 | 100 | Execute the following command 101 | ``chameleon upgrade_replica_schema --config upgraded`` 102 | 103 | The procedure checks if the start catalogue version is 1.7 and fails if the value is different. 104 | After answering YES the procedure executes the following steps. 105 | 106 | * Replays any exising batches present in the catalogue 1.7 107 | * Checks if the schema_mappings are compatible with the values stored in the schema ``sch_chameleon`` 108 | * Renames the schema ``sch_chameleon`` to ``_sch_chameleon_version1`` 109 | * Installs a new 2.0 schema in ``sch_chameleon`` 110 | * Stores a new source using the schema mappings 111 | * Migrates the existing tables into the new catalogue using the replica batch data to store the tables start of consistent point. 112 | * Determines maximum and minimum point for the binlog coordinates and use them for writing the new batch start point and the source's consistent point 113 | 114 | If the migration is successful, before starting the replica process is better to check that all tables are correctly mapped with 115 | 116 | ``chameleon show_status --source upgraded`` 117 | 118 | 119 | Rollback 120 | .............................. 121 | 122 | If something goes wrong during the upgrade procedure, then the changes are rolled back. 123 | The schema ``sch_chameleon`` is renamed to ``_sch_chameleon_version2`` and the previous version's schema ``_sch_chameleon_version1`` is put batck to ``sch_chameleon``. 124 | If this happens the procedure 1.8.2 will continue to work as usual. The schema ``_sch_chameleon_version2`` can be used to check what went wrong. 125 | 126 | Before attempting a new upgrade schema ``_sch_chameleon_version2`` should be dropped or renamed in order to avoid a schema conflict in the case of another failure. 127 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ************************************************** 3 | 4 | Command line reference 5 | ............................................ 6 | 7 | .. code-block:: bash 8 | 9 | chameleon command [ [ --config ] [ --source ] [ --schema ] [ --tables ] [--logid] [ --debug ] [ --rollbar-level ] ] [ --version ] [ --full ] 10 | 11 | .. csv-table:: Options 12 | :header: "Option", "Description", "Default","Example" 13 | 14 | ``--config``, Specifies the configuration to use in ``~.pg_chameleon/configuration/``. The configuration name should be the file without the extension ``.yml`` , ``default``,``--config foo`` will use the file ``~.pg_chameleon/configuration/foo.yml`` 15 | ``--source``, Specifies the source within a configuration file., N/A, ``--source bar`` 16 | ``--schema``, Specifies a schema configured within a source., N/A, ``--schema schema_foo`` 17 | ``--tables``, Specifies one or more tables configured in a schema. Multiple tables can be specified separated by comma. The table must have the schema., N/A, ``--tables schema_foo.table_bar`` 18 | ``--logid``, Specifies the log id entry for displaying the error details, N/A, ``--logid 30`` 19 | ``--debug``,When added to the command line the debug option disables any daemonisation and outputs all the logging to the console. The keybord interrupt signal is trapped correctly., N/A, ``--debug`` 20 | ``--version``,Displays the package version., N/A, ``--version`` 21 | ``--rollbar-level``, Sets the maximum level for the messages to be sent to rolllbar. Accepted values: "critical" "error" "warning" "info", ``info`` ,``--rollbar-level error`` 22 | ``--full``,Runs a VACUUM FULL on the log tables when the run_maintenance is executed, N/A,``--full`` 23 | 24 | 25 | .. csv-table:: Command list reference 26 | :header: "Command", "Description", "Options" 27 | 28 | ``set_configuration_files``, Setup the example configuration files and directories in ``~/.pg_chameleon`` 29 | ``show_config``, Displays the configuration for the configuration, ``--config`` 30 | ``show_sources``, Displays the sourcches configured for the configuration, ``--config`` 31 | ``show_status``,Displays an overview of the status of the sources configured within the configuration. Specifying the source gives more details about that source , ``--config`` ``--source`` 32 | ``show_errors``,Displays the errors logged by the replay function. If a log id is specified then the log entry is displayed entirely, ``--config`` ``--logid`` 33 | ``create_replica_schema``, Creates a new replication schema into the config's destination database, ``--config`` 34 | ``drop_replica_schema``, Drops an existing replication schema from the config's destination database, ``--config`` 35 | ``upgrade_replica_schema``,Upgrades the replica schema from a an older version,``--config`` 36 | ``add_source``, Adds a new source to the replica catalogue, ``--config`` ``--source`` 37 | ``drop_source``, Remove an existing source from the replica catalogue, ``--config`` ``--source`` 38 | ``init_replica``, Initialise the replica for an existing source , ``--config`` ``--source`` 39 | ``copy_schema``, Copy only the schema from mysql to PostgreSQL., ``--config`` ``--source`` 40 | ``update_schema_mappings``,Update the schema mappings stored in the replica catalogue using the data from the configuration file. , ``--config`` ``--source`` 41 | ``refresh_schema``, Synchronise all the tables for a given schema within an already initialised source. , ``--config`` ``--source`` ``--schema`` 42 | ``sync_tables``, Synchronise one or more tables within an already initialised source. The switch ``--tables`` accepts the special name ``disabled`` to resync all the tables with replica disabled., ``--config`` ``--source`` ``--tables`` 43 | ``start_replica``, Starts the replica process daemon, ``--config`` ``--source`` 44 | ``stop_replica``, Stops the replica process daemon, ``--config`` ``--source`` 45 | ``detach_replica``, Detaches a replica from the mysql master configuring the postgres schemas to work as a standalone system. Useful for migrations., ``--config`` ``--source`` 46 | ``enable_replica``, Enables the replica for the given source changing the source status to stopped. It's useful if the replica crashes., ``--config`` ``--source`` 47 | ``run_maintenance``, Runs a VACUUM on the log tables for the given source. If is specified then the maintenance runs a VACUUM FULL, ``--config`` ``--source`` ``--full`` 48 | ``stop_all_replicas``, Stops all the running sources within the target postgresql database., ``--config`` 49 | 50 | 51 | Example 52 | ............................................ 53 | 54 | Create a virtualenv and activate it 55 | 56 | .. code-block:: none 57 | 58 | python3 -m venv venv 59 | source venv/bin/activate 60 | 61 | 62 | Install pg_chameleon 63 | 64 | .. code-block:: none 65 | 66 | pip install pip --upgrade 67 | pip install pg_chameleon 68 | 69 | Run the ``set_configuration_files`` command in order to create the configuration directory. 70 | 71 | .. code-block:: none 72 | 73 | chameleon set_configuration_files 74 | 75 | 76 | cd in ``~/.pg_chameleon/configuration`` and copy the file ``config-example.yml` to ``default.yml``. 77 | 78 | 79 | 80 | In MySQL create a user for the replica. 81 | 82 | .. code-block:: sql 83 | 84 | CREATE USER usr_replica ; 85 | SET PASSWORD FOR usr_replica=PASSWORD('replica'); 86 | GRANT ALL ON sakila.* TO 'usr_replica'; 87 | GRANT RELOAD ON *.* to 'usr_replica'; 88 | GRANT REPLICATION CLIENT ON *.* to 'usr_replica'; 89 | GRANT REPLICATION SLAVE ON *.* to 'usr_replica'; 90 | FLUSH PRIVILEGES; 91 | 92 | Add the configuration for the replica to my.cnf. It requires a MySQL restart. 93 | 94 | 95 | 96 | .. code-block:: ini 97 | 98 | binlog_format= ROW 99 | binlog_row_image=FULL 100 | log-bin = mysql-bin 101 | server-id = 1 102 | expire_logs_days = 10 103 | # MARIADB 10.5.0+ OR MYSQL 8.0.14+ versions 104 | binlog_row_metadata = FULL 105 | 106 | 107 | 108 | 109 | In PostgreSQL create a user for the replica and a database owned by the user 110 | 111 | .. code-block:: sql 112 | 113 | CREATE USER usr_replica WITH PASSWORD 'replica'; 114 | CREATE DATABASE db_replica WITH OWNER usr_replica; 115 | 116 | Check you can connect to both databases from the machine where pg_chameleon is installed. 117 | 118 | For MySQL 119 | 120 | .. code-block:: none 121 | 122 | mysql -p -h derpy -u usr_replica sakila 123 | Enter password: 124 | Reading table information for completion of table and column names 125 | You can turn off this feature to get a quicker startup with -A 126 | 127 | Welcome to the MySQL monitor. Commands end with ; or \g. 128 | Your MySQL connection id is 116 129 | Server version: 5.6.30-log Source distribution 130 | 131 | Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. 132 | 133 | Oracle is a registered trademark of Oracle Corporation and/or its 134 | affiliates. Other names may be trademarks of their respective 135 | owners. 136 | 137 | Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. 138 | 139 | mysql> 140 | 141 | For PostgreSQL 142 | 143 | .. code-block:: none 144 | 145 | psql -h derpy -U usr_replica db_replica 146 | Password for user usr_replica: 147 | psql (9.5.5) 148 | Type "help" for help. 149 | db_replica=> 150 | 151 | Check the docs for the configuration file reference. It will help you to configure correctly the connections. 152 | 153 | Initialise the replica 154 | 155 | 156 | .. code-block:: none 157 | 158 | chameleon create_replica_schema --debug 159 | chameleon add_source --config default --debug 160 | chameleon init_replica --config default --debug 161 | 162 | 163 | Start the replica with 164 | 165 | 166 | .. code-block:: none 167 | 168 | chameleon start_replica --config default --source example 169 | 170 | Check the source status 171 | 172 | .. code-block:: none 173 | 174 | chameleon show_status --source example 175 | 176 | Check the error log 177 | 178 | .. code-block:: none 179 | 180 | chameleon show_errors 181 | 182 | .. code-block:: none 183 | 184 | chameleon start_replica --config default --source example 185 | 186 | 187 | To stop the replica 188 | 189 | .. code-block:: none 190 | 191 | chameleon stop_replica --config default --source example 192 | 193 | 194 | To detach the replica 195 | 196 | .. code-block:: none 197 | 198 | chameleon detach_replica --config default --source example 199 | 200 | 201 | 202 | 203 | -------------------------------------------------------------------------------- /images/pgchameleon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/the4thdoctor/pg_chameleon/5458575165565b4593d33af912e7fdbeb4db49f8/images/pgchameleon.png -------------------------------------------------------------------------------- /parse.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pg_chameleon import sql_token 3 | 4 | 5 | #statement="""alter table test_pk drop primary key;""" 6 | #statement="""ALTER TABLE test ADD COLUMN `dkdkd` timestamp NULL;""" 7 | #statement="""create table test_pk (id int ,PRIMARY KEY (id) ); """ 8 | #statement="""alter table test change date_create_new date_create_new timestamp;""" 9 | #statement = """alter table test add column `test_default` varchar(30) not null default 20 """ 10 | #statement = """ALTER TABLE test 11 | #ADD COLUMN `count` SMALLINT(6) NULL , 12 | #ADD COLUMN `log` VARCHAR(12) NULL default 'blah' AFTER `count`, 13 | #ADD COLUMN new_enum ENUM('asd','r') NULL AFTER `log`, 14 | #ADD COLUMN status INT(10) UNSIGNED NULL AFTER `new_enum` 15 | #""" 16 | 17 | statement = """ALTER TABLE foo DROP FOREIGN KEY fk_trigger_bar,ADD COLUMN `count` SMALLINT(6) NULL;""" 18 | statement = """ALTER TABLE test 19 | ADD `count` SMALLINT(6) NULL , 20 | ADD COLUMN `log` VARCHAR(12) default 'blah' NULL AFTER `count`, 21 | ADD COLUMN new_enum ENUM('asd','r') NULL AFTER `log`, 22 | ADD COLUMN status INT(10) UNSIGNED NULL AFTER `new_enum`, 23 | ADD COLUMN mydate datetime NULL AFTER `status`, 24 | ADD COLUMN mytstamp timestamp NULL AFTER `status`, 25 | DROP FOREIGN KEY fk_trigger_bar, 26 | add primary key, 27 | drop unique index asdf 28 | """ 29 | statement=""" 30 | CREATE TABLE film_text ( 31 | film_id SMALLINT NOT NULL, 32 | title VARCHAR(255) NOT NULL, 33 | description TEXT, 34 | PRIMARY KEY (film_id), 35 | KEY idx_title_description (title,description) 36 | )ENGINE=InnoDB DEFAULT CHARSET=utf8 37 | """ 38 | 39 | statement=""" 40 | CREATE TABLE film_text ( 41 | film_id SMALLINT NOT NULL PRIMARY KEY 42 | )ENGINE=InnoDB DEFAULT CHARSET=utf8 43 | """ 44 | #statement="""RENAME TABLE `sakila`.`test_partition` TO `sakila`.`_test_partition_old`, `_test_partition_new` TO `test_partition`;""" 45 | #statement="""RENAME TABLE test_partition TO _test_partition_old, _test_partition_new TO test_partition; """ 46 | #statement="""RENAME TABLE sakila.test_partition TO sakila._test_partition_old, sakila._test_partition_new TO sakila.test_partition ;""" 47 | #statement="""RENAME TABLE `sakila`.`test_partition` TO `sakila`.`_test_partition_old`, `sakila`.`_test_partition_new` TO `sakila`.`test_partition`;""" 48 | #statement = """create table blah(id integer(30) not null auto_increment, datevalue datetime,primary key (id,datevalue))""" 49 | #statement = """alter table dd add column(foo varchar(30)); alter table dd add column foo varchar(30);""" 50 | #statement = """create table test_tiny(id int(4) auto_increment, value tinyint(1), unique key(id),unique key (id,value),unique key (value,id)); """ 51 | 52 | 53 | statement=""" 54 | 55 | -- drop table 56 | DROP TABLE `test`; 57 | -- create table 58 | CREATE TABLE `test` ( 59 | store_id TINYINT UNSIGNED NULL AUTO_INCREMENT, 60 | manager_staff_id TINYINT UNSIGNED NOT NULL, 61 | address_id SMALLINT UNSIGNED NOT NULL, 62 | `address_txt` varchar (30) NOT NULL default 'default_t;ext', 63 | `address_dp` double precision (30,2) NOT NULL, 64 | `test_enum` enum ('a','b'), 65 | size ENUM('x-small', 'small', 'medium', 'large', 'x-large'), 66 | last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 67 | PRIMARY KEY (store_id,address_id), 68 | UNIQUE KEY idx_unique_manager (manager_staff_id), 69 | KEY idx_fk_address_id2 (address_id), 70 | index 71 | idx_fk_address_id (address_id,store_id) 72 | )ENGINE=InnoDB DEFAULT CHARSET=utf8; 73 | 74 | 75 | ALTER TABLE test 76 | ADD COLUMN `count` SMALLINT(6) NOT NULL default 3 AFTER `test_enum`, 77 | ADD COLUMN `log` VARCHAR(12) NOT NULL AFTER `count`, 78 | ADD COLUMN new_enum ENUM('asd','r') NOT NULL AFTER `log`, 79 | ADD COLUMN status INT(10) UNSIGNED NOT NULL AFTER `new_enum`; 80 | 81 | 82 | ALTER TABLE `test` 83 | DROP COLUMN `count` , 84 | ADD COLUMN newstatus INT(10) UNSIGNED NOT NULL AFTER `log`; 85 | 86 | ALTER TABLE `test` DROP PRIMARY KEY; 87 | 88 | """ 89 | statement="""ALTER TABLE t_user_info ADD ( 90 | group_id INT(11) UNSIGNED DEFAULT NULL, 91 | contact_phone VARCHAR(20) DEFAULT NULL 92 | );""" 93 | statement = """ALTER TABLE foo RENAME TO bar;""" 94 | statement = """RENAME TABLE `sakila`.`test_partition` TO `sakila`.`_test_partition_old`, `_test_partition_new` TO `test_partition`;""" 95 | #statement="""ALTER TABLE foo MODIFY bar INT UNSIGNED DEFAULT NULL;""" 96 | #statement="""ALTER TABLE foo change bar bar INT UNSIGNED;""" 97 | statement="""ALTER TABLE `some_sch`.`my_great_table` CHANGE COLUMN `IMEI` `IMEI` VARCHAR(255) NULL DEFAULT NULL COMMENT 'IMEI datatype changed'""" 98 | token_sql=sql_token() 99 | token_sql.parse_sql(statement) 100 | print (token_sql.tokenised) 101 | #for token in token_sql.tokenised: 102 | #print (token) 103 | # for column in token["columns"]: 104 | # print(column) 105 | #else: 106 | 107 | -------------------------------------------------------------------------------- /pg_chameleon/__init__.py: -------------------------------------------------------------------------------- 1 | from .lib.sql_util import * 2 | from .lib.mysql_lib import * 3 | from .lib.pg_lib import * 4 | from .lib.global_lib import * -------------------------------------------------------------------------------- /pg_chameleon/configuration/config-example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # global settings 3 | pid_dir: '~/.pg_chameleon/pid/' 4 | log_dir: '~/.pg_chameleon/logs/' 5 | log_dest: file 6 | log_level: info 7 | log_days_keep: 10 8 | rollbar_key: '' 9 | rollbar_env: '' 10 | 11 | # type_override allows the user to override the default type conversion 12 | # into a different one. 13 | 14 | type_override: 15 | "tinyint(1)": 16 | override_to: boolean 17 | override_tables: 18 | - "*" 19 | 20 | 21 | # postgres destination connection 22 | pg_conn: 23 | host: "localhost" 24 | port: "5432" 25 | user: "usr_replica" 26 | password: "never_commit_password" 27 | database: "db_replica" 28 | charset: "utf8" 29 | 30 | sources: 31 | mysql: 32 | db_conn: 33 | host: "localhost" 34 | port: "3306" 35 | user: "usr_replica" 36 | password: "never_commit_passwords" 37 | charset: 'utf8' 38 | connect_timeout: 10 39 | schema_mappings: 40 | delphis_mediterranea: loxodonta_africana 41 | limit_tables: 42 | - delphis_mediterranea.foo 43 | skip_tables: 44 | - delphis_mediterranea.bar 45 | grant_select_to: 46 | - usr_readonly 47 | lock_timeout: "120s" 48 | my_server_id: 100 49 | replica_batch_size: 10000 50 | replay_max_rows: 10000 51 | batch_retention: '1 day' 52 | copy_max_memory: "300M" 53 | copy_mode: 'file' 54 | out_dir: /tmp 55 | sleep_loop: 1 56 | on_error_replay: continue 57 | on_error_read: continue 58 | auto_maintenance: "disabled" 59 | gtid_enable: false 60 | type: mysql 61 | skip_events: 62 | insert: 63 | - delphis_mediterranea.foo # skips inserts on delphis_mediterranea.foo 64 | delete: 65 | - delphis_mediterranea # skips deletes on schema delphis_mediterranea 66 | update: 67 | keep_existing_schema: No 68 | net_read_timeout: 600 69 | 70 | pgsql: 71 | db_conn: 72 | host: "localhost" 73 | port: "5432" 74 | user: "usr_replica" 75 | password: "never_commit_passwords" 76 | database: "db_replica" 77 | charset: 'utf8' 78 | connect_timeout: 10 79 | schema_mappings: 80 | loxodonta_africana: elephas_maximus 81 | limit_tables: 82 | - loxodonta_africana.foo 83 | skip_tables: 84 | - loxodonta_africana.bar 85 | copy_max_memory: "300M" 86 | grant_select_to: 87 | - usr_readonly 88 | lock_timeout: "10s" 89 | my_server_id: 100 90 | replica_batch_size: 3000 91 | replay_max_rows: 10000 92 | sleep_loop: 5 93 | batch_retention: '1 day' 94 | copy_mode: 'file' 95 | out_dir: /tmp 96 | type: pgsql 97 | 98 | # the dictionary fillfactor is used to set the fillfactor for tables that are expected to work with large updates 99 | # The key name defines the fillfactor level (The allowed values range is 10 to 100) 100 | # If key name is set to "*" then the fillfactor applies to all tables in the replicated schema 101 | # If the table appears in multiple fillfactors then only the last matched value will be applied 102 | fillfactor: 103 | "30": 104 | tables: 105 | - "foo.bar" 106 | "40": 107 | tables: 108 | - "foo.foobar" 109 | -------------------------------------------------------------------------------- /pg_chameleon/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/the4thdoctor/pg_chameleon/5458575165565b4593d33af912e7fdbeb4db49f8/pg_chameleon/lib/__init__.py -------------------------------------------------------------------------------- /pg_chameleon/sql/dev/custom_aggregate.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_binlog_pos(integer[],integer[]) 2 | RETURNS integer[] AS 3 | $BODY$ 4 | DECLARE 5 | binlog_max ALIAS FOR $1; 6 | binlog_value ALIAS FOR $2; 7 | binlog_state integer[]; 8 | BEGIN 9 | IF binlog_value[1]>binlog_max[1] 10 | THEN 11 | binlog_state:=binlog_value; 12 | ELSEIF binlog_value[1]=binlog_max[2] 16 | THEN 17 | binlog_state:=binlog_value; 18 | ELSE 19 | binlog_state:=binlog_max; 20 | END IF; 21 | 22 | RETURN binlog_state; 23 | 24 | END; 25 | $BODY$ 26 | LANGUAGE plpgsql; 27 | 28 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_binlog_final(integer[]) 29 | RETURNS integer[] as 30 | $BODY$ 31 | SELECT 32 | CASE 33 | WHEN ( $1[1] = 0 AND $1[2] = 0 ) 34 | THEN NULL 35 | ELSE $1 36 | END; 37 | $BODY$ 38 | LANGUAGE sql; 39 | 40 | 41 | create aggregate sch_chameleon.binlog_max(integer[]) 42 | ( 43 | SFUNC = sch_chameleon.fn_binlog_pos, 44 | STYPE = integer[], 45 | FINALFUNC = sch_chameleon.fn_binlog_final, 46 | INITCOND = '{0,0}' 47 | ); 48 | 49 | 50 | SELECT sch_chameleon.binlog_max(array[(string_to_array(t_binlog_name,'.'))[2]::integer,i_binlog_position]) FROM sch_chameleon.t_replica_tables; 51 | --SELECT sch_chameleon.fn_binlog_final(array[0,0]) -------------------------------------------------------------------------------- /pg_chameleon/sql/dev/fn_parse_json.sql: -------------------------------------------------------------------------------- 1 | -- parsing function prototype 2 | SELECT 3 | agg.i_id_event, 4 | agg.v_table_name, 5 | agg.v_schema_name, 6 | agg.enm_binlog_event, 7 | agg.t_query, 8 | array_to_string(array_agg(quote_ident(t_column)),',') as t_event_columns, 9 | array_to_string(array_agg(quote_nullable(jsb_event_after->>t_column)),',') as t_event_data, 10 | array_agg(agg.t_pk_data) as t_pk_data, 11 | array_agg(format('%I=%L',t_column,agg.jsb_event_after->>t_column)) as t_update 12 | FROM 13 | ( 14 | SELECT 15 | trn.i_id_event, 16 | trn.v_table_name, 17 | trn.v_schema_name, 18 | trn.enm_binlog_event, 19 | trn.jsb_event_after, 20 | trn.t_query, 21 | trn.t_column, 22 | trn.t_pk_data 23 | 24 | FROM 25 | ( 26 | SELECT 27 | evt.i_id_event, 28 | evt.v_table_name, 29 | evt.v_schema_name, 30 | evt.enm_binlog_event, 31 | evt.jsb_event_after, 32 | evt.t_query, 33 | v_table_pkey, 34 | format( 35 | '%I=%L', 36 | evt.v_table_pkey, 37 | CASE 38 | WHEN evt.enm_binlog_event = 'update' 39 | THEN 40 | jsb_event_before->>v_table_pkey 41 | ELSE 42 | jsb_event_after->>v_table_pkey 43 | END 44 | ) as t_pk_data, 45 | 46 | 47 | FROM 48 | 49 | ( 50 | SELECT 51 | log.i_id_event, 52 | log.v_table_name, 53 | log.v_schema_name, 54 | log.enm_binlog_event, 55 | coalesce(log.jsb_event_after,'{"foo":"bar"}'::jsonb) as jsb_event_after, 56 | (jsonb_each_text(log.jsb_event_after)).key AS t_column, 57 | log.jsb_event_before, 58 | log.t_query as t_query, 59 | log.ts_event_datetime, 60 | unnest(v_table_pkey) as v_table_pkey 61 | FROM 62 | --sch_chameleon.t_log_replica_mysql_1 log 63 | sch_chameleon.t_log_replica log 64 | INNER JOIN sch_chameleon.t_replica_tables tab 65 | ON 66 | tab.v_table_name=log.v_table_name 67 | AND tab.v_schema_name=log.v_schema_name 68 | WHERE 69 | True 70 | --i_id_event = any('{{809600}}'::integer[]) 71 | LIMIT 30 72 | ) evt 73 | ) trn 74 | GROUP BY 75 | trn.i_id_event, 76 | trn.v_table_name, 77 | trn.v_schema_name, 78 | trn.enm_binlog_event, 79 | trn.jsb_event_after, 80 | trn.t_query, 81 | trn.t_column, 82 | trn.t_pk_data 83 | ) agg 84 | GROUP BY 85 | agg.i_id_event, 86 | agg.v_table_name, 87 | agg.v_schema_name, 88 | agg.enm_binlog_event, 89 | agg.t_query 90 | ORDER BY i_id_event ASC -------------------------------------------------------------------------------- /pg_chameleon/sql/dev/fn_parse_json_2.sql: -------------------------------------------------------------------------------- 1 | --select * from sch_chameleon.t_error_log 2 | SELECT 3 | i_id_event, 4 | v_table_name, 5 | v_schema_name, 6 | t_binlog_name, 7 | i_binlog_position, 8 | enm_binlog_event, 9 | (enm_binlog_event='ddl')::int as i_ddl, 10 | (enm_binlog_event<>'ddl')::int as v_i_replayed, 11 | CASE 12 | WHEN enm_binlog_event = 'ddl' 13 | THEN 14 | t_query 15 | WHEN enm_binlog_event = 'insert' 16 | THEN 17 | format( 18 | 'INSERT INTO %I.%I %s;', 19 | v_schema_name, 20 | v_table_name, 21 | t_dec_data 22 | 23 | ) 24 | WHEN enm_binlog_event = 'update' 25 | THEN 26 | format( 27 | 'UPDATE %I.%I SET %s WHERE %s;', 28 | v_schema_name, 29 | v_table_name, 30 | t_dec_data, 31 | t_pk_data 32 | ) 33 | WHEN enm_binlog_event = 'delete' 34 | THEN 35 | format( 36 | 'DELETE FROM %I.%I WHERE %s;', 37 | v_schema_name, 38 | v_table_name, 39 | t_pk_data 40 | ) 41 | 42 | END AS t_sql 43 | FROM 44 | ( 45 | SELECT 46 | dec.i_id_event, 47 | dec.v_table_name, 48 | dec.v_schema_name, 49 | dec.enm_binlog_event, 50 | dec.t_binlog_name, 51 | dec.i_binlog_position, 52 | dec.t_query as t_query, 53 | dec.ts_event_datetime, 54 | CASE 55 | WHEN dec.enm_binlog_event = 'insert' 56 | THEN 57 | format('(%s) VALUES (%s)',string_agg(format('%I',dec.t_column),','),string_agg(format('%L',jsb_event_after->>t_column),',')) 58 | WHEN dec.enm_binlog_event = 'update' 59 | THEN 60 | string_agg(format('%I=%L',dec.t_column,jsb_event_after->>t_column),',') 61 | 62 | END AS t_dec_data, 63 | string_agg(DISTINCT format( 64 | '%I=%L', 65 | dec.v_table_pkey, 66 | CASE 67 | WHEN dec.enm_binlog_event = 'update' 68 | THEN 69 | jsb_event_before->>v_table_pkey 70 | ELSE 71 | jsb_event_after->>v_table_pkey 72 | END 73 | ),' AND ') as t_pk_data 74 | FROM 75 | ( 76 | SELECT 77 | log.i_id_event, 78 | log.v_table_name, 79 | log.v_schema_name, 80 | log.enm_binlog_event, 81 | log.t_binlog_name, 82 | log.i_binlog_position, 83 | coalesce(log.jsb_event_after,'{"foo":"bar"}'::jsonb) as jsb_event_after, 84 | (jsonb_each_text(coalesce(log.jsb_event_after,'{"foo":"bar"}'::jsonb))).key AS t_column, 85 | log.jsb_event_before, 86 | log.t_query as t_query, 87 | log.ts_event_datetime, 88 | unnest(v_table_pkey) as v_table_pkey 89 | FROM 90 | --sch_chameleon.t_log_replica_mysql_1 log 91 | sch_chameleon.t_log_replica log 92 | INNER JOIN sch_chameleon.t_replica_tables tab 93 | ON 94 | tab.v_table_name=log.v_table_name 95 | AND tab.v_schema_name=log.v_schema_name 96 | WHERE 97 | tab.b_replica_enabled 98 | AND TRUE--i_id_event = any('{{809611,809614,809615}}'::integer[]) 99 | ) dec 100 | GROUP BY 101 | dec.i_id_event, 102 | dec.v_table_name, 103 | dec.v_schema_name, 104 | dec.enm_binlog_event, 105 | dec.t_query, 106 | dec.ts_event_datetime, 107 | dec.t_binlog_name, 108 | dec.i_binlog_position 109 | ) par 110 | -------------------------------------------------------------------------------- /pg_chameleon/sql/dev/fn_process_batch.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_process_batch(integer,integer) 2 | RETURNS BOOLEAN AS 3 | $BODY$ 4 | DECLARE 5 | p_i_max_events ALIAS FOR $1; 6 | p_i_source_id ALIAS FOR $2; 7 | v_b_loop boolean; 8 | v_r_rows record; 9 | v_i_id_batch bigint; 10 | v_t_ddl text; 11 | v_i_replayed integer; 12 | v_i_skipped integer; 13 | v_i_ddl integer; 14 | v_i_evt_replay bigint[]; 15 | v_i_evt_queue bigint[]; 16 | BEGIN 17 | v_b_loop:=FALSE; 18 | v_i_replayed:=0; 19 | v_i_ddl:=0; 20 | v_i_skipped:=0; 21 | 22 | v_i_id_batch:= ( 23 | SELECT 24 | i_id_batch 25 | FROM ONLY 26 | sch_chameleon.t_replica_batch 27 | WHERE 28 | b_started 29 | AND b_processed 30 | AND NOT b_replayed 31 | AND i_id_source=p_i_source_id 32 | ORDER BY 33 | ts_created 34 | LIMIT 1 35 | ) 36 | ; 37 | 38 | 39 | 40 | v_i_evt_replay:=( 41 | SELECT 42 | i_id_event[1:p_i_max_events] 43 | FROM 44 | sch_chameleon.t_batch_events 45 | WHERE 46 | i_id_batch=v_i_id_batch 47 | ); 48 | 49 | v_i_evt_queue:=( 50 | SELECT 51 | i_id_event[p_i_max_events+1:array_length(i_id_event,1)] 52 | FROM 53 | sch_chameleon.t_batch_events 54 | WHERE 55 | i_id_batch=v_i_id_batch 56 | ); 57 | 58 | IF v_i_id_batch IS NULL 59 | THEN 60 | RETURN v_b_loop; 61 | END IF; 62 | RAISE DEBUG 'Found id_batch %', v_i_id_batch; 63 | 64 | FOR v_r_rows IN 65 | SELECT 66 | CASE 67 | WHEN enm_binlog_event = 'ddl' 68 | THEN 69 | t_query 70 | WHEN enm_binlog_event = 'insert' 71 | THEN 72 | format( 73 | 'INSERT INTO %I.%I (%s) VALUES (%s);', 74 | v_schema_name, 75 | v_table_name, 76 | array_to_string(t_colunm,','), 77 | array_to_string(t_event_data,',') 78 | 79 | ) 80 | WHEN enm_binlog_event = 'update' 81 | THEN 82 | format( 83 | 'UPDATE %I.%I SET %s WHERE %s;', 84 | v_schema_name, 85 | v_table_name, 86 | t_update, 87 | t_pk_update 88 | ) 89 | WHEN enm_binlog_event = 'delete' 90 | THEN 91 | format( 92 | 'DELETE FROM %I.%I WHERE %s;', 93 | v_schema_name, 94 | v_table_name, 95 | t_pk_data 96 | ) 97 | 98 | END AS t_sql, 99 | i_id_event, 100 | i_id_batch, 101 | enm_binlog_event 102 | FROM 103 | ( 104 | <<<<<<< HEAD 105 | <<<<<<< 98364345ea29577df3de6311fc350e8f57876fe9 106 | SELECT 107 | ======= 108 | SELECT 109 | >>>>>>> new function seems to work properly 110 | ======= 111 | SELECT 112 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 113 | i_id_event, 114 | i_id_batch, 115 | v_table_name, 116 | v_schema_name, 117 | enm_binlog_event, 118 | <<<<<<< HEAD 119 | <<<<<<< 98364345ea29577df3de6311fc350e8f57876fe9 120 | ======= 121 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 122 | t_query, 123 | ts_event_datetime, 124 | t_pk_data, 125 | t_pk_update, 126 | array_agg(quote_ident(t_column)) AS t_colunm, 127 | string_agg(distinct format('%I=%L',t_column,jsb_event_data->>t_column),',') as t_update, 128 | array_agg(quote_nullable(jsb_event_data->>t_column)) as t_event_data 129 | FROM 130 | ( 131 | SELECT 132 | i_id_event, 133 | i_id_batch, 134 | v_table_name, 135 | v_schema_name, 136 | enm_binlog_event, 137 | jsb_event_data, 138 | jsb_event_update, 139 | t_query, 140 | ts_event_datetime, 141 | string_agg(distinct format('%I=%L',v_pkey,jsb_event_data->>v_pkey),' AND ') as t_pk_data, 142 | string_agg(distinct format('%I=%L',v_pkey,jsb_event_update->>v_pkey),' AND ') as t_pk_update, 143 | (jsonb_each_text(coalesce(jsb_event_data,'{"foo":"bar"}'::jsonb))).key AS t_column 144 | FROM 145 | ( 146 | SELECT 147 | i_id_event, 148 | i_id_batch, 149 | v_table_name, 150 | v_schema_name, 151 | enm_binlog_event, 152 | jsb_event_data, 153 | jsb_event_update, 154 | t_query, 155 | ts_event_datetime, 156 | replace(unnest(string_to_array(v_table_pkey[1],',')),'"','') as v_pkey 157 | 158 | 159 | 160 | FROM 161 | ( 162 | SELECT 163 | log.i_id_event, 164 | log.i_id_batch, 165 | log.v_table_name, 166 | log.v_schema_name, 167 | log.enm_binlog_event, 168 | log.jsb_event_data, 169 | log.jsb_event_update, 170 | log.t_query, 171 | ts_event_datetime, 172 | v_table_pkey 173 | 174 | 175 | 176 | FROM 177 | sch_chameleon.t_log_replica log 178 | INNER JOIN sch_chameleon.t_replica_tables tab 179 | ON 180 | tab.v_table_name=log.v_table_name 181 | AND tab.v_schema_name=log.v_schema_name 182 | WHERE 183 | log.i_id_batch=v_i_id_batch 184 | AND log.i_id_event=ANY(v_i_evt_replay) 185 | ) t_log 186 | 187 | ) t_pkey 188 | GROUP BY 189 | i_id_event, 190 | i_id_batch, 191 | v_table_name, 192 | v_schema_name, 193 | enm_binlog_event, 194 | jsb_event_data, 195 | jsb_event_update, 196 | t_query, 197 | ts_event_datetime 198 | ) t_columns 199 | <<<<<<< HEAD 200 | ======= 201 | array_agg(quote_ident(t_column)) AS t_colunm, 202 | array_agg(quote_literal(jsb_event_data->>t_column)) as t_event_data, 203 | array_agg(jsb_event_update->>t_column) as t_event_update, 204 | string_agg(distinct format('%I=%L',t_column,jsb_event_update->>t_column),',') as t_update, 205 | string_agg(distinct format('%I=%L',v_pkey,jsb_event_data->>v_pkey),' AND ') as t_pk_data, 206 | string_agg(distinct format('%I=%L',v_pkey,jsb_event_update->>v_pkey),' AND ') as t_pk_update, 207 | t_query 208 | FROM 209 | ( 210 | 211 | SELECT 212 | log.i_id_event, 213 | log.i_id_batch, 214 | log.v_table_name, 215 | log.v_schema_name, 216 | log.enm_binlog_event, 217 | log.jsb_event_data, 218 | log.jsb_event_update, 219 | log.t_query, 220 | replace(unnest(string_to_array(v_table_pkey[1],',')),'"','') as v_pkey, 221 | ts_event_datetime, 222 | (jsonb_each_text(coalesce(log.jsb_event_data,'{"foo":"bar"}'::jsonb))).key AS t_column 223 | 224 | 225 | FROM 226 | sch_chameleon.t_log_replica log 227 | INNER JOIN sch_chameleon.t_replica_tables tab 228 | ON 229 | tab.v_table_name=log.v_table_name 230 | AND tab.v_schema_name=log.v_schema_name 231 | WHERE 232 | log.i_id_batch=v_i_id_batch 233 | 234 | ) t_dat 235 | >>>>>>> new function seems to work properly 236 | ======= 237 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 238 | GROUP BY 239 | i_id_event, 240 | i_id_batch, 241 | v_table_name, 242 | v_schema_name, 243 | enm_binlog_event, 244 | t_query, 245 | <<<<<<< HEAD 246 | <<<<<<< 98364345ea29577df3de6311fc350e8f57876fe9 247 | ======= 248 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 249 | ts_event_datetime, 250 | t_pk_data, 251 | t_pk_update 252 | ) t_sql 253 | <<<<<<< HEAD 254 | ======= 255 | ts_event_datetime 256 | ORDER BY ts_event_datetime 257 | ) t_query 258 | >>>>>>> new function seems to work properly 259 | ======= 260 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 261 | LOOP 262 | EXECUTE v_r_rows.t_sql; 263 | IF v_r_rows.enm_binlog_event='ddl' 264 | THEN 265 | v_i_ddl:=v_i_ddl+1; 266 | ELSE 267 | v_i_replayed:=v_i_replayed+1; 268 | END IF; 269 | 270 | 271 | <<<<<<< HEAD 272 | <<<<<<< 98364345ea29577df3de6311fc350e8f57876fe9 273 | 274 | ======= 275 | DELETE FROM sch_chameleon.t_log_replica 276 | WHERE 277 | i_id_event=v_r_rows.i_id_event 278 | ; 279 | >>>>>>> new function seems to work properly 280 | ======= 281 | 282 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 283 | 284 | END LOOP; 285 | 286 | 287 | IF v_i_replayed=0 AND v_i_ddl=0 288 | THEN 289 | <<<<<<< HEAD 290 | <<<<<<< 4d3438102c559b55a0e3c42a1d5fb123edec5e61 291 | <<<<<<< 98364345ea29577df3de6311fc350e8f57876fe9 292 | ======= 293 | >>>>>>> improve performance for the replay plpgsql function 294 | ======= 295 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 296 | DELETE FROM sch_chameleon.t_log_replica 297 | WHERE 298 | i_id_batch=v_i_id_batch 299 | ; 300 | 301 | GET DIAGNOSTICS v_i_skipped = ROW_COUNT; 302 | <<<<<<< HEAD 303 | <<<<<<< 4d3438102c559b55a0e3c42a1d5fb123edec5e61 304 | ======= 305 | ======= 306 | 307 | >>>>>>> improve performance for the replay plpgsql function 308 | UPDATE ONLY sch_chameleon.t_replica_batch 309 | SET 310 | b_replayed=True, 311 | i_skipped=v_i_skipped, 312 | ts_replayed=clock_timestamp() 313 | 314 | WHERE 315 | i_id_batch=v_i_id_batch 316 | ; 317 | 318 | 319 | 320 | v_b_loop=False; 321 | ELSE 322 | UPDATE ONLY sch_chameleon.t_replica_batch 323 | SET 324 | i_ddl=coalesce(i_ddl,0)+v_i_ddl, 325 | i_replayed=coalesce(i_replayed,0)+v_i_replayed, 326 | ts_replayed=clock_timestamp() 327 | WHERE 328 | i_id_batch=v_r_rows.i_id_batch 329 | ; 330 | v_b_loop=True; 331 | END IF; 332 | >>>>>>> new function seems to work properly 333 | ======= 334 | >>>>>>> 16ec83d0b6f3e30ab282c65489405f1ae01609e9 335 | 336 | UPDATE ONLY sch_chameleon.t_replica_batch 337 | SET 338 | b_replayed=True, 339 | i_skipped=v_i_skipped, 340 | ts_replayed=clock_timestamp() 341 | 342 | WHERE 343 | i_id_batch=v_i_id_batch 344 | ; 345 | 346 | DELETE FROM sch_chameleon.t_batch_events 347 | WHERE 348 | i_id_batch=v_i_id_batch 349 | ; 350 | 351 | v_b_loop=False; 352 | ELSE 353 | UPDATE ONLY sch_chameleon.t_replica_batch 354 | SET 355 | i_ddl=coalesce(i_ddl,0)+v_i_ddl, 356 | i_replayed=coalesce(i_replayed,0)+v_i_replayed, 357 | ts_replayed=clock_timestamp() 358 | WHERE 359 | i_id_batch=v_r_rows.i_id_batch 360 | ; 361 | 362 | UPDATE sch_chameleon.t_batch_events 363 | SET 364 | i_id_event = v_i_evt_queue 365 | WHERE 366 | i_id_batch=v_i_id_batch 367 | ; 368 | 369 | DELETE FROM sch_chameleon.t_log_replica 370 | WHERE 371 | i_id_batch=v_i_id_batch 372 | AND i_id_event=ANY(v_i_evt_replay) 373 | ; 374 | 375 | v_b_loop=True; 376 | 377 | 378 | 379 | END IF; 380 | 381 | 382 | 383 | RETURN v_b_loop; 384 | 385 | 386 | END; 387 | $BODY$ 388 | LANGUAGE plpgsql; 389 | -------------------------------------------------------------------------------- /pg_chameleon/sql/dev/fn_replay_mysql_v3.sql: -------------------------------------------------------------------------------- 1 | -- fn_replay_mysql_v3.sql 2 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_replay_mysql_blocks(integer,integer,boolean) 3 | RETURNS sch_chameleon.ty_replay_status AS 4 | $BODY$ 5 | DECLARE 6 | p_i_max_events ALIAS FOR $1; 7 | p_i_id_source ALIAS FOR $2; 8 | p_b_exit_on_error ALIAS FOR $3; 9 | v_ty_status sch_chameleon.ty_replay_status; 10 | v_r_statements record; 11 | v_i_id_batch bigint; 12 | v_v_log_table text; 13 | v_t_ddl text; 14 | v_t_main_sql text; 15 | v_t_delete_sql text; 16 | v_i_replayed integer; 17 | v_i_skipped integer; 18 | v_i_ddl integer; 19 | v_i_evt_replay bigint[]; 20 | v_i_evt_queue bigint[]; 21 | v_ts_evt_source timestamp without time zone; 22 | v_tab_enabled boolean; 23 | 24 | BEGIN 25 | v_i_replayed:=0; 26 | v_i_ddl:=0; 27 | v_i_skipped:=0; 28 | v_ty_status.b_continue:=FALSE; 29 | 30 | RAISE DEBUG 'Searching batches to replay for source id: %', p_i_id_source; 31 | v_i_id_batch:= ( 32 | SELECT 33 | bat.i_id_batch 34 | FROM 35 | sch_chameleon.t_replica_batch bat 36 | INNER JOIN sch_chameleon.t_batch_events evt 37 | ON 38 | evt.i_id_batch=bat.i_id_batch 39 | WHERE 40 | bat.b_started 41 | AND bat.b_processed 42 | AND NOT bat.b_replayed 43 | AND bat.i_id_source=p_i_id_source 44 | ORDER BY 45 | bat.ts_created 46 | LIMIT 1 47 | ) 48 | ; 49 | 50 | v_v_log_table:=( 51 | SELECT 52 | v_log_table 53 | FROM 54 | sch_chameleon.t_replica_batch 55 | WHERE 56 | i_id_batch=v_i_id_batch 57 | ) 58 | ; 59 | IF v_i_id_batch IS NULL 60 | THEN 61 | RAISE DEBUG 'There are no batches available for replay'; 62 | RETURN v_ty_status; 63 | END IF; 64 | 65 | RAISE DEBUG 'Found id_batch %, data in log table %', v_i_id_batch,v_v_log_table; 66 | RAISE DEBUG 'Building a list of event id with max length %...', p_i_max_events; 67 | v_i_evt_replay:=( 68 | SELECT 69 | i_id_event[1:p_i_max_events] 70 | FROM 71 | sch_chameleon.t_batch_events 72 | WHERE 73 | i_id_batch=v_i_id_batch 74 | ); 75 | RAISE DEBUG 'got: % ',v_i_evt_replay; 76 | 77 | v_i_evt_queue:=( 78 | SELECT 79 | i_id_event[p_i_max_events+1:array_length(i_id_event,1)] 80 | FROM 81 | sch_chameleon.t_batch_events 82 | WHERE 83 | i_id_batch=v_i_id_batch 84 | ); 85 | 86 | RAISE DEBUG 'Finding the last executed event''s timestamp...'; 87 | v_ts_evt_source:=( 88 | SELECT 89 | to_timestamp(i_my_event_time) 90 | FROM 91 | sch_chameleon.t_log_replica 92 | WHERE 93 | i_id_event=v_i_evt_replay[array_length(v_i_evt_replay,1)] 94 | AND i_id_batch=v_i_id_batch 95 | ); 96 | 97 | RAISE DEBUG 'Generating the main loop sql'; 98 | 99 | v_t_main_sql:=format(' 100 | SELECT 101 | array_agg(i_id_event) AS i_id_event, 102 | count(enm_binlog_event) FILTER (WHERE enm_binlog_event = ''ddl'') as i_ddl, 103 | count(enm_binlog_event) FILTER (WHERE enm_binlog_event <> ''ddl'') as i_replayed, 104 | 105 | v_table_name, 106 | v_schema_name, 107 | string_agg( 108 | CASE 109 | WHEN enm_binlog_event = ''ddl'' 110 | THEN 111 | t_query 112 | WHEN enm_binlog_event = ''insert'' 113 | THEN 114 | format( 115 | ''INSERT INTO %%I.%%I %%s;'', 116 | v_schema_name, 117 | v_table_name, 118 | t_dec_data 119 | 120 | ) 121 | WHEN enm_binlog_event = ''update'' 122 | THEN 123 | format( 124 | ''UPDATE %%I.%%I SET %%s WHERE %%s;'', 125 | v_schema_name, 126 | v_table_name, 127 | t_dec_data, 128 | t_pk_data 129 | ) 130 | WHEN enm_binlog_event = ''delete'' 131 | THEN 132 | format( 133 | ''DELETE FROM %%I.%%I WHERE %%s;'', 134 | v_schema_name, 135 | v_table_name, 136 | t_pk_data 137 | ) 138 | 139 | END,'' '') AS t_sql 140 | FROM 141 | ( 142 | SELECT 143 | dec.i_id_event, 144 | dec.v_table_name, 145 | dec.v_schema_name, 146 | dec.enm_binlog_event, 147 | dec.t_query as t_query, 148 | dec.ts_event_datetime, 149 | CASE 150 | WHEN dec.enm_binlog_event = ''insert'' 151 | THEN 152 | format(''(%%s) VALUES (%%s)'',string_agg(format(''%%I'',dec.t_column),'',''),string_agg(format(''%%L'',jsb_event_after->>t_column),'','')) 153 | WHEN dec.enm_binlog_event = ''update'' 154 | THEN 155 | string_agg(format(''%%I=%%L'',dec.t_column,jsb_event_after->>t_column),'','') 156 | 157 | END AS t_dec_data, 158 | string_agg(DISTINCT format( 159 | ''%%I=%%L'', 160 | dec.v_table_pkey, 161 | CASE 162 | WHEN dec.enm_binlog_event = ''update'' 163 | THEN 164 | jsb_event_before->>v_table_pkey 165 | ELSE 166 | jsb_event_after->>v_table_pkey 167 | END 168 | ),'' AND '') as t_pk_data 169 | FROM 170 | ( 171 | SELECT 172 | log.i_id_event, 173 | log.v_table_name, 174 | log.v_schema_name, 175 | log.enm_binlog_event, 176 | coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb) as jsb_event_after, 177 | (jsonb_each_text(coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb))).key AS t_column, 178 | log.jsb_event_before, 179 | log.t_query as t_query, 180 | log.ts_event_datetime, 181 | unnest(v_table_pkey) as v_table_pkey 182 | FROM 183 | sch_chameleon.%I log 184 | INNER JOIN sch_chameleon.t_replica_tables tab 185 | ON 186 | tab.v_table_name=log.v_table_name 187 | AND tab.v_schema_name=log.v_schema_name 188 | WHERE 189 | tab.b_replica_enabled 190 | AND i_id_event = ANY(%L) 191 | ) dec 192 | GROUP BY 193 | dec.i_id_event, 194 | dec.v_table_name, 195 | dec.v_schema_name, 196 | dec.enm_binlog_event, 197 | dec.t_query, 198 | dec.ts_event_datetime 199 | ) par 200 | GROUP BY 201 | v_table_name, 202 | v_schema_name 203 | ; 204 | 205 | ',v_v_log_table,v_i_evt_replay); 206 | --RAISE DEBUG 'Generated SQL: %', v_t_main_sql; 207 | FOR v_r_statements IN EXECUTE v_t_main_sql 208 | LOOP 209 | RAISE DEBUG 'Replaying data for table %.%', v_r_statements.v_schema_name,v_r_statements.v_table_name; 210 | BEGIN 211 | EXECUTE v_r_statements.t_sql; 212 | v_i_ddl:=v_i_ddl+v_r_statements.i_ddl; 213 | v_i_replayed:=v_i_replayed+v_r_statements.i_replayed; 214 | v_t_delete_sql:=format('DELETE FROM sch_chameleon.%I WHERE i_id_event=ANY(%L);',v_v_log_table,v_r_statements.i_id_event); 215 | RAISE DEBUG 'DELETING THE PROCESSED ROWS'; 216 | 217 | EXCEPTION 218 | WHEN OTHERS 219 | THEN 220 | RAISE NOTICE 'An error occurred when replaying data for the table %.%',v_r_statements.v_schema_name,v_r_statements.v_table_name; 221 | RAISE NOTICE 'SQLSTATE: % - ERROR MESSAGE %',SQLSTATE, SQLERRM; 222 | RAISE DEBUG 'SQL EXECUTED: % ',v_r_statements.t_sql; 223 | RAISE NOTICE 'The table %.% has been removed from the replica',v_r_statements.v_schema_name,v_r_statements.v_table_name; 224 | UPDATE sch_chameleon.t_replica_tables 225 | SET b_replica_enabled=False 226 | WHERE 227 | v_table_name=v_r_statements.v_table_name 228 | AND v_schema_name=v_r_statements.v_schema_name 229 | ; 230 | v_ty_status.v_table_error:=array_append(v_ty_status.v_table_error, format('%I.%I SQLSTATE: %s - ERROR MESSAGE: %s',v_r_statements.v_schema_name,v_r_statements.v_table_name,SQLSTATE, SQLERRM)::character varying) ; 231 | RAISE NOTICE 'Adding error log entry for table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 232 | END; 233 | END LOOP; 234 | IF v_ts_evt_source IS NOT NULL 235 | THEN 236 | UPDATE sch_chameleon.t_last_replayed 237 | SET 238 | ts_last_replayed=v_ts_evt_source 239 | WHERE 240 | i_id_source=p_i_id_source 241 | ; 242 | END IF; 243 | IF v_i_replayed=0 AND v_i_ddl=0 244 | THEN 245 | DELETE FROM sch_chameleon.t_log_replica 246 | WHERE 247 | i_id_batch=v_i_id_batch 248 | ; 249 | 250 | GET DIAGNOSTICS v_i_skipped = ROW_COUNT; 251 | 252 | UPDATE ONLY sch_chameleon.t_replica_batch 253 | SET 254 | b_replayed=True, 255 | i_skipped=v_i_skipped, 256 | ts_replayed=clock_timestamp() 257 | 258 | WHERE 259 | i_id_batch=v_i_id_batch 260 | ; 261 | 262 | DELETE FROM sch_chameleon.t_batch_events 263 | WHERE 264 | i_id_batch=v_i_id_batch 265 | ; 266 | 267 | v_ty_status.b_continue:=FALSE; 268 | ELSE 269 | UPDATE ONLY sch_chameleon.t_replica_batch 270 | SET 271 | i_ddl=coalesce(i_ddl,0)+v_i_ddl, 272 | i_replayed=coalesce(i_replayed,0)+v_i_replayed, 273 | i_skipped=v_i_skipped, 274 | ts_replayed=clock_timestamp() 275 | 276 | WHERE 277 | i_id_batch=v_i_id_batch 278 | ; 279 | 280 | UPDATE sch_chameleon.t_batch_events 281 | SET 282 | i_id_event = v_i_evt_queue 283 | WHERE 284 | i_id_batch=v_i_id_batch 285 | ; 286 | 287 | DELETE FROM sch_chameleon.t_log_replica 288 | WHERE 289 | i_id_batch=v_i_id_batch 290 | AND i_id_event=ANY(v_i_evt_replay) 291 | ; 292 | v_ty_status.b_continue:=TRUE; 293 | RETURN v_ty_status; 294 | END IF; 295 | v_i_id_batch:= ( 296 | SELECT 297 | bat.i_id_batch 298 | FROM 299 | sch_chameleon.t_replica_batch bat 300 | INNER JOIN sch_chameleon.t_batch_events evt 301 | ON 302 | evt.i_id_batch=bat.i_id_batch 303 | WHERE 304 | bat.b_started 305 | AND bat.b_processed 306 | AND NOT bat.b_replayed 307 | AND bat.i_id_source=p_i_id_source 308 | ORDER BY 309 | bat.ts_created 310 | LIMIT 1 311 | ) 312 | ; 313 | 314 | IF v_i_id_batch IS NOT NULL 315 | THEN 316 | v_ty_status.b_continue:=TRUE; 317 | END IF; 318 | 319 | 320 | RETURN v_ty_status; 321 | 322 | 323 | 324 | END; 325 | 326 | $BODY$ 327 | LANGUAGE plpgsql; 328 | -------------------------------------------------------------------------------- /pg_chameleon/sql/dev/fn_replay_mysql_v3_stream.sql: -------------------------------------------------------------------------------- 1 | -- fn_replay_mysql_v3.sql 2 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_replay_mysql(integer,integer,boolean) 3 | RETURNS sch_chameleon.ty_replay_status AS 4 | $BODY$ 5 | DECLARE 6 | p_i_max_events ALIAS FOR $1; 7 | p_i_id_source ALIAS FOR $2; 8 | p_b_exit_on_error ALIAS FOR $3; 9 | v_ty_status sch_chameleon.ty_replay_status; 10 | v_r_statements record; 11 | v_i_id_batch bigint; 12 | v_v_log_table text; 13 | v_t_ddl text; 14 | v_t_main_sql text; 15 | v_t_delete_sql text; 16 | v_i_replayed integer; 17 | v_i_skipped integer; 18 | v_i_ddl integer; 19 | v_i_evt_replay bigint[]; 20 | v_i_evt_queue bigint[]; 21 | v_ts_evt_source timestamp without time zone; 22 | v_tab_enabled boolean; 23 | 24 | BEGIN 25 | v_i_replayed:=0; 26 | v_i_ddl:=0; 27 | v_i_skipped:=0; 28 | v_ty_status.b_continue:=FALSE; 29 | v_ty_status.b_error:=FALSE; 30 | RAISE DEBUG 'Searching batches to replay for source id: %', p_i_id_source; 31 | v_i_id_batch:= ( 32 | SELECT 33 | bat.i_id_batch 34 | FROM 35 | sch_chameleon.t_replica_batch bat 36 | INNER JOIN sch_chameleon.t_batch_events evt 37 | ON 38 | evt.i_id_batch=bat.i_id_batch 39 | WHERE 40 | bat.b_started 41 | AND bat.b_processed 42 | AND NOT bat.b_replayed 43 | AND bat.i_id_source=p_i_id_source 44 | ORDER BY 45 | bat.ts_created 46 | LIMIT 1 47 | ) 48 | ; 49 | 50 | v_v_log_table:=( 51 | SELECT 52 | v_log_table 53 | FROM 54 | sch_chameleon.t_replica_batch 55 | WHERE 56 | i_id_batch=v_i_id_batch 57 | ) 58 | ; 59 | IF v_i_id_batch IS NULL 60 | THEN 61 | RAISE DEBUG 'There are no batches available for replay'; 62 | RETURN v_ty_status; 63 | END IF; 64 | 65 | RAISE DEBUG 'Found id_batch %, data in log table %', v_i_id_batch,v_v_log_table; 66 | RAISE DEBUG 'Building a list of event id with max length %...', p_i_max_events; 67 | v_i_evt_replay:=( 68 | SELECT 69 | i_id_event[1:p_i_max_events] 70 | FROM 71 | sch_chameleon.t_batch_events 72 | WHERE 73 | i_id_batch=v_i_id_batch 74 | ); 75 | 76 | 77 | v_i_evt_queue:=( 78 | SELECT 79 | i_id_event[p_i_max_events+1:array_length(i_id_event,1)] 80 | FROM 81 | sch_chameleon.t_batch_events 82 | WHERE 83 | i_id_batch=v_i_id_batch 84 | ); 85 | 86 | RAISE DEBUG 'Finding the last executed event''s timestamp...'; 87 | v_ts_evt_source:=( 88 | SELECT 89 | to_timestamp(i_my_event_time) 90 | FROM 91 | sch_chameleon.t_log_replica 92 | WHERE 93 | i_id_event=v_i_evt_replay[array_length(v_i_evt_replay,1)] 94 | AND i_id_batch=v_i_id_batch 95 | ); 96 | 97 | RAISE DEBUG 'Generating the main loop sql'; 98 | 99 | v_t_main_sql:=format(' 100 | SELECT 101 | i_id_event AS i_id_event, 102 | enm_binlog_event, 103 | (enm_binlog_event=''ddl'')::integer as i_ddl, 104 | (enm_binlog_event<>''ddl'')::integer as i_replay, 105 | t_binlog_name, 106 | i_binlog_position, 107 | v_table_name, 108 | v_schema_name, 109 | t_pk_data, 110 | CASE 111 | WHEN enm_binlog_event = ''ddl'' 112 | THEN 113 | t_query 114 | WHEN enm_binlog_event = ''insert'' 115 | THEN 116 | format( 117 | ''INSERT INTO %%I.%%I %%s;'', 118 | v_schema_name, 119 | v_table_name, 120 | t_dec_data 121 | 122 | ) 123 | WHEN enm_binlog_event = ''update'' 124 | THEN 125 | format( 126 | ''UPDATE %%I.%%I SET %%s WHERE %%s;'', 127 | v_schema_name, 128 | v_table_name, 129 | t_dec_data, 130 | t_pk_data 131 | ) 132 | WHEN enm_binlog_event = ''delete'' 133 | THEN 134 | format( 135 | ''DELETE FROM %%I.%%I WHERE %%s;'', 136 | v_schema_name, 137 | v_table_name, 138 | t_pk_data 139 | ) 140 | 141 | END AS t_sql 142 | FROM 143 | ( 144 | SELECT 145 | dec.i_id_event, 146 | dec.v_table_name, 147 | dec.v_schema_name, 148 | dec.enm_binlog_event, 149 | dec.t_binlog_name, 150 | dec.i_binlog_position, 151 | dec.t_query as t_query, 152 | dec.ts_event_datetime, 153 | CASE 154 | WHEN dec.enm_binlog_event = ''insert'' 155 | THEN 156 | format(''(%%s) VALUES (%%s)'',string_agg(format(''%%I'',dec.t_column),'',''),string_agg(format(''%%L'',jsb_event_after->>t_column),'','')) 157 | WHEN dec.enm_binlog_event = ''update'' 158 | THEN 159 | string_agg(format(''%%I=%%L'',dec.t_column,jsb_event_after->>t_column),'','') 160 | 161 | END AS t_dec_data, 162 | string_agg(DISTINCT format( 163 | ''%%I=%%L'', 164 | dec.v_table_pkey, 165 | CASE 166 | WHEN dec.enm_binlog_event = ''update'' 167 | THEN 168 | jsb_event_before->>v_table_pkey 169 | ELSE 170 | jsb_event_after->>v_table_pkey 171 | END 172 | ),'' AND '') as t_pk_data 173 | FROM 174 | ( 175 | SELECT 176 | log.i_id_event, 177 | log.v_table_name, 178 | log.v_schema_name, 179 | log.enm_binlog_event, 180 | log.t_binlog_name, 181 | log.i_binlog_position, 182 | coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb) as jsb_event_after, 183 | (jsonb_each_text(coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb))).key AS t_column, 184 | log.jsb_event_before, 185 | log.t_query as t_query, 186 | log.ts_event_datetime, 187 | unnest(v_table_pkey) as v_table_pkey 188 | FROM 189 | sch_chameleon.%I log 190 | INNER JOIN sch_chameleon.t_replica_tables tab 191 | ON 192 | tab.v_table_name=log.v_table_name 193 | AND tab.v_schema_name=log.v_schema_name 194 | WHERE 195 | tab.b_replica_enabled 196 | AND i_id_event = ANY(%L) 197 | ) dec 198 | GROUP BY 199 | dec.i_id_event, 200 | dec.v_table_name, 201 | dec.v_schema_name, 202 | dec.enm_binlog_event, 203 | dec.t_query, 204 | dec.ts_event_datetime, 205 | dec.t_binlog_name, 206 | dec.i_binlog_position 207 | ) par 208 | ORDER BY 209 | i_id_event ASC 210 | ; 211 | 212 | ',v_v_log_table,v_i_evt_replay); 213 | 214 | FOR v_r_statements IN EXECUTE v_t_main_sql 215 | LOOP 216 | 217 | BEGIN 218 | EXECUTE v_r_statements.t_sql; 219 | v_i_ddl:=v_i_ddl+v_r_statements.i_ddl; 220 | v_i_replayed:=v_i_replayed+v_r_statements.i_replay; 221 | 222 | 223 | EXCEPTION 224 | WHEN OTHERS 225 | THEN 226 | RAISE NOTICE 'An error occurred when replaying data for the table %.%',v_r_statements.v_schema_name,v_r_statements.v_table_name; 227 | RAISE NOTICE 'SQLSTATE: % - ERROR MESSAGE %',SQLSTATE, SQLERRM; 228 | RAISE DEBUG 'SQL EXECUTED: % ',v_r_statements.t_sql; 229 | RAISE NOTICE 'The table %.% has been removed from the replica',v_r_statements.v_schema_name,v_r_statements.v_table_name; 230 | v_ty_status.v_table_error:=array_append(v_ty_status.v_table_error, format('%I.%I SQLSTATE: %s - ERROR MESSAGE: %s',v_r_statements.v_schema_name,v_r_statements.v_table_name,SQLSTATE, SQLERRM)::character varying) ; 231 | RAISE NOTICE 'Adding error log entry for table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 232 | INSERT INTO sch_chameleon.t_error_log 233 | ( 234 | i_id_batch, 235 | i_id_source, 236 | v_schema_name, 237 | v_table_name, 238 | t_table_pkey, 239 | t_binlog_name, 240 | i_binlog_position, 241 | ts_error, 242 | t_sql, 243 | t_error_message 244 | ) 245 | SELECT 246 | i_id_batch, 247 | p_i_id_source, 248 | v_schema_name, 249 | v_table_name, 250 | v_r_statements.t_pk_data as t_table_pkey, 251 | t_binlog_name, 252 | i_binlog_position, 253 | clock_timestamp(), 254 | quote_literal(v_r_statements.t_sql) as t_sql, 255 | format('%s - %s',SQLSTATE, SQLERRM) as t_error_message 256 | FROM 257 | sch_chameleon.t_log_replica log 258 | WHERE 259 | log.i_id_event=v_r_statements.i_id_event 260 | ; 261 | IF p_b_exit_on_error 262 | THEN 263 | v_ty_status.b_continue:=FALSE; 264 | v_ty_status.b_error:=TRUE; 265 | RETURN v_ty_status; 266 | ELSE 267 | 268 | RAISE NOTICE 'Statement %', v_r_statements.t_sql; 269 | UPDATE sch_chameleon.t_replica_tables 270 | SET 271 | b_replica_enabled=FALSE 272 | WHERE 273 | v_schema_name=v_r_statements.v_schema_name 274 | AND v_table_name=v_r_statements.v_table_name 275 | ; 276 | 277 | RAISE NOTICE 'Deleting the log entries for the table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 278 | DELETE FROM sch_chameleon.t_log_replica log 279 | WHERE 280 | v_table_name=v_r_statements.v_table_name 281 | AND v_schema_name=v_r_statements.v_schema_name 282 | AND i_id_batch=v_i_id_batch 283 | ; 284 | END IF; 285 | END; 286 | END LOOP; 287 | IF v_ts_evt_source IS NOT NULL 288 | THEN 289 | UPDATE sch_chameleon.t_last_replayed 290 | SET 291 | ts_last_replayed=v_ts_evt_source 292 | WHERE 293 | i_id_source=p_i_id_source 294 | ; 295 | END IF; 296 | IF v_i_replayed=0 AND v_i_ddl=0 297 | THEN 298 | DELETE FROM sch_chameleon.t_log_replica 299 | WHERE 300 | i_id_batch=v_i_id_batch 301 | ; 302 | 303 | GET DIAGNOSTICS v_i_skipped = ROW_COUNT; 304 | RAISE DEBUG 'SKIPPED ROWS: % ',v_i_skipped; 305 | 306 | UPDATE ONLY sch_chameleon.t_replica_batch 307 | SET 308 | b_replayed=True, 309 | i_skipped=v_i_skipped, 310 | ts_replayed=clock_timestamp() 311 | 312 | WHERE 313 | i_id_batch=v_i_id_batch 314 | ; 315 | 316 | DELETE FROM sch_chameleon.t_batch_events 317 | WHERE 318 | i_id_batch=v_i_id_batch 319 | ; 320 | 321 | v_ty_status.b_continue:=FALSE; 322 | ELSE 323 | UPDATE ONLY sch_chameleon.t_replica_batch 324 | SET 325 | i_ddl=coalesce(i_ddl,0)+v_i_ddl, 326 | i_replayed=coalesce(i_replayed,0)+v_i_replayed, 327 | i_skipped=v_i_skipped, 328 | ts_replayed=clock_timestamp() 329 | 330 | WHERE 331 | i_id_batch=v_i_id_batch 332 | ; 333 | 334 | UPDATE sch_chameleon.t_batch_events 335 | SET 336 | i_id_event = v_i_evt_queue 337 | WHERE 338 | i_id_batch=v_i_id_batch 339 | ; 340 | 341 | DELETE FROM sch_chameleon.t_log_replica 342 | WHERE 343 | i_id_batch=v_i_id_batch 344 | AND i_id_event=ANY(v_i_evt_replay) 345 | ; 346 | v_ty_status.b_continue:=TRUE; 347 | RETURN v_ty_status; 348 | END IF; 349 | v_i_id_batch:= ( 350 | SELECT 351 | bat.i_id_batch 352 | FROM 353 | sch_chameleon.t_replica_batch bat 354 | INNER JOIN sch_chameleon.t_batch_events evt 355 | ON 356 | evt.i_id_batch=bat.i_id_batch 357 | WHERE 358 | bat.b_started 359 | AND bat.b_processed 360 | AND NOT bat.b_replayed 361 | AND bat.i_id_source=p_i_id_source 362 | ORDER BY 363 | bat.ts_created 364 | LIMIT 1 365 | ) 366 | ; 367 | 368 | IF v_i_id_batch IS NOT NULL 369 | THEN 370 | v_ty_status.b_continue:=TRUE; 371 | END IF; 372 | 373 | 374 | RETURN v_ty_status; 375 | 376 | 377 | 378 | END; 379 | 380 | $BODY$ 381 | LANGUAGE plpgsql; 382 | -------------------------------------------------------------------------------- /pg_chameleon/sql/drop_schema.sql: -------------------------------------------------------------------------------- 1 | --drop schema 2 | DROP SCHEMA IF EXISTS sch_chameleon CASCADE; 3 | -------------------------------------------------------------------------------- /pg_chameleon/sql/fn_process_batch.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_process_batch(integer) 2 | RETURNS BOOLEAN AS 3 | $BODY$ 4 | DECLARE 5 | p_max_events ALIAS FOR $1; 6 | v_r_rows record; 7 | v_t_fields text[]; 8 | v_t_values text[]; 9 | v_t_sql_rep text; 10 | v_t_pkey text; 11 | v_t_vals text; 12 | v_t_update text; 13 | v_t_ins_fld text; 14 | v_t_ins_val text; 15 | v_t_ddl text; 16 | v_b_loop boolean; 17 | v_i_id_batch integer; 18 | v_i_replayed integer; 19 | v_i_skipped integer; 20 | 21 | BEGIN 22 | v_b_loop:=True; 23 | v_i_replayed=0; 24 | FOR v_r_rows IN WITH t_batch AS 25 | ( 26 | SELECT 27 | i_id_batch 28 | FROM ONLY 29 | sch_chameleon.t_replica_batch 30 | WHERE 31 | b_started 32 | AND b_processed 33 | AND NOT b_replayed 34 | ORDER BY 35 | ts_created 36 | LIMIT 1 37 | ), 38 | t_events AS 39 | ( 40 | SELECT 41 | log.i_id_event, 42 | bat.i_id_batch, 43 | log.v_table_name, 44 | log.v_schema_name, 45 | log.enm_binlog_event, 46 | log.jsb_event_data, 47 | log.jsb_event_update, 48 | log.t_query, 49 | tab.v_table_pkey as v_pkey_where, 50 | replace(array_to_string(tab.v_table_pkey,','),'"','') as t_pkeys, 51 | array_length(tab.v_table_pkey,1) as i_pkeys 52 | FROM 53 | sch_chameleon.t_log_replica log 54 | INNER JOIN sch_chameleon.t_replica_tables tab 55 | ON 56 | tab.v_table_name=log.v_table_name 57 | AND tab.v_schema_name=log.v_schema_name 58 | INNER JOIN t_batch bat 59 | ON bat.i_id_batch=log.i_id_batch 60 | 61 | ORDER BY ts_event_datetime 62 | LIMIT p_max_events 63 | ) 64 | SELECT 65 | i_id_event, 66 | i_id_batch, 67 | v_table_name, 68 | v_schema_name, 69 | enm_binlog_event, 70 | jsb_event_data, 71 | jsb_event_update, 72 | t_query, 73 | string_to_array(t_pkeys,',') as v_table_pkey, 74 | array_to_string(v_pkey_where,',') as v_pkey_where, 75 | t_pkeys, 76 | i_pkeys 77 | FROM 78 | t_events 79 | LOOP 80 | 81 | IF v_r_rows.enm_binlog_event='ddl' 82 | THEN 83 | v_t_ddl=format('SET search_path=%I;%s',v_r_rows.v_schema_name,v_r_rows.t_query); 84 | RAISE DEBUG 'DDL: %',v_t_ddl; 85 | EXECUTE v_t_ddl; 86 | DELETE FROM sch_chameleon.t_log_replica 87 | WHERE 88 | i_id_event=v_r_rows.i_id_event 89 | ; 90 | UPDATE ONLY sch_chameleon.t_replica_batch 91 | SET 92 | i_ddl=coalesce(i_ddl,0)+1 93 | WHERE 94 | i_id_batch=v_r_rows.i_id_batch 95 | ; 96 | ELSE 97 | SELECT 98 | array_agg(key) evt_fields, 99 | array_agg(value) evt_values 100 | INTO 101 | v_t_fields, 102 | v_t_values 103 | FROM ( 104 | SELECT 105 | key , 106 | value 107 | FROM 108 | jsonb_each_text(v_r_rows.jsb_event_data) js_event 109 | ) js_dat 110 | ; 111 | 112 | 113 | WITH t_jsb AS 114 | ( 115 | SELECT 116 | CASE 117 | WHEN v_r_rows.enm_binlog_event='update' 118 | THEN 119 | v_r_rows.jsb_event_update 120 | ELSE 121 | v_r_rows.jsb_event_data 122 | END jsb_event_data , 123 | v_r_rows.v_table_pkey v_table_pkey 124 | ), 125 | t_subscripts AS 126 | ( 127 | SELECT 128 | generate_subscripts(v_table_pkey,1) sub 129 | FROM 130 | t_jsb 131 | ) 132 | SELECT 133 | array_to_string(v_table_pkey,','), 134 | ''''||array_to_string(array_agg((jsb_event_data->>v_table_pkey[sub])::text),''',''')||'''' as pk_value 135 | INTO 136 | v_t_pkey, 137 | v_t_vals 138 | 139 | FROM 140 | t_subscripts,t_jsb 141 | GROUP BY v_table_pkey 142 | ; 143 | 144 | RAISE DEBUG '% % % % % %',v_r_rows.v_table_name, 145 | v_r_rows.v_schema_name, 146 | v_r_rows.v_table_pkey, 147 | v_r_rows.enm_binlog_event,v_t_fields,v_t_values; 148 | IF v_r_rows.enm_binlog_event='delete' 149 | THEN 150 | v_t_sql_rep=format('DELETE FROM %I.%I WHERE (%s)=(%s) ;', 151 | v_r_rows.v_schema_name, 152 | v_r_rows.v_table_name, 153 | v_r_rows.v_pkey_where, 154 | v_t_vals 155 | ); 156 | RAISE DEBUG '%',v_t_sql_rep; 157 | ELSEIF v_r_rows.enm_binlog_event='update' 158 | THEN 159 | SELECT 160 | array_to_string(array_agg(format('%I=%L',t_field,t_value)),',') 161 | INTO 162 | v_t_update 163 | FROM 164 | ( 165 | SELECT 166 | unnest(v_t_fields) t_field, 167 | unnest(v_t_values) t_value 168 | ) t_val 169 | ; 170 | 171 | v_t_sql_rep=format('UPDATE %I.%I 172 | SET 173 | %s 174 | WHERE (%s)=(%s) ;', 175 | v_r_rows.v_schema_name, 176 | v_r_rows.v_table_name, 177 | v_t_update, 178 | v_r_rows.v_pkey_where, 179 | v_t_vals 180 | ); 181 | RAISE DEBUG '%',v_t_sql_rep; 182 | ELSEIF v_r_rows.enm_binlog_event='insert' 183 | THEN 184 | SELECT 185 | array_to_string(array_agg(format('%I',t_field)),',') t_field, 186 | array_to_string(array_agg(format('%L',t_value)),',') t_value 187 | INTO 188 | v_t_ins_fld, 189 | v_t_ins_val 190 | FROM 191 | ( 192 | SELECT 193 | unnest(v_t_fields) t_field, 194 | unnest(v_t_values) t_value 195 | ) t_val 196 | ; 197 | v_t_sql_rep=format('INSERT INTO %I.%I 198 | ( 199 | %s 200 | ) 201 | VALUES 202 | ( 203 | %s 204 | ) 205 | ;', 206 | v_r_rows.v_schema_name, 207 | v_r_rows.v_table_name, 208 | v_t_ins_fld, 209 | v_t_ins_val 210 | 211 | ); 212 | 213 | RAISE DEBUG '%',v_t_sql_rep; 214 | END IF; 215 | EXECUTE v_t_sql_rep; 216 | 217 | DELETE FROM sch_chameleon.t_log_replica 218 | WHERE 219 | i_id_event=v_r_rows.i_id_event 220 | ; 221 | v_i_replayed=v_i_replayed+1; 222 | v_i_id_batch=v_r_rows.i_id_batch; 223 | 224 | END IF; 225 | END LOOP; 226 | IF v_i_replayed>0 227 | THEN 228 | UPDATE ONLY sch_chameleon.t_replica_batch 229 | SET 230 | i_replayed=v_i_replayed, 231 | ts_replayed=clock_timestamp() 232 | 233 | WHERE 234 | i_id_batch=v_i_id_batch 235 | ; 236 | END IF; 237 | 238 | IF v_r_rows IS NULL 239 | THEN 240 | RAISE DEBUG 'v_r_rows: %',v_r_rows.i_id_event; 241 | v_b_loop=False; 242 | 243 | 244 | UPDATE ONLY sch_chameleon.t_replica_batch 245 | SET 246 | b_replayed=True, 247 | ts_replayed=clock_timestamp() 248 | 249 | WHERE 250 | i_id_batch=( 251 | SELECT 252 | i_id_batch 253 | FROM ONLY 254 | sch_chameleon.t_replica_batch 255 | WHERE 256 | b_started 257 | AND b_processed 258 | AND NOT b_replayed 259 | ORDER BY 260 | ts_created 261 | LIMIT 1 262 | ) 263 | RETURNING i_id_batch INTO v_i_id_batch 264 | ; 265 | 266 | DELETE FROM sch_chameleon.t_log_replica 267 | WHERE 268 | i_id_batch=v_i_id_batch 269 | ; 270 | 271 | GET DIAGNOSTICS v_i_skipped = ROW_COUNT; 272 | UPDATE ONLY sch_chameleon.t_replica_batch 273 | SET 274 | i_skipped=v_i_skipped 275 | WHERE 276 | i_id_batch=v_i_id_batch 277 | ; 278 | SELECT 279 | count(*)>0 280 | INTO 281 | v_b_loop 282 | FROM ONLY 283 | sch_chameleon.t_replica_batch 284 | WHERE 285 | b_started 286 | AND b_processed 287 | AND NOT b_replayed 288 | ; 289 | 290 | END IF; 291 | 292 | RETURN v_b_loop ; 293 | END; 294 | $BODY$ 295 | LANGUAGE plpgsql; 296 | -------------------------------------------------------------------------------- /pg_chameleon/sql/fn_replay_data.sql: -------------------------------------------------------------------------------- 1 | --select * from sch_chameleon.t_replica_batch 2 | --delete from sch_chameleon.t_replica_batch where i_id_batch=1 3 | --REPLAY FUNCTION V2 4 | SELECT 5 | bat.i_id_batch 6 | FROM 7 | sch_chameleon.t_replica_batch bat 8 | INNER JOIN sch_chameleon.t_batch_events evt 9 | ON 10 | evt.i_id_batch=bat.i_id_batch 11 | WHERE 12 | bat.b_started 13 | AND bat.b_processed 14 | AND NOT bat.b_replayed 15 | AND bat.i_id_source=1 16 | ORDER BY 17 | bat.ts_created 18 | LIMIT 1 19 | ; 20 | 21 | SELECT 22 | i_id_event[1:5] 23 | FROM 24 | sch_chameleon.t_batch_events 25 | WHERE 26 | i_id_batch=3 27 | ; 28 | 29 | SELECT 30 | i_id_event[5+1:array_length(i_id_event,1)] 31 | FROM 32 | sch_chameleon.t_batch_events 33 | WHERE 34 | i_id_batch=3 35 | ; 36 | 37 | SELECT 38 | to_timestamp(i_my_event_time) 39 | FROM 40 | sch_chameleon.t_log_replica 41 | WHERE 42 | i_id_event=7 43 | AND i_id_batch=3 44 | 45 | ; 46 | 47 | WITH 48 | t_tables AS 49 | ( 50 | SELECT 51 | v_table_name, 52 | v_schema_name, 53 | unnest(v_table_pkey) as v_table_pkey 54 | FROM 55 | sch_chameleon.t_replica_tables 56 | WHERE 57 | b_replica_enabled 58 | ), 59 | t_events AS 60 | ( 61 | SELECT 62 | i_id_event 63 | FROM 64 | unnest('{3,4,5,6,7}'::bigint[]) AS i_id_event 65 | ) 66 | SELECT 67 | CASE 68 | WHEN enm_binlog_event = 'ddl' 69 | THEN 70 | t_query 71 | WHEN enm_binlog_event = 'insert' 72 | THEN 73 | format( 74 | 'INSERT INTO %I.%I (%s) VALUES (%s);', 75 | v_schema_name, 76 | v_table_name, 77 | array_to_string(t_colunm,','), 78 | array_to_string(t_event_data,',') 79 | 80 | ) 81 | WHEN enm_binlog_event = 'update' 82 | THEN 83 | format( 84 | 'UPDATE %I.%I SET %s WHERE %s;', 85 | v_schema_name, 86 | v_table_name, 87 | t_update, 88 | t_pk_update 89 | ) 90 | WHEN enm_binlog_event = 'delete' 91 | THEN 92 | format( 93 | 'DELETE FROM %I.%I WHERE %s;', 94 | v_schema_name, 95 | v_table_name, 96 | t_pk_data 97 | ) 98 | 99 | END AS t_sql, 100 | i_id_event, 101 | i_id_batch, 102 | enm_binlog_event, 103 | v_schema_name, 104 | v_table_name 105 | FROM 106 | ( 107 | SELECT 108 | i_id_event, 109 | i_id_batch, 110 | v_table_name, 111 | v_schema_name, 112 | enm_binlog_event, 113 | t_query, 114 | ts_event_datetime, 115 | t_pk_data, 116 | t_pk_update, 117 | array_agg(quote_ident(t_column)) AS t_colunm, 118 | string_agg(distinct format('%I=%L',t_column,jsb_event_after->>t_column),',') as t_update, 119 | array_agg(quote_nullable(jsb_event_after->>t_column)) as t_event_data 120 | FROM 121 | ( 122 | SELECT 123 | i_id_event, 124 | i_id_batch, 125 | v_table_name, 126 | v_schema_name, 127 | enm_binlog_event, 128 | jsb_event_after, 129 | jsb_event_before, 130 | t_query, 131 | ts_event_datetime, 132 | string_agg(distinct format('%I=%L',v_pkey,jsb_event_after->>v_pkey),' AND ') as t_pk_data, 133 | string_agg(distinct format('%I=%L',v_pkey,jsb_event_before->>v_pkey),' AND ') as t_pk_update, 134 | (jsonb_each_text(coalesce(jsb_event_after,'{"foo":"bar"}'::jsonb))).key AS t_column 135 | FROM 136 | ( 137 | SELECT 138 | log.i_id_event, 139 | log.i_id_batch, 140 | log.v_table_name, 141 | log.v_schema_name, 142 | log.enm_binlog_event, 143 | log.jsb_event_after, 144 | log.jsb_event_before, 145 | log.t_query, 146 | ts_event_datetime, 147 | v_table_pkey as v_pkey 148 | 149 | 150 | 151 | FROM 152 | sch_chameleon.t_log_replica log 153 | INNER JOIN t_tables tab 154 | ON 155 | tab.v_table_name=log.v_table_name 156 | AND tab.v_schema_name=log.v_schema_name 157 | INNER JOIN t_events evt 158 | ON log.i_id_event=evt.i_id_event 159 | ) t_pkey 160 | GROUP BY 161 | i_id_event, 162 | i_id_batch, 163 | v_table_name, 164 | v_schema_name, 165 | enm_binlog_event, 166 | jsb_event_after, 167 | jsb_event_before, 168 | t_query, 169 | ts_event_datetime 170 | ) t_columns 171 | GROUP BY 172 | i_id_event, 173 | i_id_batch, 174 | v_table_name, 175 | v_schema_name, 176 | enm_binlog_event, 177 | t_query, 178 | ts_event_datetime, 179 | t_pk_data, 180 | t_pk_update 181 | ) t_sql 182 | ORDER BY i_id_event -------------------------------------------------------------------------------- /pg_chameleon/sql/get_fkeys.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | table_name, 3 | constraint_name, 4 | referenced_table_name, 5 | referenced_table_schema, 6 | GROUP_CONCAT(concat('"',column_name,'"') ORDER BY POSITION_IN_UNIQUE_CONSTRAINT) as fk_cols, 7 | GROUP_CONCAT(concat('"',REFERENCED_COLUMN_NAME,'"') ORDER BY POSITION_IN_UNIQUE_CONSTRAINT) as ref_columns 8 | FROM 9 | information_schema.key_column_usage 10 | WHERE 11 | table_schema='obfuscated_dummy' 12 | AND referenced_table_name IS NOT NULL 13 | GROUP BY 14 | table_name, 15 | constraint_name, 16 | referenced_table_name, 17 | referenced_table_schema 18 | ORDER BY 19 | table_name, 20 | constraint_name, 21 | ordinal_position 22 | ; -------------------------------------------------------------------------------- /pg_chameleon/sql/scratch.sql: -------------------------------------------------------------------------------- 1 | select 2 | logdat[1] as prefix, 3 | logdat[2]::integer as sequence 4 | from 5 | ( 6 | select 7 | string_to_array(t_binlog_name,'.') logdat 8 | from 9 | sch_chameleon.t_replica_batch 10 | ) log 11 | -------------------------------------------------------------------------------- /pg_chameleon/sql/scratch3.sql: -------------------------------------------------------------------------------- 1 | --select * from sch_chameleon.t_sources; 2 | set client_min_messages=debug; 3 | --select * from sch_chameleon.fn_replay_mysql(1000,1,true) 4 | 5 | 6 | SELECT 7 | i_id_event AS i_id_event, 8 | enm_binlog_event, 9 | (enm_binlog_event='ddl')::integer as i_ddl, 10 | (enm_binlog_event<>'ddl')::integer as i_replay, 11 | t_binlog_name, 12 | i_binlog_position, 13 | v_table_name, 14 | v_schema_name, 15 | t_pk_data, 16 | CASE 17 | WHEN enm_binlog_event = 'ddl' 18 | THEN 19 | t_query 20 | WHEN enm_binlog_event = 'insert' 21 | THEN 22 | format( 23 | 'INSERT INTO %I.%I %s;', 24 | v_schema_name, 25 | v_table_name, 26 | t_dec_data 27 | 28 | ) 29 | WHEN enm_binlog_event = 'update' 30 | THEN 31 | format( 32 | 'UPDATE %I.%I SET %s WHERE %s;', 33 | v_schema_name, 34 | v_table_name, 35 | t_dec_data, 36 | t_pk_data 37 | ) 38 | WHEN enm_binlog_event = 'delete' 39 | THEN 40 | format( 41 | 'DELETE FROM %I.%I WHERE %s;', 42 | v_schema_name, 43 | v_table_name, 44 | t_pk_data 45 | ) 46 | 47 | END AS t_sql 48 | FROM 49 | ( 50 | SELECT 51 | dec.i_id_event, 52 | dec.v_table_name, 53 | dec.v_schema_name, 54 | dec.enm_binlog_event, 55 | dec.t_binlog_name, 56 | dec.i_binlog_position, 57 | dec.t_query as t_query, 58 | dec.ts_event_datetime, 59 | CASE 60 | WHEN dec.enm_binlog_event = 'insert' 61 | THEN 62 | format('(%s) VALUES (%s)',string_agg(format('%I',dec.t_column),','),string_agg(format('%L',jsb_event_after->>t_column),',')) 63 | WHEN dec.enm_binlog_event = 'update' 64 | THEN 65 | string_agg(format('%I=%L',dec.t_column,jsb_event_after->>t_column),',') 66 | 67 | END AS t_dec_data, 68 | string_agg(DISTINCT 69 | CASE 70 | WHEN dec.v_table_pkey IS NOT NULL 71 | THEN 72 | format( 73 | '%I=%L', 74 | dec.v_table_pkey, 75 | CASE 76 | WHEN dec.enm_binlog_event = 'update' 77 | THEN 78 | jsb_event_before->>v_table_pkey 79 | ELSE 80 | jsb_event_after->>v_table_pkey 81 | END 82 | 83 | ) 84 | END 85 | ,' AND ') as t_pk_data 86 | FROM 87 | ( 88 | SELECT 89 | log.i_id_event, 90 | log.v_table_name, 91 | log.v_schema_name, 92 | log.enm_binlog_event, 93 | log.t_binlog_name, 94 | log.i_binlog_position, 95 | coalesce(log.jsb_event_after,'{"foo":"bar"}'::jsonb) as jsb_event_after, 96 | (jsonb_each_text(coalesce(log.jsb_event_after,'{"foo":"bar"}'::jsonb))).key AS t_column, 97 | log.jsb_event_before, 98 | log.t_query as t_query, 99 | log.ts_event_datetime, 100 | unnest(v_table_pkey) as v_table_pkey 101 | FROM 102 | sch_chameleon.t_log_replica_mysql_2 log 103 | INNER JOIN sch_chameleon.t_replica_tables tab 104 | ON 105 | tab.v_table_name=log.v_table_name 106 | AND tab.v_schema_name=log.v_schema_name 107 | WHERE 108 | tab.b_replica_enabled 109 | AND i_id_event = ANY('{10}') 110 | ) dec 111 | GROUP BY 112 | dec.i_id_event, 113 | dec.v_table_name, 114 | dec.v_schema_name, 115 | dec.enm_binlog_event, 116 | dec.t_query, 117 | dec.ts_event_datetime, 118 | dec.t_binlog_name, 119 | dec.i_binlog_position 120 | ) par 121 | ORDER BY 122 | i_id_event ASC 123 | ; -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/01_create.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE test ( 2 | id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, 3 | value1 VARCHAR(45) NOT NULL, 4 | value2 VARCHAR(45) NOT NULL, 5 | last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 6 | date_create TIMESTAMP NOT NULL, 7 | PRIMARY KEY (id), 8 | KEY idx_actor_last_name (value2) 9 | )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 10 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/02_insert.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO test (value1,value2,date_create) 2 | VALUES 3 | ('hello','dave',now()), 4 | ('knock knock','neo','2015-01-01'), 5 | ('the answer','is 42','0000-00-00'); 6 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/03_update.sql: -------------------------------------------------------------------------------- 1 | UPDATE test SET value2 = 'world' WHERE value1 = 'hello'; 2 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/04_delete.sql: -------------------------------------------------------------------------------- 1 | DELETE FROM test WHERE value1='the answer'; 2 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/05_drop.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE test ; 2 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/06_create_broken.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE test_broken ( 2 | id SMALLINT UNSIGNED NULL AUTO_INCREMENT, 3 | value1 VARCHAR(45) NOT NULL, 4 | value2 VARCHAR(45) NOT NULL, 5 | val_enum enum('postgresql','rocks') null, 6 | last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 7 | PRIMARY KEY (id), 8 | KEY idx_actor_last_name (value2) 9 | )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 10 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/07_insert_broken.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO test_broken (value1,value2,val_enum) 2 | VALUES 3 | ('hello','dave','postgresql'), 4 | ('knock knock','neo','rocks'), 5 | ('the answer','is 42',NULL); 6 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/08_drop_broken.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE test_broken ; 2 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/09_alter.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE test 2 | ADD COLUMN `count` SMALLINT(6) NOT NULL , 3 | ADD COLUMN `log` VARCHAR(12) NOT NULL AFTER `count`, 4 | ADD COLUMN new_enum ENUM('asd','r') NOT NULL AFTER `log`, 5 | ADD COLUMN status INT(10) UNSIGNED NULL AFTER `new_enum`; 6 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/10_insert_expanded.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO test (value1,value2,date_create,count,log,status) 2 | VALUES 3 | ('the answer','is 42','0000-00-00',0,'foo',10); 4 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/fn_load_data.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS test_partition; 2 | CREATE TABLE test_partition ( 3 | id INT UNSIGNED NOT NULL AUTO_INCREMENT, 4 | id_partition int(10) NULL, 5 | PRIMARY KEY (id) 6 | )ENGINE=InnoDB DEFAULT CHARSET=utf8; 7 | 8 | 9 | DELIMITER $$ 10 | DROP PROCEDURE IF EXISTS prepare_data$$ 11 | DELIMITER ; 12 | DELIMITER $$ 13 | CREATE PROCEDURE prepare_data() 14 | BEGIN 15 | DECLARE v_part INT DEFAULT 1; 16 | DECLARE rnd_val INT DEFAULT 1; 17 | WHILE v_part < 50000 DO 18 | SET rnd_val = FLOOR(RAND() * (30000 - 1 + 1)) + 1 ; 19 | INSERT INTO test_partition (id_partition) VALUES (rnd_val); 20 | SET v_part = v_part + 1; 21 | END WHILE; 22 | END$$ 23 | DELIMITER ; 24 | 25 | 26 | DROP TABLE IF EXISTS test_partition2; 27 | CREATE TABLE test_partition2 ( 28 | id INT UNSIGNED NOT NULL AUTO_INCREMENT, 29 | id_partition int(10) NOT NULL, 30 | PRIMARY KEY (id) 31 | )ENGINE=InnoDB DEFAULT CHARSET=utf8; 32 | 33 | 34 | DELIMITER $$ 35 | DROP PROCEDURE IF EXISTS prepare_data2$$ 36 | DELIMITER ; 37 | DELIMITER $$ 38 | CREATE PROCEDURE prepare_data2() 39 | BEGIN 40 | DECLARE v_part INT DEFAULT 1; 41 | DECLARE rnd_val INT DEFAULT 1; 42 | WHILE v_part < 50000 DO 43 | SET rnd_val = FLOOR(RAND() * (30000 - 1 + 1)) + 1 ; 44 | INSERT INTO test_partition2 (id_partition) VALUES (rnd_val); 45 | SET v_part = v_part + 1; 46 | END WHILE; 47 | END$$ 48 | DELIMITER ; 49 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/get_default_val.sql: -------------------------------------------------------------------------------- 1 | SELECT 2 | pg_catalog.format_type(a.atttypid, a.atttypmod), 3 | ( 4 | SELECT 5 | split_part(substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) for 128),'::',1) 6 | FROM 7 | pg_catalog.pg_attrdef d 8 | WHERE 9 | d.adrelid = a.attrelid 10 | AND d.adnum = a.attnum 11 | AND a.atthasdef 12 | ) as default_value, 13 | ( 14 | SELECT 15 | pg_catalog.pg_get_expr(d.adbin, d.adrelid) 16 | FROM 17 | pg_catalog.pg_attrdef d 18 | WHERE 19 | d.adrelid = a.attrelid 20 | AND d.adnum = a.attnum 21 | AND a.atthasdef 22 | ) as full_definition, 23 | * 24 | FROM 25 | pg_catalog.pg_attribute a 26 | WHERE 27 | a.attrelid = 'sch_chameleon.t_sources'::regclass 28 | AND a.attname='enm_status' 29 | AND NOT a.attisdropped 30 | ; 31 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS test; 2 | CREATE TABLE test ( 3 | id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, 4 | value1 VARCHAR(45) NOT NULL, 5 | value2 VARCHAR(45) NOT NULL, 6 | last_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 7 | PRIMARY KEY (id), 8 | KEY idx_actor_last_name (value2) 9 | )ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 10 | 11 | INSERT INTO test (value1,value2) 12 | VALUES 13 | ('hello','dave'), 14 | ('knock knock','neo'), 15 | ('the','phoenix'), 16 | ('the answer','is 42'); 17 | ALTER TABLE test 18 | ADD COLUMN `count` SMALLINT(6) NULL , 19 | ADD COLUMN `log` VARCHAR(12) default 'blah' NULL AFTER `count`, 20 | ADD COLUMN status INT(10) UNSIGNED NULL AFTER `count`; 21 | 22 | ALTER TABLE test 23 | ADD COLUMN new_enum ENUM('asd','r') NULL AFTER `log`; 24 | 25 | ALTER TABLE test 26 | DROP COLUMN `count` , 27 | ADD COLUMN status_2 INT(10) UNSIGNED NULL AFTER `new_enum`, 28 | ADD COLUMN `boolean_default` bool DEFAULT 0 NOT NULL; 29 | DELETE FROM test WHERE value1='the answer'; 30 | UPDATE test SET value2 = 'world' WHERE value1 = 'hello'; 31 | alter table test add constraint dd unique(value2); 32 | 33 | ALTER TABLE `test` MODIFY `log` enum('blah','dd') DEFAULT 'blah'; 34 | 35 | 36 | TRUNCATE TABLE `sakila`.`test`; 37 | 38 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/test_pkeyless.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS test_pkeyless; 2 | CREATE TABLE test_pkeyless ( 3 | id SMALLINT UNSIGNED NOT NULL, 4 | value1 VARCHAR(45) NOT NULL 5 | )ENGINE=InnoDB DEFAULT CHARSET=utf8; 6 | 7 | INSERT INTO test_pkeyless (id,value1) 8 | VALUES 9 | (1,'dave'), 10 | (2,'neo'), 11 | (3,'phoenix'), 12 | (4,'is 42'); 13 | 14 | /* 15 | stop the replica then run call prepare_data then run this additional inserts 16 | 17 | call prepare_data; 18 | INSERT INTO test_pkeyless (id,value1) 19 | VALUES 20 | (5,'rainbow'), 21 | (6,'twilight'), 22 | (7,'pinkie'), 23 | (8,'apple'), 24 | (9,'rarity') 25 | ; 26 | 27 | 28 | 29 | */ 30 | 31 | ALTER TABLE `test_pkeyless` ADD COLUMN `id_pkey` INT AUTO_INCREMENT PRIMARY KEY FIRST; 32 | -------------------------------------------------------------------------------- /pg_chameleon/sql/tests/wrong_log_position.sql: -------------------------------------------------------------------------------- 1 | TRUNCATE TABLE test; 2 | CREATE TEMPORARY TABLE tmp_test( 3 | id SMALLINT UNSIGNED NOT NULL AUTO_INCREMENT, 4 | value1 VARCHAR(45) NOT NULL, 5 | PRIMARY KEY (id) 6 | )ENGINE=InnoDB DEFAULT CHARSET=utf8; 7 | insert into tmp_test (value1) values('blah'),('blah'); 8 | insert into test (value1) values('blah'); 9 | DROP TEMPORARY TABLE if exists tmp_test ; 10 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/the4thdoctor/pg_chameleon/5458575165565b4593d33af912e7fdbeb4db49f8/pg_chameleon/sql/upgrade/.placeholder -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/200_to_201.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.0 to 2.0.1 2 | 3 | ALTER TABLE sch_chameleon.t_error_log 4 | ALTER COLUMN i_binlog_position SET DATA TYPE bigint; 5 | 6 | ALTER TABLE sch_chameleon.t_sources 7 | ALTER COLUMN i_binlog_position SET DATA TYPE bigint; 8 | 9 | ALTER TABLE sch_chameleon.t_replica_batch 10 | ALTER COLUMN i_binlog_position SET DATA TYPE bigint; 11 | 12 | ALTER TABLE sch_chameleon.t_log_replica 13 | ALTER COLUMN i_binlog_position SET DATA TYPE bigint; 14 | 15 | ALTER TABLE sch_chameleon.t_replica_tables 16 | ALTER COLUMN i_binlog_position SET DATA TYPE bigint; 17 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/201_to_202.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.1 to 2.0.2 2 | 3 | ALTER TABLE sch_chameleon.t_sources 4 | ADD COLUMN b_paused boolean NOT NULL DEFAULT False, 5 | ADD COLUMN ts_last_maintenance timestamp without time zone NULL ; 6 | 7 | ALTER TABLE sch_chameleon.t_last_received 8 | ADD COLUMN b_paused boolean NOT NULL DEFAULT False; 9 | 10 | ALTER TABLE sch_chameleon.t_last_replayed 11 | ADD COLUMN b_paused boolean NOT NULL DEFAULT False; 12 | 13 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/202_to_203.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.2 to 2.0.3 2 | 3 | ALTER TABLE sch_chameleon.t_sources 4 | ADD COLUMN b_maintenance boolean NOT NULL DEFAULT False; 5 | 6 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/203_to_204.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.3 to 2.0.4 2 | 3 | ALTER TABLE sch_chameleon.t_replica_batch 4 | ADD COLUMN t_gtid_set text; 5 | 6 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/204_to_205.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.4 to 2.0.5 2 | 3 | ALTER TABLE sch_chameleon.t_replica_batch 4 | ADD COLUMN v_log_table character varying NOT NULL DEFAULT 't_log_replica'; 5 | 6 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_replay_mysql(integer,integer,boolean) 7 | RETURNS sch_chameleon.ty_replay_status AS 8 | $BODY$ 9 | DECLARE 10 | p_i_max_events ALIAS FOR $1; 11 | p_i_id_source ALIAS FOR $2; 12 | p_b_exit_on_error ALIAS FOR $3; 13 | v_ty_status sch_chameleon.ty_replay_status; 14 | v_r_statements record; 15 | v_i_id_batch bigint; 16 | v_v_log_table text; 17 | v_t_ddl text; 18 | v_t_main_sql text; 19 | v_t_delete_sql text; 20 | v_i_replayed integer; 21 | v_i_skipped integer; 22 | v_i_ddl integer; 23 | v_i_evt_replay bigint[]; 24 | v_i_evt_queue bigint[]; 25 | v_ts_evt_source timestamp without time zone; 26 | v_tab_enabled boolean; 27 | 28 | BEGIN 29 | v_i_replayed:=0; 30 | v_i_ddl:=0; 31 | v_i_skipped:=0; 32 | v_ty_status.b_continue:=FALSE; 33 | v_ty_status.b_error:=FALSE; 34 | RAISE DEBUG 'Searching batches to replay for source id: %', p_i_id_source; 35 | v_i_id_batch:= ( 36 | SELECT 37 | bat.i_id_batch 38 | FROM 39 | sch_chameleon.t_replica_batch bat 40 | INNER JOIN sch_chameleon.t_batch_events evt 41 | ON 42 | evt.i_id_batch=bat.i_id_batch 43 | WHERE 44 | bat.b_started 45 | AND bat.b_processed 46 | AND NOT bat.b_replayed 47 | AND bat.i_id_source=p_i_id_source 48 | ORDER BY 49 | bat.ts_created 50 | LIMIT 1 51 | ) 52 | ; 53 | 54 | v_v_log_table:=( 55 | SELECT 56 | v_log_table 57 | FROM 58 | sch_chameleon.t_replica_batch 59 | WHERE 60 | i_id_batch=v_i_id_batch 61 | ) 62 | ; 63 | IF v_i_id_batch IS NULL 64 | THEN 65 | RAISE DEBUG 'There are no batches available for replay'; 66 | RETURN v_ty_status; 67 | END IF; 68 | 69 | RAISE DEBUG 'Found id_batch %, data in log table %', v_i_id_batch,v_v_log_table; 70 | RAISE DEBUG 'Building a list of event id with max length %...', p_i_max_events; 71 | v_i_evt_replay:=( 72 | SELECT 73 | i_id_event[1:p_i_max_events] 74 | FROM 75 | sch_chameleon.t_batch_events 76 | WHERE 77 | i_id_batch=v_i_id_batch 78 | ); 79 | 80 | 81 | v_i_evt_queue:=( 82 | SELECT 83 | i_id_event[p_i_max_events+1:array_length(i_id_event,1)] 84 | FROM 85 | sch_chameleon.t_batch_events 86 | WHERE 87 | i_id_batch=v_i_id_batch 88 | ); 89 | 90 | RAISE DEBUG 'Finding the last executed event''s timestamp...'; 91 | v_ts_evt_source:=( 92 | SELECT 93 | to_timestamp(i_my_event_time) 94 | FROM 95 | sch_chameleon.t_log_replica 96 | WHERE 97 | i_id_event=v_i_evt_replay[array_length(v_i_evt_replay,1)] 98 | AND i_id_batch=v_i_id_batch 99 | ); 100 | 101 | RAISE DEBUG 'Generating the main loop sql'; 102 | 103 | v_t_main_sql:=format(' 104 | SELECT 105 | i_id_event AS i_id_event, 106 | enm_binlog_event, 107 | (enm_binlog_event=''ddl'')::integer as i_ddl, 108 | (enm_binlog_event<>''ddl'')::integer as i_replay, 109 | t_binlog_name, 110 | i_binlog_position, 111 | v_table_name, 112 | v_schema_name, 113 | t_pk_data, 114 | CASE 115 | WHEN enm_binlog_event = ''ddl'' 116 | THEN 117 | t_query 118 | WHEN enm_binlog_event = ''insert'' 119 | THEN 120 | format( 121 | ''INSERT INTO %%I.%%I %%s;'', 122 | v_schema_name, 123 | v_table_name, 124 | t_dec_data 125 | 126 | ) 127 | WHEN enm_binlog_event = ''update'' 128 | THEN 129 | format( 130 | ''UPDATE %%I.%%I SET %%s WHERE %%s;'', 131 | v_schema_name, 132 | v_table_name, 133 | t_dec_data, 134 | t_pk_data 135 | ) 136 | WHEN enm_binlog_event = ''delete'' 137 | THEN 138 | format( 139 | ''DELETE FROM %%I.%%I WHERE %%s;'', 140 | v_schema_name, 141 | v_table_name, 142 | t_pk_data 143 | ) 144 | 145 | END AS t_sql 146 | FROM 147 | ( 148 | SELECT 149 | dec.i_id_event, 150 | dec.v_table_name, 151 | dec.v_schema_name, 152 | dec.enm_binlog_event, 153 | dec.t_binlog_name, 154 | dec.i_binlog_position, 155 | dec.t_query as t_query, 156 | dec.ts_event_datetime, 157 | CASE 158 | WHEN dec.enm_binlog_event = ''insert'' 159 | THEN 160 | format(''(%%s) VALUES (%%s)'',string_agg(format(''%%I'',dec.t_column),'',''),string_agg(format(''%%L'',jsb_event_after->>t_column),'','')) 161 | WHEN dec.enm_binlog_event = ''update'' 162 | THEN 163 | string_agg(format(''%%I=%%L'',dec.t_column,jsb_event_after->>t_column),'','') 164 | 165 | END AS t_dec_data, 166 | string_agg(DISTINCT format( 167 | ''%%I=%%L'', 168 | dec.v_table_pkey, 169 | CASE 170 | WHEN dec.enm_binlog_event = ''update'' 171 | THEN 172 | jsb_event_before->>v_table_pkey 173 | ELSE 174 | jsb_event_after->>v_table_pkey 175 | END 176 | ),'' AND '') as t_pk_data 177 | FROM 178 | ( 179 | SELECT 180 | log.i_id_event, 181 | log.v_table_name, 182 | log.v_schema_name, 183 | log.enm_binlog_event, 184 | log.t_binlog_name, 185 | log.i_binlog_position, 186 | coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb) as jsb_event_after, 187 | (jsonb_each_text(coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb))).key AS t_column, 188 | log.jsb_event_before, 189 | log.t_query as t_query, 190 | log.ts_event_datetime, 191 | unnest(v_table_pkey) as v_table_pkey 192 | FROM 193 | sch_chameleon.%I log 194 | INNER JOIN sch_chameleon.t_replica_tables tab 195 | ON 196 | tab.v_table_name=log.v_table_name 197 | AND tab.v_schema_name=log.v_schema_name 198 | WHERE 199 | tab.b_replica_enabled 200 | AND i_id_event = ANY(%L) 201 | ) dec 202 | GROUP BY 203 | dec.i_id_event, 204 | dec.v_table_name, 205 | dec.v_schema_name, 206 | dec.enm_binlog_event, 207 | dec.t_query, 208 | dec.ts_event_datetime, 209 | dec.t_binlog_name, 210 | dec.i_binlog_position 211 | ) par 212 | ORDER BY 213 | i_id_event ASC 214 | ; 215 | 216 | ',v_v_log_table,v_i_evt_replay); 217 | 218 | FOR v_r_statements IN EXECUTE v_t_main_sql 219 | LOOP 220 | 221 | BEGIN 222 | EXECUTE v_r_statements.t_sql; 223 | v_i_ddl:=v_i_ddl+v_r_statements.i_ddl; 224 | v_i_replayed:=v_i_replayed+v_r_statements.i_replay; 225 | 226 | 227 | EXCEPTION 228 | WHEN OTHERS 229 | THEN 230 | RAISE NOTICE 'An error occurred when replaying data for the table %.%',v_r_statements.v_schema_name,v_r_statements.v_table_name; 231 | RAISE NOTICE 'SQLSTATE: % - ERROR MESSAGE %',SQLSTATE, SQLERRM; 232 | RAISE DEBUG 'SQL EXECUTED: % ',v_r_statements.t_sql; 233 | RAISE NOTICE 'The table %.% has been removed from the replica',v_r_statements.v_schema_name,v_r_statements.v_table_name; 234 | v_ty_status.v_table_error:=array_append(v_ty_status.v_table_error, format('%I.%I SQLSTATE: %s - ERROR MESSAGE: %s',v_r_statements.v_schema_name,v_r_statements.v_table_name,SQLSTATE, SQLERRM)::character varying) ; 235 | RAISE NOTICE 'Adding error log entry for table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 236 | INSERT INTO sch_chameleon.t_error_log 237 | ( 238 | i_id_batch, 239 | i_id_source, 240 | v_schema_name, 241 | v_table_name, 242 | t_table_pkey, 243 | t_binlog_name, 244 | i_binlog_position, 245 | ts_error, 246 | t_sql, 247 | t_error_message 248 | ) 249 | SELECT 250 | i_id_batch, 251 | p_i_id_source, 252 | v_schema_name, 253 | v_table_name, 254 | v_r_statements.t_pk_data as t_table_pkey, 255 | t_binlog_name, 256 | i_binlog_position, 257 | clock_timestamp(), 258 | quote_literal(v_r_statements.t_sql) as t_sql, 259 | format('%s - %s',SQLSTATE, SQLERRM) as t_error_message 260 | FROM 261 | sch_chameleon.t_log_replica log 262 | WHERE 263 | log.i_id_event=v_r_statements.i_id_event 264 | ; 265 | IF p_b_exit_on_error 266 | THEN 267 | v_ty_status.b_continue:=FALSE; 268 | v_ty_status.b_error:=TRUE; 269 | RETURN v_ty_status; 270 | ELSE 271 | 272 | RAISE NOTICE 'Statement %', v_r_statements.t_sql; 273 | UPDATE sch_chameleon.t_replica_tables 274 | SET 275 | b_replica_enabled=FALSE 276 | WHERE 277 | v_schema_name=v_r_statements.v_schema_name 278 | AND v_table_name=v_r_statements.v_table_name 279 | ; 280 | 281 | RAISE NOTICE 'Deleting the log entries for the table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 282 | DELETE FROM sch_chameleon.t_log_replica log 283 | WHERE 284 | v_table_name=v_r_statements.v_table_name 285 | AND v_schema_name=v_r_statements.v_schema_name 286 | AND i_id_batch=v_i_id_batch 287 | ; 288 | END IF; 289 | END; 290 | END LOOP; 291 | IF v_ts_evt_source IS NOT NULL 292 | THEN 293 | UPDATE sch_chameleon.t_last_replayed 294 | SET 295 | ts_last_replayed=v_ts_evt_source 296 | WHERE 297 | i_id_source=p_i_id_source 298 | ; 299 | END IF; 300 | IF v_i_replayed=0 AND v_i_ddl=0 301 | THEN 302 | DELETE FROM sch_chameleon.t_log_replica 303 | WHERE 304 | i_id_batch=v_i_id_batch 305 | ; 306 | 307 | GET DIAGNOSTICS v_i_skipped = ROW_COUNT; 308 | RAISE DEBUG 'SKIPPED ROWS: % ',v_i_skipped; 309 | 310 | UPDATE ONLY sch_chameleon.t_replica_batch 311 | SET 312 | b_replayed=True, 313 | i_skipped=v_i_skipped, 314 | ts_replayed=clock_timestamp() 315 | 316 | WHERE 317 | i_id_batch=v_i_id_batch 318 | ; 319 | 320 | DELETE FROM sch_chameleon.t_batch_events 321 | WHERE 322 | i_id_batch=v_i_id_batch 323 | ; 324 | 325 | v_ty_status.b_continue:=FALSE; 326 | ELSE 327 | UPDATE ONLY sch_chameleon.t_replica_batch 328 | SET 329 | i_ddl=coalesce(i_ddl,0)+v_i_ddl, 330 | i_replayed=coalesce(i_replayed,0)+v_i_replayed, 331 | i_skipped=v_i_skipped, 332 | ts_replayed=clock_timestamp() 333 | 334 | WHERE 335 | i_id_batch=v_i_id_batch 336 | ; 337 | 338 | UPDATE sch_chameleon.t_batch_events 339 | SET 340 | i_id_event = v_i_evt_queue 341 | WHERE 342 | i_id_batch=v_i_id_batch 343 | ; 344 | 345 | DELETE FROM sch_chameleon.t_log_replica 346 | WHERE 347 | i_id_batch=v_i_id_batch 348 | AND i_id_event=ANY(v_i_evt_replay) 349 | ; 350 | v_ty_status.b_continue:=TRUE; 351 | RETURN v_ty_status; 352 | END IF; 353 | v_i_id_batch:= ( 354 | SELECT 355 | bat.i_id_batch 356 | FROM 357 | sch_chameleon.t_replica_batch bat 358 | INNER JOIN sch_chameleon.t_batch_events evt 359 | ON 360 | evt.i_id_batch=bat.i_id_batch 361 | WHERE 362 | bat.b_started 363 | AND bat.b_processed 364 | AND NOT bat.b_replayed 365 | AND bat.i_id_source=p_i_id_source 366 | ORDER BY 367 | bat.ts_created 368 | LIMIT 1 369 | ) 370 | ; 371 | 372 | IF v_i_id_batch IS NOT NULL 373 | THEN 374 | v_ty_status.b_continue:=TRUE; 375 | END IF; 376 | 377 | 378 | RETURN v_ty_status; 379 | 380 | 381 | 382 | END; 383 | 384 | $BODY$ 385 | LANGUAGE plpgsql; 386 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/205_to_206.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.5 to 2.0.6 2 | CREATE OR REPLACE FUNCTION sch_chameleon.fn_replay_mysql(integer,integer,boolean) 3 | RETURNS sch_chameleon.ty_replay_status AS 4 | $BODY$ 5 | DECLARE 6 | p_i_max_events ALIAS FOR $1; 7 | p_i_id_source ALIAS FOR $2; 8 | p_b_exit_on_error ALIAS FOR $3; 9 | v_ty_status sch_chameleon.ty_replay_status; 10 | v_r_statements record; 11 | v_i_id_batch bigint; 12 | v_v_log_table text; 13 | v_t_ddl text; 14 | v_t_main_sql text; 15 | v_t_delete_sql text; 16 | v_i_replayed integer; 17 | v_i_skipped integer; 18 | v_i_ddl integer; 19 | v_i_evt_replay bigint[]; 20 | v_i_evt_queue bigint[]; 21 | v_ts_evt_source timestamp without time zone; 22 | v_tab_enabled boolean; 23 | 24 | BEGIN 25 | v_i_replayed:=0; 26 | v_i_ddl:=0; 27 | v_i_skipped:=0; 28 | v_ty_status.b_continue:=FALSE; 29 | v_ty_status.b_error:=FALSE; 30 | RAISE DEBUG 'Searching batches to replay for source id: %', p_i_id_source; 31 | v_i_id_batch:= ( 32 | SELECT 33 | bat.i_id_batch 34 | FROM 35 | sch_chameleon.t_replica_batch bat 36 | INNER JOIN sch_chameleon.t_batch_events evt 37 | ON 38 | evt.i_id_batch=bat.i_id_batch 39 | WHERE 40 | bat.b_started 41 | AND bat.b_processed 42 | AND NOT bat.b_replayed 43 | AND bat.i_id_source=p_i_id_source 44 | ORDER BY 45 | bat.ts_created 46 | LIMIT 1 47 | ) 48 | ; 49 | 50 | v_v_log_table:=( 51 | SELECT 52 | v_log_table 53 | FROM 54 | sch_chameleon.t_replica_batch 55 | WHERE 56 | i_id_batch=v_i_id_batch 57 | ) 58 | ; 59 | IF v_i_id_batch IS NULL 60 | THEN 61 | RAISE DEBUG 'There are no batches available for replay'; 62 | RETURN v_ty_status; 63 | END IF; 64 | 65 | RAISE DEBUG 'Found id_batch %, data in log table %', v_i_id_batch,v_v_log_table; 66 | RAISE DEBUG 'Building a list of event id with max length %...', p_i_max_events; 67 | v_i_evt_replay:=( 68 | SELECT 69 | i_id_event[1:p_i_max_events] 70 | FROM 71 | sch_chameleon.t_batch_events 72 | WHERE 73 | i_id_batch=v_i_id_batch 74 | ); 75 | 76 | 77 | v_i_evt_queue:=( 78 | SELECT 79 | i_id_event[p_i_max_events+1:array_length(i_id_event,1)] 80 | FROM 81 | sch_chameleon.t_batch_events 82 | WHERE 83 | i_id_batch=v_i_id_batch 84 | ); 85 | 86 | RAISE DEBUG 'Finding the last executed event''s timestamp...'; 87 | v_ts_evt_source:=( 88 | SELECT 89 | to_timestamp(i_my_event_time) 90 | FROM 91 | sch_chameleon.t_log_replica 92 | WHERE 93 | i_id_event=v_i_evt_replay[array_length(v_i_evt_replay,1)] 94 | AND i_id_batch=v_i_id_batch 95 | ); 96 | 97 | RAISE DEBUG 'Generating the main loop sql'; 98 | 99 | v_t_main_sql:=format(' 100 | SELECT 101 | i_id_event AS i_id_event, 102 | enm_binlog_event, 103 | (enm_binlog_event=''ddl'')::integer as i_ddl, 104 | (enm_binlog_event<>''ddl'')::integer as i_replay, 105 | t_binlog_name, 106 | i_binlog_position, 107 | v_table_name, 108 | v_schema_name, 109 | t_pk_data, 110 | CASE 111 | WHEN enm_binlog_event = ''ddl'' 112 | THEN 113 | t_query 114 | WHEN enm_binlog_event = ''insert'' 115 | THEN 116 | format( 117 | ''INSERT INTO %%I.%%I %%s;'', 118 | v_schema_name, 119 | v_table_name, 120 | t_dec_data 121 | 122 | ) 123 | WHEN enm_binlog_event = ''update'' 124 | THEN 125 | format( 126 | ''UPDATE %%I.%%I SET %%s WHERE %%s;'', 127 | v_schema_name, 128 | v_table_name, 129 | t_dec_data, 130 | t_pk_data 131 | ) 132 | WHEN enm_binlog_event = ''delete'' 133 | THEN 134 | format( 135 | ''DELETE FROM %%I.%%I WHERE %%s;'', 136 | v_schema_name, 137 | v_table_name, 138 | t_pk_data 139 | ) 140 | 141 | END AS t_sql 142 | FROM 143 | ( 144 | SELECT 145 | pk.i_id_event, 146 | pk.v_table_name, 147 | pk.v_schema_name, 148 | pk.enm_binlog_event, 149 | pk.t_binlog_name, 150 | pk.i_binlog_position, 151 | pk.t_query as t_query, 152 | pk.ts_event_datetime, 153 | pk.t_dec_data, 154 | string_agg(DISTINCT 155 | CASE 156 | WHEN pk.v_table_pkey IS NOT NULL 157 | THEN 158 | format( 159 | ''%%I=%%L'', 160 | pk.v_table_pkey, 161 | CASE 162 | WHEN pk.enm_binlog_event = ''update'' 163 | THEN 164 | pk.jsb_event_before->>v_table_pkey 165 | ELSE 166 | pk.jsb_event_after->>v_table_pkey 167 | END 168 | 169 | ) 170 | END 171 | ,'' AND '') as t_pk_data 172 | 173 | FROM 174 | ( 175 | SELECT 176 | dec.i_id_event, 177 | dec.v_table_name, 178 | dec.v_schema_name, 179 | dec.enm_binlog_event, 180 | dec.t_binlog_name, 181 | dec.i_binlog_position, 182 | dec.t_query as t_query, 183 | dec.ts_event_datetime, 184 | CASE 185 | WHEN dec.enm_binlog_event = ''insert'' 186 | THEN 187 | format(''(%%s) VALUES (%%s)'',string_agg(format(''%%I'',dec.t_column),'',''),string_agg(format(''%%L'',dec.jsb_event_after->>t_column),'','')) 188 | WHEN dec.enm_binlog_event = ''update'' 189 | THEN 190 | string_agg(format(''%%I=%%L'',dec.t_column,dec.jsb_event_after->>t_column),'','') 191 | 192 | END AS t_dec_data, 193 | unnest(v_table_pkey) as v_table_pkey, 194 | dec.jsb_event_after, 195 | dec.jsb_event_before 196 | 197 | FROM 198 | ( 199 | SELECT 200 | log.i_id_event, 201 | log.v_table_name, 202 | log.v_schema_name, 203 | log.enm_binlog_event, 204 | log.t_binlog_name, 205 | log.i_binlog_position, 206 | coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb) as jsb_event_after, 207 | (jsonb_each_text(coalesce(log.jsb_event_after,''{"foo":"bar"}''::jsonb))).key AS t_column, 208 | log.jsb_event_before, 209 | log.t_query as t_query, 210 | log.ts_event_datetime, 211 | v_table_pkey 212 | FROM 213 | sch_chameleon.%I log 214 | INNER JOIN sch_chameleon.t_replica_tables tab 215 | ON 216 | tab.v_table_name=log.v_table_name 217 | AND tab.v_schema_name=log.v_schema_name 218 | WHERE 219 | tab.b_replica_enabled 220 | AND i_id_event = ANY(%L) 221 | 222 | ) dec 223 | GROUP BY 224 | dec.i_id_event, 225 | dec.v_table_name, 226 | dec.v_schema_name, 227 | dec.enm_binlog_event, 228 | dec.t_query, 229 | dec.ts_event_datetime, 230 | dec.t_binlog_name, 231 | dec.i_binlog_position, 232 | dec.v_table_pkey, 233 | dec.jsb_event_after, 234 | dec.jsb_event_before 235 | ) pk 236 | GROUP BY 237 | pk.i_id_event, 238 | pk.v_table_name, 239 | pk.v_schema_name, 240 | pk.enm_binlog_event, 241 | pk.t_binlog_name, 242 | pk.i_binlog_position, 243 | pk.t_query, 244 | pk.ts_event_datetime, 245 | pk.t_dec_data 246 | 247 | 248 | ) par 249 | ORDER BY 250 | i_id_event ASC 251 | ; 252 | ',v_v_log_table,v_i_evt_replay); 253 | RAISE DEBUG '%',v_t_main_sql; 254 | FOR v_r_statements IN EXECUTE v_t_main_sql 255 | LOOP 256 | 257 | BEGIN 258 | EXECUTE v_r_statements.t_sql; 259 | v_i_ddl:=v_i_ddl+v_r_statements.i_ddl; 260 | v_i_replayed:=v_i_replayed+v_r_statements.i_replay; 261 | 262 | 263 | EXCEPTION 264 | WHEN OTHERS 265 | THEN 266 | RAISE NOTICE 'An error occurred when replaying data for the table %.%',v_r_statements.v_schema_name,v_r_statements.v_table_name; 267 | RAISE NOTICE 'SQLSTATE: % - ERROR MESSAGE %',SQLSTATE, SQLERRM; 268 | RAISE DEBUG 'SQL EXECUTED: % ',v_r_statements.t_sql; 269 | RAISE NOTICE 'The table %.% has been removed from the replica',v_r_statements.v_schema_name,v_r_statements.v_table_name; 270 | v_ty_status.v_table_error:=array_append(v_ty_status.v_table_error, format('%I.%I SQLSTATE: %s - ERROR MESSAGE: %s',v_r_statements.v_schema_name,v_r_statements.v_table_name,SQLSTATE, SQLERRM)::character varying) ; 271 | RAISE NOTICE 'Adding error log entry for table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 272 | INSERT INTO sch_chameleon.t_error_log 273 | ( 274 | i_id_batch, 275 | i_id_source, 276 | v_schema_name, 277 | v_table_name, 278 | t_table_pkey, 279 | t_binlog_name, 280 | i_binlog_position, 281 | ts_error, 282 | t_sql, 283 | t_error_message 284 | ) 285 | SELECT 286 | i_id_batch, 287 | p_i_id_source, 288 | v_schema_name, 289 | v_table_name, 290 | v_r_statements.t_pk_data as t_table_pkey, 291 | t_binlog_name, 292 | i_binlog_position, 293 | clock_timestamp(), 294 | quote_literal(v_r_statements.t_sql) as t_sql, 295 | format('%s - %s',SQLSTATE, SQLERRM) as t_error_message 296 | FROM 297 | sch_chameleon.t_log_replica log 298 | WHERE 299 | log.i_id_event=v_r_statements.i_id_event 300 | ; 301 | IF p_b_exit_on_error 302 | THEN 303 | v_ty_status.b_continue:=FALSE; 304 | v_ty_status.b_error:=TRUE; 305 | RETURN v_ty_status; 306 | ELSE 307 | 308 | RAISE NOTICE 'Statement %', v_r_statements.t_sql; 309 | UPDATE sch_chameleon.t_replica_tables 310 | SET 311 | b_replica_enabled=FALSE 312 | WHERE 313 | v_schema_name=v_r_statements.v_schema_name 314 | AND v_table_name=v_r_statements.v_table_name 315 | ; 316 | 317 | RAISE NOTICE 'Deleting the log entries for the table %.% ',v_r_statements.v_schema_name,v_r_statements.v_table_name; 318 | DELETE FROM sch_chameleon.t_log_replica log 319 | WHERE 320 | v_table_name=v_r_statements.v_table_name 321 | AND v_schema_name=v_r_statements.v_schema_name 322 | AND i_id_batch=v_i_id_batch 323 | ; 324 | END IF; 325 | END; 326 | END LOOP; 327 | IF v_ts_evt_source IS NOT NULL 328 | THEN 329 | UPDATE sch_chameleon.t_last_replayed 330 | SET 331 | ts_last_replayed=v_ts_evt_source 332 | WHERE 333 | i_id_source=p_i_id_source 334 | ; 335 | END IF; 336 | IF v_i_replayed=0 AND v_i_ddl=0 337 | THEN 338 | DELETE FROM sch_chameleon.t_log_replica 339 | WHERE 340 | i_id_batch=v_i_id_batch 341 | ; 342 | 343 | GET DIAGNOSTICS v_i_skipped = ROW_COUNT; 344 | RAISE DEBUG 'SKIPPED ROWS: % ',v_i_skipped; 345 | 346 | UPDATE ONLY sch_chameleon.t_replica_batch 347 | SET 348 | b_replayed=True, 349 | i_skipped=v_i_skipped, 350 | ts_replayed=clock_timestamp() 351 | 352 | WHERE 353 | i_id_batch=v_i_id_batch 354 | ; 355 | 356 | DELETE FROM sch_chameleon.t_batch_events 357 | WHERE 358 | i_id_batch=v_i_id_batch 359 | ; 360 | 361 | v_ty_status.b_continue:=FALSE; 362 | ELSE 363 | UPDATE ONLY sch_chameleon.t_replica_batch 364 | SET 365 | i_ddl=coalesce(i_ddl,0)+v_i_ddl, 366 | i_replayed=coalesce(i_replayed,0)+v_i_replayed, 367 | i_skipped=v_i_skipped, 368 | ts_replayed=clock_timestamp() 369 | 370 | WHERE 371 | i_id_batch=v_i_id_batch 372 | ; 373 | 374 | UPDATE sch_chameleon.t_batch_events 375 | SET 376 | i_id_event = v_i_evt_queue 377 | WHERE 378 | i_id_batch=v_i_id_batch 379 | ; 380 | 381 | DELETE FROM sch_chameleon.t_log_replica 382 | WHERE 383 | i_id_batch=v_i_id_batch 384 | AND i_id_event=ANY(v_i_evt_replay) 385 | ; 386 | v_ty_status.b_continue:=TRUE; 387 | RETURN v_ty_status; 388 | END IF; 389 | v_i_id_batch:= ( 390 | SELECT 391 | bat.i_id_batch 392 | FROM 393 | sch_chameleon.t_replica_batch bat 394 | INNER JOIN sch_chameleon.t_batch_events evt 395 | ON 396 | evt.i_id_batch=bat.i_id_batch 397 | WHERE 398 | bat.b_started 399 | AND bat.b_processed 400 | AND NOT bat.b_replayed 401 | AND bat.i_id_source=p_i_id_source 402 | ORDER BY 403 | bat.ts_created 404 | LIMIT 1 405 | ) 406 | ; 407 | 408 | IF v_i_id_batch IS NOT NULL 409 | THEN 410 | v_ty_status.b_continue:=TRUE; 411 | END IF; 412 | 413 | 414 | RETURN v_ty_status; 415 | 416 | 417 | 418 | END; 419 | 420 | $BODY$ 421 | LANGUAGE plpgsql; 422 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/206_to_207.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.6 to 2.0.7 2 | --TABLES 3 | CREATE TABLE sch_chameleon.t_indexes 4 | ( 5 | i_id_index bigserial, 6 | v_schema_name varchar NOT NULL, 7 | v_table_name varchar NOT NULL, 8 | v_index_name varchar NOT NULL, 9 | t_index_drop text NULL, 10 | t_index_create text NULL, 11 | CONSTRAINT pk_t_indexes PRIMARY KEY (i_id_index) 12 | ); 13 | CREATE UNIQUE INDEX idx_t_indexes_idx_table_schema ON sch_chameleon.t_indexes USING btree(v_schema_name,v_table_name,v_index_name); 14 | 15 | CREATE TABLE sch_chameleon.t_pkeys 16 | ( 17 | i_id_pkey bigserial, 18 | v_schema_name varchar NOT NULL, 19 | v_table_name varchar NOT NULL, 20 | v_index_name varchar NOT NULL, 21 | t_pkey_drop text NULL, 22 | t_pkey_create text NULL, 23 | CONSTRAINT pk_t_pkeys PRIMARY KEY (i_id_pkey) 24 | ); 25 | CREATE UNIQUE INDEX idx_t_pkeys_table_schema ON sch_chameleon.t_pkeys USING btree(v_schema_name,v_table_name); 26 | 27 | CREATE TABLE sch_chameleon.t_fkeys 28 | ( 29 | i_id_fkey bigserial, 30 | v_schema_name varchar NOT NULL, 31 | v_table_name varchar NOT NULL, 32 | v_constraint_name varchar NOT NULL, 33 | t_fkey_drop text NULL, 34 | t_fkey_create text NULL, 35 | t_fkey_validate text NULL, 36 | CONSTRAINT pk_t_fkeys PRIMARY KEY (i_id_fkey) 37 | ); 38 | CREATE UNIQUE INDEX idx_t_fkeys_idx_table_schema ON sch_chameleon.t_fkeys USING btree(v_schema_name,v_table_name,v_constraint_name); 39 | 40 | 41 | --VIEWS 42 | CREATE OR REPLACE VIEW sch_chameleon.v_idx_pkeys 43 | AS 44 | SELECT 45 | oid_conid IS NOT NULL AS b_idx_pkey, 46 | CASE WHEN oid_conid IS NULL 47 | THEN 48 | format('DROP INDEX %I.%I;',v_schema_name,v_index_name) 49 | ELSE 50 | format('ALTER TABLE %I.%I DROP CONSTRAINT %I;',v_schema_name,v_table_name,v_index_name) 51 | END AS t_sql_drop, 52 | CASE WHEN oid_conid IS NULL 53 | THEN 54 | format('%s %s; SET default_tablespace=DEFAULT;', 55 | CASE WHEN v_index_tablespace IS NOT NULL 56 | THEN 57 | format('SET default_tablespace=%I;',v_index_tablespace) 58 | END, 59 | v_index_def 60 | ) 61 | ELSE 62 | format('%s ALTER TABLE %I.%I ADD CONSTRAINT %I %s; SET default_tablespace=DEFAULT;', 63 | CASE WHEN v_index_tablespace IS NOT NULL 64 | THEN 65 | format('SET default_tablespace=%I;',v_index_tablespace) 66 | END, 67 | v_schema_name, 68 | v_table_name, 69 | v_index_name, 70 | v_constraint_def 71 | ) 72 | END AS t_sql_create, 73 | v_index_name, 74 | v_table_name, 75 | v_schema_name 76 | FROM 77 | ( 78 | SELECT distinct 79 | idx.tablename AS v_table_name , 80 | idx.schemaname AS v_schema_name, 81 | idx.indexname AS v_index_name, 82 | idx.indexdef AS v_index_def, 83 | pg_get_constraintdef(con.oid_conid) AS v_constraint_def, 84 | idx.tablespace AS v_index_tablespace, 85 | con.oid_conid 86 | 87 | FROM 88 | pg_indexes idx 89 | LEFT OUTER JOIN 90 | ( 91 | SELECT 92 | con.oid AS oid_conid, 93 | tab.relname AS v_table_name, 94 | sch.nspname AS v_schema_name, 95 | con.conname AS v_constraint_name, 96 | con.contype AS v_constraint_type 97 | FROM 98 | pg_constraint con 99 | INNER JOIN pg_class tab 100 | ON tab.oid= con.conrelid 101 | INNER JOIN pg_namespace sch 102 | ON sch."oid" = tab.relnamespace 103 | WHERE con.contype='p' 104 | ) con 105 | ON 106 | con.v_table_name=idx.tablename 107 | AND con.v_schema_name=idx.schemaname 108 | AND con.v_constraint_name=idx.indexname 109 | ) idx_con 110 | ; 111 | 112 | 113 | CREATE OR REPLACE VIEW sch_chameleon.v_fkeys AS 114 | SELECT 115 | v_schema_referenced, 116 | v_table_referenced, 117 | v_schema_referencing, 118 | v_table_referencing, 119 | format('ALTER TABLE ONLY %I.%I DROP CONSTRAINT %I ;',v_schema_referencing,v_table_referencing ,v_fk_name) AS t_con_drop, 120 | format('ALTER TABLE ONLY %I.%I ADD CONSTRAINT %I %s NOT VALID ;',v_schema_referencing,v_table_referencing,v_fk_name,v_fk_definition) AS t_con_create, 121 | format('ALTER TABLE ONLY %I.%I VALIDATE CONSTRAINT %I ;',v_schema_referencing,v_table_referencing ,v_fk_name) AS t_con_validate, 122 | v_fk_name 123 | 124 | FROM 125 | ( 126 | SELECT 127 | tab.relname AS v_table_referenced, 128 | sch.nspname AS v_schema_referenced, 129 | pg_get_constraintdef(con.oid) AS v_fk_definition, 130 | tabref.relname AS v_table_referencing, 131 | schref.nspname AS v_schema_referencing, 132 | con.conname AS v_fk_name 133 | 134 | FROM 135 | pg_class tab 136 | INNER JOIN pg_namespace sch 137 | ON sch.oid=tab.relnamespace 138 | INNER JOIN pg_constraint con 139 | ON con.confrelid=tab.oid 140 | INNER JOIN pg_class tabref 141 | ON tabref.oid=con.conrelid 142 | INNER JOIN pg_namespace schref 143 | ON schref.oid=tabref.relnamespace 144 | WHERE 145 | tab.relkind='r' 146 | AND con.contype='f' 147 | ) fk 148 | ; 149 | -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/207_to_208.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.7 to 2.0.8 2 | --TABLE 3 | CREATE TABLE sch_chameleon.t_ukeys 4 | ( 5 | i_id_ukey bigserial, 6 | v_schema_name varchar NOT NULL, 7 | v_table_name varchar NOT NULL, 8 | v_index_name varchar NOT NULL, 9 | t_ukey_drop text NULL, 10 | t_ukey_create text NULL, 11 | CONSTRAINT pk_t_ukeys PRIMARY KEY (i_id_ukey) 12 | ); 13 | CREATE UNIQUE INDEX idx_t_ukeys_table_schema ON sch_chameleon.t_ukeys USING btree(v_schema_name,v_table_name,v_index_name); 14 | 15 | --DROP VIEW 16 | DROP VIEW IF EXISTS sch_chameleon.v_idx_pkeys ; 17 | 18 | --VIEW 19 | CREATE OR REPLACE VIEW sch_chameleon.v_idx_cons 20 | AS 21 | SELECT 22 | COALESCE(v_constraint_type,'i') v_constraint_type, 23 | CASE WHEN v_constraint_type IS NULL 24 | THEN 25 | format('DROP INDEX %I.%I;',v_schema_name,v_index_name) 26 | ELSE 27 | format('ALTER TABLE %I.%I DROP CONSTRAINT %I;',v_schema_name,v_table_name,v_index_name) 28 | END AS t_sql_drop, 29 | CASE WHEN v_constraint_type IS NULL 30 | THEN 31 | format('%s %s; SET default_tablespace=DEFAULT;', 32 | CASE WHEN v_index_tablespace IS NOT NULL 33 | THEN 34 | format('SET default_tablespace=%I;',v_index_tablespace) 35 | END, 36 | v_index_def 37 | ) 38 | ELSE 39 | format('%s ALTER TABLE %I.%I ADD CONSTRAINT %I %s; SET default_tablespace=DEFAULT;', 40 | CASE WHEN v_index_tablespace IS NOT NULL 41 | THEN 42 | format('SET default_tablespace=%I;',v_index_tablespace) 43 | END, 44 | v_schema_name, 45 | v_table_name, 46 | v_index_name, 47 | v_constraint_def 48 | ) 49 | END AS t_sql_create, 50 | v_index_name, 51 | v_table_name, 52 | v_schema_name 53 | FROM 54 | ( 55 | SELECT distinct 56 | idx.tablename AS v_table_name , 57 | idx.schemaname AS v_schema_name, 58 | idx.indexname AS v_index_name, 59 | idx.indexdef AS v_index_def, 60 | pg_get_constraintdef(con.oid_conid) AS v_constraint_def, 61 | idx.tablespace AS v_index_tablespace, 62 | con.oid_conid, 63 | v_constraint_type 64 | 65 | FROM 66 | pg_indexes idx 67 | LEFT OUTER JOIN 68 | ( 69 | SELECT 70 | con.oid AS oid_conid, 71 | tab.relname AS v_table_name, 72 | sch.nspname AS v_schema_name, 73 | con.conname AS v_constraint_name, 74 | con.contype AS v_constraint_type 75 | FROM 76 | pg_constraint con 77 | INNER JOIN pg_class tab 78 | ON tab.oid= con.conrelid 79 | INNER JOIN pg_namespace sch 80 | ON sch."oid" = tab.relnamespace 81 | WHERE con.contype IN ('p','u') 82 | ) con 83 | ON 84 | con.v_table_name=idx.tablename 85 | AND con.v_schema_name=idx.schemaname 86 | AND con.v_constraint_name=idx.indexname 87 | ) idx_con 88 | ; -------------------------------------------------------------------------------- /pg_chameleon/sql/upgrade/208_to_209.sql: -------------------------------------------------------------------------------- 1 | -- upgrade catalogue script 2.0.8 to 2.0.9 2 | --CHANGE VARCHARS TO VARCHAR 64 FOR THE IDENTIFIERS NAMES 3 | ALTER TABLE sch_chameleon.t_replica_tables 4 | ALTER COLUMN v_table_name TYPE character varying(64), 5 | ALTER COLUMN v_schema_name TYPE character varying(64); 6 | 7 | ALTER TABLE sch_chameleon.t_discarded_rows 8 | ALTER COLUMN v_table_name TYPE character varying(64), 9 | ALTER COLUMN v_schema_name TYPE character varying(64); 10 | 11 | 12 | ALTER TABLE sch_chameleon.t_indexes 13 | ALTER COLUMN v_table_name TYPE character varying(64), 14 | ALTER COLUMN v_schema_name TYPE character varying(64), 15 | ALTER COLUMN v_index_name TYPE character varying(64); 16 | 17 | ALTER TABLE sch_chameleon.t_pkeys 18 | ALTER COLUMN v_table_name TYPE character varying(64), 19 | ALTER COLUMN v_schema_name TYPE character varying(64), 20 | ALTER COLUMN v_index_name TYPE character varying(64); 21 | 22 | ALTER TABLE sch_chameleon.t_ukeys 23 | ALTER COLUMN v_table_name TYPE character varying(64), 24 | ALTER COLUMN v_schema_name TYPE character varying(64), 25 | ALTER COLUMN v_index_name TYPE character varying(64); 26 | 27 | ALTER TABLE sch_chameleon.t_fkeys 28 | ALTER COLUMN v_table_name TYPE character varying(64), 29 | ALTER COLUMN v_schema_name TYPE character varying(64), 30 | ALTER COLUMN v_constraint_name TYPE character varying(64); -------------------------------------------------------------------------------- /scripts/chameleon: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | from os import path 4 | __file__ = __file__ = '%s/chameleon.py' % (path.abspath(path.dirname(sys.argv[0]))) 5 | exec(compile(open(__file__).read(), __file__, 'exec')) 6 | -------------------------------------------------------------------------------- /scripts/chameleon.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pkg_resources import get_distribution 3 | __version__ = get_distribution('pg_chameleon') 4 | import argparse 5 | from pg_chameleon import replica_engine 6 | 7 | commands = [ 8 | 'show_config', 9 | 'show_sources', 10 | 'show_status', 11 | 'create_replica_schema', 12 | 'drop_replica_schema', 13 | 'upgrade_replica_schema', 14 | 'add_source', 15 | 'drop_source', 16 | 'init_replica', 17 | 'enable_replica', 18 | 'update_schema_mappings', 19 | 'refresh_schema', 20 | 'sync_tables', 21 | 'start_replica', 22 | 'stop_replica', 23 | 'detach_replica', 24 | 'set_configuration_files', 25 | 'show_errors', 26 | 'run_maintenance', 27 | 'stop_all_replicas' 28 | ] 29 | 30 | command_help = ','.join(commands) 31 | config_help = """Specifies the configuration to use without the suffix yml. If the parameter is omitted then ~/.pg_chameleon/configuration/default.yml is used""" 32 | schema_help = """Specifies the schema within a source. If omitted all schemas for the given source are affected by the command. Requires the argument --source to be specified""" 33 | source_help = """Specifies the source within a configuration. If omitted all sources are affected by the command.""" 34 | tables_help = """Specifies the tables within a source . If omitted all tables are affected by the command.""" 35 | logid_help = """Specifies the log id entry for displaying the error details""" 36 | debug_help = """Forces the debug mode with logging on stdout and log level debug.""" 37 | version_help = """Displays pg_chameleon's installed version.""" 38 | rollbar_help = """Overrides the level for messages to be sent to rolllbar. One of: "critical", "error", "warning", "info". The Default is "info" """ 39 | full_help = """When specified with run_maintenance the switch performs a vacuum full instead of a normal vacuum. """ 40 | truncate_help = """Truncate the existing tables instead of replacing them.""" 41 | 42 | parser = argparse.ArgumentParser(description='Command line for pg_chameleon.', add_help=True) 43 | parser.add_argument('command', type=str, help=command_help) 44 | parser.add_argument('--config', type=str, default='default', required=False, help=config_help) 45 | parser.add_argument('--schema', type=str, default='*', required=False, help=schema_help) 46 | parser.add_argument('--source', type=str, default='*', required=False, help=source_help) 47 | parser.add_argument('--tables', type=str, default='*', required=False, help=tables_help) 48 | parser.add_argument('--logid', type=str, default='*', required=False, help=logid_help) 49 | parser.add_argument('--debug', default=False, required=False, help=debug_help, action='store_true') 50 | parser.add_argument('--version', action='version', help=version_help,version='{version}'.format(version=__version__)) 51 | parser.add_argument('--rollbar-level', type=str, default="info", required=False, help=rollbar_help) 52 | parser.add_argument('--full', default=False, required=False, help=full_help, action='store_true') 53 | args = parser.parse_args() 54 | 55 | 56 | replica = replica_engine(args) 57 | if args.debug: 58 | getattr(replica, args.command)() 59 | else: 60 | try: 61 | getattr(replica, args.command)() 62 | except AttributeError: 63 | print("ERROR - Invalid command" ) 64 | print(command_help) 65 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.rst 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import setuptools 4 | 5 | def readme(): 6 | with open('README.rst') as f: 7 | return f.read() 8 | 9 | package_data = {'pg_chameleon': ['configuration/config-example.yml','sql/upgrade/*.sql','sql/drop_schema.sql','sql/create_schema.sql', 'LICENSE.txt']} 10 | 11 | setuptools.setup( 12 | name="pg_chameleon", 13 | version="2.0.21", 14 | description="MySQL to PostgreSQL replica and migration", 15 | long_description=readme(), 16 | author = "Federico Campoli", 17 | author_email = "thedoctor@pgdba.org", 18 | maintainer = "Federico Campoli", 19 | maintainer_email = "thedoctor@pgdba.org", 20 | url="https://codeberg.org/the4thdoctor/pg_chameleon", 21 | license="BSD License", 22 | platforms=[ 23 | "linux" 24 | ], 25 | classifiers=[ 26 | "License :: OSI Approved :: BSD License", 27 | "Environment :: Console", 28 | "Intended Audience :: Developers", 29 | "Intended Audience :: Information Technology", 30 | "Intended Audience :: System Administrators", 31 | "Natural Language :: English", 32 | "Operating System :: POSIX :: BSD", 33 | "Operating System :: POSIX :: Linux", 34 | "Programming Language :: Python", 35 | "Programming Language :: Python :: 3", 36 | "Topic :: Database :: Database Engines/Servers", 37 | "Topic :: Other/Nonlisted Topic" 38 | ], 39 | py_modules=[ 40 | "pg_chameleon.__init__", 41 | "pg_chameleon.lib.global_lib", 42 | "pg_chameleon.lib.mysql_lib", 43 | "pg_chameleon.lib.pg_lib", 44 | "pg_chameleon.lib.sql_util" 45 | ], 46 | scripts=[ 47 | "scripts/chameleon.py", 48 | "scripts/chameleon" 49 | ], 50 | install_requires=[ 51 | 'PyMySQL>=0.10.0', 52 | 'mysql-replication>=0.31', 53 | 'psycopg2-binary>=2.8.3', 54 | 'PyYAML>=3.13', 55 | 'tabulate>=0.8.1', 56 | 'daemonize>=2.4.7', 57 | 'rollbar>=0.13.17', 58 | 'parsy>=2.1', 59 | 'Sphinx>=7.4.7' 60 | 61 | ], 62 | include_package_data = True, 63 | package_data=package_data, 64 | packages=setuptools.find_packages(), 65 | python_requires='>=3.7', 66 | keywords='postgresql mysql replica migration database', 67 | 68 | ) 69 | -------------------------------------------------------------------------------- /test_logical_decoding.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import time 3 | import psycopg2 4 | from psycopg2.extras import LogicalReplicationConnection 5 | from psycopg2.extras import REPLICATION_LOGICAL 6 | strconn = "dbname=db_replica user=usr_replica host=foo password=bar port=5432" 7 | log_conn = psycopg2.connect(strconn, connection_factory=LogicalReplicationConnection) 8 | log_cur = log_conn.cursor() 9 | log_cur.create_replication_slot("logical1", slot_type=REPLICATION_LOGICAL, output_plugin="test_decoding") 10 | #log_cur.start_replication(slot_name="logical1", slot_type=REPLICATION_LOGICAL, start_lsn=0, timeline=0, options=None, decode=True) 11 | """while True: 12 | log_cur.send_feedback() 13 | msg = log_cur.read_message() 14 | if msg: 15 | print(msg.payload) 16 | time.sleep(1) 17 | """ 18 | log_cur.drop_replication_slot("logical1") 19 | log_conn.close() 20 | -------------------------------------------------------------------------------- /tests/install_mysql.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | sudo service mysql stop 3 | sudo apt-get remove -y mysql-common mysql-server-5.6 mysql-server-core-5.6 mysql-client-5.6 mysql-client-core-5.6 4 | sudo apt-get -y autoremove 5 | sudo apt-get -y autoclean 6 | sudo rm -rf /var/lib/mysql 7 | sudo rm -rf /var/log/mysql 8 | sudo rm -rf /etc/mysql 9 | echo mysql-apt-config mysql-apt-config/enable-repo select mysql-5.7 | sudo debconf-set-selections 10 | wget https://dev.mysql.com/get/mysql-apt-config_0.8.12-1_all.deb 11 | sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 0x762E3157 12 | sudo apt-key adv --keyserver keys.gnupg.net --recv-keys 0x5072E1F5 13 | sudo DEBIAN_FRONTEND=noninteractive dpkg --install mysql-apt-config_0.8.12-1_all.deb 14 | 15 | 16 | 17 | sudo apt-get update -q 18 | sudo DEBIAN_FRONTEND=noninteractive apt-get install -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" mysql-server 19 | echo "trying to connect to mysql via socket" 20 | sudo mysql -h localhost -u root -e "SELECT version();" 21 | -------------------------------------------------------------------------------- /tests/my5.5.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | server-id = 1 3 | log_bin = /var/log/mysql/mysql-bin.log 4 | expire_logs_days = 10 5 | max_binlog_size = 100M 6 | binlog_format= ROW 7 | -------------------------------------------------------------------------------- /tests/my5.6.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | server-id = 1 3 | log_bin = /var/log/mysql/mysql-bin.log 4 | expire_logs_days = 10 5 | max_binlog_size = 100M 6 | binlog_format= ROW 7 | binlog_row_image=FULL 8 | -------------------------------------------------------------------------------- /tests/my5.7.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | server-id = 1 3 | log_bin = /var/log/mysql/mysql-bin.log 4 | expire_logs_days = 10 5 | max_binlog_size = 100M 6 | binlog_format= ROW 7 | binlog_row_image=FULL 8 | -------------------------------------------------------------------------------- /tests/setup_db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | here=`dirname $0` 3 | psql -c "CREATE USER usr_test WITH PASSWORD 'test';" -U postgres 4 | psql -c 'CREATE DATABASE db_test WITH OWNER usr_test;' -U postgres 5 | 6 | 7 | sudo cp -f ${here}/my5.7.cnf /etc/mysql/conf.d/my.cnf 8 | sudo service mysql restart 9 | sudo cat /var/log/mysql/error.log 10 | 11 | wget http://downloads.mysql.com/docs/sakila-db.tar.gz 12 | tar xfz sakila-db.tar.gz 13 | 14 | sudo mysql -u root < sakila-db/sakila-schema.sql 15 | sudo mysql -u root < sakila-db/sakila-data.sql 16 | 17 | sudo mysql -u root < ${here}/setup_mysql.sql 18 | 19 | 20 | -------------------------------------------------------------------------------- /tests/setup_mysql.sql: -------------------------------------------------------------------------------- 1 | CREATE USER usr_test ; 2 | SET PASSWORD FOR usr_test =PASSWORD('test'); 3 | GRANT ALL ON sakila.* TO 'usr_test'; 4 | GRANT RELOAD ON *.* to 'usr_test'; 5 | GRANT REPLICATION CLIENT ON *.* to 'usr_test'; 6 | GRANT REPLICATION SLAVE ON *.* to 'usr_test'; 7 | FLUSH PRIVILEGES; 8 | -------------------------------------------------------------------------------- /tests/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | here=`dirname $0` 4 | chameleon.py set_configuration_files 5 | cp ${here}/test.yml ~/.pg_chameleon/configuration/ 6 | chameleon.py create_replica_schema --config test 7 | chameleon.py add_source --config test --source mysql 8 | chameleon.py add_source --config test --source pgsql 9 | chameleon.py init_replica --config test --source mysql --debug 10 | chameleon.py init_replica --config test --source pgsql --debug 11 | chameleon.py start_replica --config test --source mysql 12 | chameleon.py show_status --config test --source mysql 13 | chameleon.py stop_replica --config test --source mysql 14 | chameleon.py start_replica --config test --source mysql 15 | chameleon.py stop_all_replicas --config test 16 | chameleon.py drop_replica_schema --config test 17 | 18 | 19 | chameleon create_replica_schema --config test 20 | chameleon add_source --config test --source mysql 21 | chameleon add_source --config test --source pgsql 22 | chameleon init_replica --config test --source mysql --debug 23 | chameleon init_replica --config test --source pgsql --debug 24 | chameleon start_replica --config test --source mysql 25 | chameleon show_status --config test --source mysql 26 | chameleon stop_replica --config test --source mysql 27 | chameleon start_replica --config test --source mysql 28 | chameleon stop_all_replicas --config test 29 | chameleon drop_replica_schema --config test 30 | 31 | 32 | -------------------------------------------------------------------------------- /tests/test.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # global settings 3 | pid_dir: '~/.pg_chameleon/pid/' 4 | log_dir: '~/.pg_chameleon/logs/' 5 | log_dest: file 6 | log_level: info 7 | log_days_keep: 10 8 | rollbar_key: '' 9 | rollbar_env: '' 10 | 11 | # postgres destination connection 12 | pg_conn: 13 | host: "localhost" 14 | port: "5432" 15 | user: "usr_test" 16 | password: "test" 17 | database: "db_test" 18 | charset: "utf8" 19 | 20 | sources: 21 | mysql: 22 | db_conn: 23 | host: "localhost" 24 | port: "3306" 25 | user: "usr_test" 26 | password: "test" 27 | charset: 'utf8' 28 | connect_timeout: 10 29 | schema_mappings: 30 | sakila: my_sakila 31 | limit_tables: 32 | skip_tables: 33 | grant_select_to: 34 | lock_timeout: "120s" 35 | my_server_id: 100 36 | replica_batch_size: 10000 37 | replay_max_rows: 10000 38 | batch_retention: '1 day' 39 | copy_max_memory: "300M" 40 | copy_mode: 'file' 41 | out_dir: /tmp 42 | sleep_loop: 1 43 | type: mysql 44 | 45 | pgsql: 46 | db_conn: 47 | host: "localhost" 48 | port: "5432" 49 | user: "usr_test" 50 | password: "test" 51 | database: "db_test" 52 | charset: 'utf8' 53 | connect_timeout: 10 54 | schema_mappings: 55 | my_sakila: pgsql_sakila 56 | limit_tables: 57 | skip_tables: 58 | copy_max_memory: "300M" 59 | grant_select_to: 60 | lock_timeout: "10s" 61 | my_server_id: 100 62 | replica_batch_size: 3000 63 | replay_max_rows: 10000 64 | sleep_loop: 5 65 | batch_retention: '1 day' 66 | copy_mode: 'file' 67 | out_dir: /tmp 68 | type: pgsql 69 | 70 | 71 | # type_override allows the user to override the default 72 | # type conversion into a different one. 73 | # override_to specifies the destination type which must be a postgresql type 74 | # and the type cast should be possible override_tables specifies 75 | # which tables the override apply. 76 | # If set to "*" then applies to all tables in the replicated schema 77 | # the override applies during the init_replica,sync_tables process 78 | # and for each matching DDL (create table/alter table) 79 | type_override: 80 | "tinyint(1)": 81 | override_to: boolean 82 | override_tables: 83 | - "*" 84 | -------------------------------------------------------------------------------- /tests/test_tokenizer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import unittest 4 | 5 | from pg_chameleon.lib.sql_util import sql_token 6 | 7 | 8 | tokenizer = sql_token() 9 | 10 | 11 | class TestTokenizer(unittest.TestCase): 12 | param_list = [ 13 | ( 14 | "column_definition", 15 | "id int PRIMARY KEY", 16 | dict(column_name="id", data_type="int", 17 | dimensions=None, enum_list=None, extras=["PRIMARY KEY"]) 18 | ), 19 | ( 20 | "any_key_definition", 21 | "CONSTRAINT pk_id PRIMARY KEY (id)", 22 | dict(index_name="PRIMARY", index_columns=["id"], non_unique=0) 23 | ), 24 | ( 25 | "any_key_definition", 26 | "CONSTRAINT pk_id1_id2 PRIMARY KEY (id1,id2)", 27 | dict(index_name="PRIMARY", index_columns=["id1", "id2"], non_unique=0) 28 | ), 29 | ( 30 | "any_key_definition", 31 | "PRIMARY\n KEY(id1, id2, `id3`, `id 4` )", 32 | dict(index_name="PRIMARY", index_columns=["id1", "id2", "id3", "id 4"], non_unique=0) 33 | ), 34 | ( 35 | "any_key_definition", 36 | "UNIQUE KEY (id1, id2)", 37 | dict(index_name="UNIQUE", index_columns=["id1", "id2"], non_unique=0) 38 | ), 39 | ( 40 | "any_key_definition", 41 | "UNIQUE KEY (id1)", 42 | dict(index_name="UNIQUE", index_columns=["id1"], non_unique=0) 43 | ), 44 | ( 45 | "any_key_definition", 46 | "UNIQUE (id1, id2)", 47 | dict(index_name="UNIQUE", index_columns=["id1", "id2"], non_unique=0) 48 | ), 49 | ( 50 | "any_key_definition", 51 | "KEY (id1)", 52 | dict(index_name="INDEX", index_columns=["id1"], non_unique=1) 53 | ), 54 | ( 55 | "any_key_definition", 56 | "FULLTEXT KEY asdf (id)", 57 | dict(index_name="OTHER", index_columns=["id"], non_unique=1), 58 | ), 59 | ( 60 | "any_key_definition", 61 | "FOREIGN KEY (column1) REFERENCES table2 (column2)", 62 | dict(index_name="FOREIGN", index_columns=["column1"], non_unique=1) 63 | ), 64 | ( 65 | "column_definition", 66 | """film_rating smallint not null 67 | DEFAULT 3""", 68 | dict(column_name="film_rating", data_type="smallint", 69 | dimensions=None, enum_list=None, 70 | extras=["NOT NULL", ["DEFAULT", "3"]]) 71 | ), 72 | ( 73 | "column_definition", 74 | """film_rating 75 | ENUM ('1 star', '2 star', '3 star', '4 star', '5 star') 76 | DEFAULT '3 star'""", 77 | dict(column_name="film_rating", data_type="enum", 78 | dimensions=None, enum_list=["1 star", "2 star", "3 star", "4 star", "5 star"], 79 | extras=[["DEFAULT", "'3 star'"]]) 80 | ) 81 | ] 82 | 83 | def test_parses_successfully(self): 84 | for parser_name, string, expected in self.param_list: 85 | parser = getattr(tokenizer, parser_name) 86 | with self.subTest( 87 | msg=f"Checking parser {parser_name}", string=string, expected=expected 88 | ): 89 | output = parser.parse(string) 90 | self.assertEqual(output, expected) 91 | 92 | 93 | if __name__ == "__main__": 94 | unittest.main() 95 | --------------------------------------------------------------------------------