├── .editorconfig ├── .github └── ISSUE_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── docs ├── Makefile ├── authors.rst ├── conf.py ├── contributing.rst ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── modules.rst ├── readme.rst ├── twindb_table_compare.rst └── usage.rst ├── pylintrc ├── requirements.txt ├── requirements_dev.txt ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── functional │ ├── conftest.py │ └── test_twindb_table_compare_functional.py └── unit │ ├── conftest.py │ └── test_twindb_table_compare.py ├── tox.ini ├── twindb_table_compare ├── __init__.py ├── cli.py └── compare.py └── vagrant ├── Vagrantfile └── environment └── puppet ├── manifests └── site.pp └── modules ├── profile ├── files │ ├── bashrc │ ├── id_rsa │ ├── id_rsa.pub │ ├── my-master.cnf │ ├── my-slave.cnf │ └── mysql_grants.sql ├── manifests │ ├── base.pp │ ├── master.pp │ └── slave.pp ├── metadata.json └── profile.iml └── role ├── manifests ├── master.pp └── slave.pp ├── metadata.json └── role.iml /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * TwinDB Table Compare version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | 56 | # Sphinx documentation 57 | docs/_build/ 58 | 59 | # PyBuilder 60 | target/ 61 | 62 | # Vagrant 63 | .vagrant 64 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | deploy: 2 | true: 3 | repo: twindb/twindb-table-compare 4 | tags: true 5 | distributions: sdist bdist_wheel 6 | password: 7 | secure: iSEjBKU+FLGl9QkmiJLlLE9YREPSd446aL6at8gN5EyQC38uIQ/8Nc/pyioEz/rduVtQbXLZqqjI4mET/avoLKLXBwpdSmkXm+sBqBNTyeR4Wc4sJ/tbzI7M+Q2MoXkg9N5ryXVV7D+zsbnBlN/076oNy2BRNuNZVouEHpiqrwMae4EfxUR7V/Iqxk7x6LIEAyYh9IJHVUa79NTx5DQqFqee1Tu1pcIasJRGcO44aOoelO9Ytp8oJJHOvLXxvPaPWRHbw9Epg8/xowm0Hrm2yBQ1SCvVC9BrjQX9nGcXVgO/ySoQJCy/b/YMe/1G4lw9SWPFNJLaQojwUWuvbGslfIbBHMxetNuA2u7z4nmgSSwsvcqvLI7KpNu1IKjKdE6/v4Y6jUesw+SoxWOVeQcNP5Rhpg+tVBPPnS5/ymGycZqHFzxoj1NFBZcn/fghX251KLqqJdZNPqR+W6DQP+72U6ocKBRmoCARDRjqXMBj+LCP38nnGQo1xYwGCB518P5qE43/0J8HL8/uAVbyZnqlGf/8MBmttPIYVD6Pf3sBkD6b/Ld5QHOZsSk3mZOZDfr/2XFlB47VPXj8QTB5xR9Ksbn5PK/2hs89padMHosBT+7/VQ1+Uk94HRs7gPOn9Diku5LBXfuUYL/UJmOXGDUiaLkZ2amKq6AprP5Z+BKQ96o= 8 | provider: pypi 9 | user: twindb 10 | skip_existing: true 11 | install: pip install -U tox 12 | language: python 13 | python: 14 | - 3.6 15 | - 3.7 16 | - 3.8 17 | script: make bootstrap lint test coverage 18 | 19 | after_success: 20 | - bash <(curl -s https://codecov.io/bash) 21 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Aleksandr Kuzminsky 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every 8 | little bit helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/twindb/twindb_table_compare/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" 30 | and "help wanted" is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "enhancement" 36 | and "help wanted" is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | TwinDB Table Compare could always use more documentation, whether as part of the 42 | official TwinDB Table Compare docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/twindb/twindb_table_compare/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up `twindb_table_compare` for local development. 61 | 62 | 1. Fork the `twindb_table_compare` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/twindb_table_compare.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv twindb_table_compare 70 | $ cd twindb_table_compare/ 71 | $ make bootstrap 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass lenter and 80 | the tests, including testing other Python versions with ``tox``:: 81 | 82 | $ make lint 83 | $ make test-all 84 | 85 | To get flake8 and tox, just pip install them into your virtualenv. 86 | 87 | 6. Commit your changes and push your branch to GitHub:: 88 | 89 | $ git add . 90 | $ git commit -m "Your detailed description of your changes." 91 | $ git push origin name-of-your-bugfix-or-feature 92 | 93 | 7. Submit a pull request through the GitHub website. 94 | 95 | Pull Request Guidelines 96 | ----------------------- 97 | 98 | Before you submit a pull request, check that it meets these guidelines: 99 | 100 | 1. The pull request should include tests. 101 | 2. If the pull request adds functionality, the docs should be updated. Put 102 | your new functionality into a function with a docstring, and add the 103 | feature to the list in README.rst. 104 | 3. The pull request should work for Python 2.6, 2.7, and for PyPy. Check 105 | https://travis-ci.org/twindb/twindb_table_compare/pull_requests 106 | and make sure that the tests pass for all supported Python versions. 107 | 108 | Tips 109 | ---- 110 | 111 | To run a subset of tests:: 112 | 113 | $ py.test tests/unit/test_twindb_table_compare.py 114 | $ py.test tests/unit/test_twindb_table_compare.py::test_build_chunk_query 115 | 116 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 3.0.2 (2020-09-13) 6 | ------------------ 7 | 8 | * Python 3 support 9 | * Command line tool renamed to twindb-table-compare 10 | 11 | 12 | 0.1.0 (2016-08-29) 13 | ------------------ 14 | 15 | * First release on PyPI. 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache Software License 2.0 3 | 4 | Copyright (c) 2016, Aleksandr Kuzminsky 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | 2 | include AUTHORS.rst 3 | 4 | include CONTRIBUTING.rst 5 | include HISTORY.rst 6 | include LICENSE 7 | include README.rst 8 | include requirements.txt 9 | include requirements_dev.txt 10 | 11 | recursive-include tests * 12 | recursive-exclude * __pycache__ 13 | recursive-exclude * *.py[co] 14 | 15 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | define BROWSER_PYSCRIPT 4 | import os, webbrowser, sys 5 | try: 6 | from urllib import pathname2url 7 | except: 8 | from urllib.request import pathname2url 9 | 10 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 11 | endef 12 | export BROWSER_PYSCRIPT 13 | 14 | define PRINT_HELP_PYSCRIPT 15 | import re, sys 16 | 17 | for line in sys.stdin: 18 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 19 | if match: 20 | target, help = match.groups() 21 | print("%-20s %s" % (target, help)) 22 | endef 23 | export PRINT_HELP_PYSCRIPT 24 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 25 | 26 | help: 27 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 28 | 29 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 30 | 31 | 32 | clean-build: ## remove build artifacts 33 | rm -fr build/ 34 | rm -fr dist/ 35 | rm -fr .eggs/ 36 | find . -name '*.egg-info' -exec rm -fr {} + 37 | find . -name '*.egg' -exec rm -f {} + 38 | 39 | clean-pyc: ## remove Python file artifacts 40 | find . -name '*.pyc' -exec rm -f {} + 41 | find . -name '*.pyo' -exec rm -f {} + 42 | find . -name '*~' -exec rm -f {} + 43 | find . -name '__pycache__' -exec rm -fr {} + 44 | 45 | clean-test: clean-pyc ## remove test and coverage artifacts 46 | rm -fr .tox/ 47 | rm -f .coverage 48 | rm -fr htmlcov/ 49 | 50 | lint: ## check style with flake8 51 | pylint twindb_table_compare 52 | 53 | vagrant-up: 54 | cd vagrant && vagrant up 55 | 56 | vagrant-provision: 57 | cd vagrant && vagrant provision 58 | 59 | .PHONY: bootstrap 60 | bootstrap: ## bootstrap the development environment 61 | pip install -U "pip ~= 20.2" 62 | pip install -U "setuptools ~= 50.0" 63 | pip install -r requirements.txt -r requirements_dev.txt 64 | pip install --editable . 65 | 66 | test: ## run tests quickly with the default Python 67 | pytest -vx --cov=twindb_table_compare --cov-report term-missing tests/unit/ 68 | 69 | test-functional: ## run functional tests 70 | pytest -vx tests/functional/ 71 | 72 | test-all: ## run tests on every Python version with tox 73 | tox 74 | 75 | docs: ## generate Sphinx HTML documentation, including API docs 76 | rm -f docs/twindb_table_compare.rst 77 | rm -f docs/modules.rst 78 | sphinx-apidoc -o docs/ twindb_table_compare 79 | $(MAKE) -C docs clean 80 | $(MAKE) -C docs html 81 | $(BROWSER) docs/_build/html/index.html 82 | 83 | coverage: 84 | codecov 85 | 86 | dist: clean ## builds source and wheel package 87 | python setup.py sdist 88 | python setup.py bdist_wheel 89 | ls -l dist 90 | 91 | install: clean ## install the package to the active Python's site-packages 92 | python setup.py install 93 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ==================== 2 | TwinDB Table Compare 3 | ==================== 4 | 5 | .. image:: https://img.shields.io/pypi/v/twindb_table_compare.svg 6 | :target: https://pypi.python.org/pypi/twindb_table_compare 7 | 8 | .. image:: https://img.shields.io/travis/twindb/twindb-table-compare.svg 9 | :target: https://travis-ci.org/twindb/twindb-table-compare 10 | 11 | .. image:: https://readthedocs.org/projects/twindb-table-compare/badge/?version=master 12 | :target: https://twindb-table-compare.readthedocs.io/en/master/?badge=master 13 | :alt: Documentation Status 14 | 15 | .. image:: https://pyup.io/repos/github/twindb/twindb-table-compare/shield.svg 16 | :target: https://pyup.io/repos/github/twindb/twindb-table-compare/ 17 | :alt: Updates 18 | 19 | .. image:: https://img.shields.io/pypi/dd/Django.svg?maxAge=2592000 20 | :target: https://pypi.python.org/pypi/twindb-table-compare 21 | :alt: Pypi 22 | 23 | 24 | TwinDB Table Compare reads ``percona``.``checksums`` from the master and slave 25 | and shows what records are difference if there are any inconsistencies. 26 | 27 | 28 | * Free software: Apache Software License 2.0 29 | * Documentation: https://twindb-table-compare.readthedocs.io. 30 | * Blogpost with usage examples: https://twindb.com/pt-table-checksum-show-differences/ 31 | 32 | 33 | 34 | Credits 35 | ------- 36 | 37 | This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. 38 | 39 | .. _Cookiecutter: https://github.com/audreyr/cookiecutter 40 | .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage 41 | 42 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/twindb_table_compare.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/twindb_table_compare.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/twindb_table_compare" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/twindb_table_compare" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # twindb_table_compare documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Jul 9 22:26:36 2013. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import sys 17 | import os 18 | 19 | # If extensions (or modules to document with autodoc) are in another 20 | # directory, add these directories to sys.path here. If the directory is 21 | # relative to the documentation root, use os.path.abspath to make it 22 | # absolute, like shown here. 23 | #sys.path.insert(0, os.path.abspath('.')) 24 | 25 | # Get the project root dir, which is the parent dir of this 26 | cwd = os.getcwd() 27 | project_root = os.path.dirname(cwd) 28 | 29 | # Insert the project root dir as the first element in the PYTHONPATH. 30 | # This lets us ensure that the source package is imported, and that its 31 | # version is used. 32 | sys.path.insert(0, project_root) 33 | 34 | import twindb_table_compare 35 | 36 | # -- General configuration --------------------------------------------- 37 | 38 | # If your documentation needs a minimal Sphinx version, state it here. 39 | #needs_sphinx = '1.0' 40 | 41 | # Add any Sphinx extension module names here, as strings. They can be 42 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 43 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # The suffix of source filenames. 49 | source_suffix = '.rst' 50 | 51 | # The encoding of source files. 52 | #source_encoding = 'utf-8-sig' 53 | 54 | # The master toctree document. 55 | master_doc = 'index' 56 | 57 | # General information about the project. 58 | project = u'TwinDB Table Compare' 59 | copyright = u"2016, Aleksandr Kuzminsky" 60 | 61 | # The version info for the project you're documenting, acts as replacement 62 | # for |version| and |release|, also used in various other places throughout 63 | # the built documents. 64 | # 65 | # The short X.Y version. 66 | version = twindb_table_compare.__version__ 67 | # The full version, including alpha/beta/rc tags. 68 | release = twindb_table_compare.__version__ 69 | 70 | # The language for content autogenerated by Sphinx. Refer to documentation 71 | # for a list of supported languages. 72 | #language = None 73 | 74 | # There are two options for replacing |today|: either, you set today to 75 | # some non-false value, then it is used: 76 | #today = '' 77 | # Else, today_fmt is used as the format for a strftime call. 78 | #today_fmt = '%B %d, %Y' 79 | 80 | # List of patterns, relative to source directory, that match files and 81 | # directories to ignore when looking for source files. 82 | exclude_patterns = ['_build'] 83 | 84 | # The reST default role (used for this markup: `text`) to use for all 85 | # documents. 86 | #default_role = None 87 | 88 | # If true, '()' will be appended to :func: etc. cross-reference text. 89 | #add_function_parentheses = True 90 | 91 | # If true, the current module name will be prepended to all description 92 | # unit titles (such as .. function::). 93 | #add_module_names = True 94 | 95 | # If true, sectionauthor and moduleauthor directives will be shown in the 96 | # output. They are ignored by default. 97 | #show_authors = False 98 | 99 | # The name of the Pygments (syntax highlighting) style to use. 100 | pygments_style = 'sphinx' 101 | 102 | # A list of ignored prefixes for module index sorting. 103 | #modindex_common_prefix = [] 104 | 105 | # If true, keep warnings as "system message" paragraphs in the built 106 | # documents. 107 | #keep_warnings = False 108 | 109 | 110 | # -- Options for HTML output ------------------------------------------- 111 | 112 | # The theme to use for HTML and HTML Help pages. See the documentation for 113 | # a list of builtin themes. 114 | html_theme = 'default' 115 | 116 | # Theme options are theme-specific and customize the look and feel of a 117 | # theme further. For a list of options available for each theme, see the 118 | # documentation. 119 | #html_theme_options = {} 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | #html_theme_path = [] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as 129 | # html_title. 130 | #html_short_title = None 131 | 132 | # The name of an image file (relative to this directory) to place at the 133 | # top of the sidebar. 134 | #html_logo = None 135 | 136 | # The name of an image file (within the static path) to use as favicon 137 | # of the docs. This file should be a Windows icon file (.ico) being 138 | # 16x16 or 32x32 pixels large. 139 | #html_favicon = None 140 | 141 | # Add any paths that contain custom static files (such as style sheets) 142 | # here, relative to this directory. They are copied after the builtin 143 | # static files, so a file named "default.css" will overwrite the builtin 144 | # "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # If not '', a 'Last updated on:' timestamp is inserted at every page 148 | # bottom, using the given strftime format. 149 | #html_last_updated_fmt = '%b %d, %Y' 150 | 151 | # If true, SmartyPants will be used to convert quotes and dashes to 152 | # typographically correct entities. 153 | #html_use_smartypants = True 154 | 155 | # Custom sidebar templates, maps document names to template names. 156 | #html_sidebars = {} 157 | 158 | # Additional templates that should be rendered to pages, maps page names 159 | # to template names. 160 | #html_additional_pages = {} 161 | 162 | # If false, no module index is generated. 163 | #html_domain_indices = True 164 | 165 | # If false, no index is generated. 166 | #html_use_index = True 167 | 168 | # If true, the index is split into individual pages for each letter. 169 | #html_split_index = False 170 | 171 | # If true, links to the reST sources are added to the pages. 172 | #html_show_sourcelink = True 173 | 174 | # If true, "Created using Sphinx" is shown in the HTML footer. 175 | # Default is True. 176 | #html_show_sphinx = True 177 | 178 | # If true, "(C) Copyright ..." is shown in the HTML footer. 179 | # Default is True. 180 | #html_show_copyright = True 181 | 182 | # If true, an OpenSearch description file will be output, and all pages 183 | # will contain a tag referring to it. The value of this option 184 | # must be the base URL from which the finished HTML is served. 185 | #html_use_opensearch = '' 186 | 187 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 188 | #html_file_suffix = None 189 | 190 | # Output file base name for HTML help builder. 191 | htmlhelp_basename = 'twindb_table_comparedoc' 192 | 193 | 194 | # -- Options for LaTeX output ------------------------------------------ 195 | 196 | latex_elements = { 197 | # The paper size ('letterpaper' or 'a4paper'). 198 | #'papersize': 'letterpaper', 199 | 200 | # The font size ('10pt', '11pt' or '12pt'). 201 | #'pointsize': '10pt', 202 | 203 | # Additional stuff for the LaTeX preamble. 204 | #'preamble': '', 205 | } 206 | 207 | # Grouping the document tree into LaTeX files. List of tuples 208 | # (source start file, target name, title, author, documentclass 209 | # [howto/manual]). 210 | latex_documents = [ 211 | ('index', 'twindb_table_compare.tex', 212 | u'TwinDB Table Compare Documentation', 213 | u'Aleksandr Kuzminsky', 'manual'), 214 | ] 215 | 216 | # The name of an image file (relative to this directory) to place at 217 | # the top of the title page. 218 | #latex_logo = None 219 | 220 | # For "manual" documents, if this is true, then toplevel headings 221 | # are parts, not chapters. 222 | #latex_use_parts = False 223 | 224 | # If true, show page references after internal links. 225 | #latex_show_pagerefs = False 226 | 227 | # If true, show URL addresses after external links. 228 | #latex_show_urls = False 229 | 230 | # Documents to append as an appendix to all manuals. 231 | #latex_appendices = [] 232 | 233 | # If false, no module index is generated. 234 | #latex_domain_indices = True 235 | 236 | 237 | # -- Options for manual page output ------------------------------------ 238 | 239 | # One entry per manual page. List of tuples 240 | # (source start file, name, description, authors, manual section). 241 | man_pages = [ 242 | ('index', 'twindb_table_compare', 243 | u'TwinDB Table Compare Documentation', 244 | [u'Aleksandr Kuzminsky'], 1) 245 | ] 246 | 247 | # If true, show URL addresses after external links. 248 | #man_show_urls = False 249 | 250 | 251 | # -- Options for Texinfo output ---------------------------------------- 252 | 253 | # Grouping the document tree into Texinfo files. List of tuples 254 | # (source start file, target name, title, author, 255 | # dir menu entry, description, category) 256 | texinfo_documents = [ 257 | ('index', 'twindb_table_compare', 258 | u'TwinDB Table Compare Documentation', 259 | u'Aleksandr Kuzminsky', 260 | 'twindb_table_compare', 261 | 'One line description of project.', 262 | 'Miscellaneous'), 263 | ] 264 | 265 | # Documents to append as an appendix to all manuals. 266 | #texinfo_appendices = [] 267 | 268 | # If false, no module index is generated. 269 | #texinfo_domain_indices = True 270 | 271 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 272 | #texinfo_show_urls = 'footnote' 273 | 274 | # If true, do not generate a @detailmenu in the "Top" node's menu. 275 | #texinfo_no_detailmenu = False 276 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. twindb_table_compare documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to TwinDB Table Compare's documentation! 7 | ================================================ 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | modules 18 | history 19 | contributing 20 | authors 21 | 22 | 23 | Indices and tables 24 | ================== 25 | 26 | * :ref:`genindex` 27 | * :ref:`modindex` 28 | * :ref:`search` 29 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Installation 5 | ============ 6 | 7 | 8 | Supported Python versions 9 | ------------------------- 10 | 11 | * Python 3.6, 3.7, and 3.8 are supported. 12 | 13 | Stable release 14 | -------------- 15 | 16 | To install TwinDB Table Compare, run this command in your terminal: 17 | 18 | .. code-block:: console 19 | 20 | $ pip install twindb-table-compare 21 | 22 | This is the preferred method to install TwinDB Table Compare, as it will always install the most recent stable release. 23 | 24 | If you don't have `pip`_ installed, this `Python installation guide`_ can guide 25 | you through the process. 26 | 27 | .. _pip: https://pip.pypa.io 28 | .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ 29 | 30 | 31 | From sources 32 | ------------ 33 | 34 | The sources for TwinDB Table Compare can be downloaded from the `Github repo`_. 35 | 36 | You can either clone the public repository: 37 | 38 | .. code-block:: console 39 | 40 | $ git clone https://github.com/twindb/twindb-table-compare.git 41 | 42 | Or download the `tarball`_: 43 | 44 | .. code-block:: console 45 | 46 | $ curl -OL https://github.com/twindb/twindb-table-compare/tarball/master 47 | 48 | Once you have a copy of the source, you can install it with: 49 | 50 | .. code-block:: console 51 | 52 | $ python setup.py install 53 | 54 | 55 | .. _Github repo: https://github.com/twindb/twindb-table-compare 56 | .. _tarball: https://github.com/twindb/twindb-table-compare/tarball/master 57 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 31 | echo. text to make text files 32 | echo. man to make manual pages 33 | echo. texinfo to make Texinfo files 34 | echo. gettext to make PO message catalogs 35 | echo. changes to make an overview over all changed/added/deprecated items 36 | echo. xml to make Docutils-native XML files 37 | echo. pseudoxml to make pseudoxml-XML files for display purposes 38 | echo. linkcheck to check all external links for integrity 39 | echo. doctest to run all doctests embedded in the documentation if enabled 40 | goto end 41 | ) 42 | 43 | if "%1" == "clean" ( 44 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 45 | del /q /s %BUILDDIR%\* 46 | goto end 47 | ) 48 | 49 | 50 | %SPHINXBUILD% 2> nul 51 | if errorlevel 9009 ( 52 | echo. 53 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 54 | echo.installed, then set the SPHINXBUILD environment variable to point 55 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 56 | echo.may add the Sphinx directory to PATH. 57 | echo. 58 | echo.If you don't have Sphinx installed, grab it from 59 | echo.http://sphinx-doc.org/ 60 | exit /b 1 61 | ) 62 | 63 | if "%1" == "html" ( 64 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 65 | if errorlevel 1 exit /b 1 66 | echo. 67 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 68 | goto end 69 | ) 70 | 71 | if "%1" == "dirhtml" ( 72 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 73 | if errorlevel 1 exit /b 1 74 | echo. 75 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 76 | goto end 77 | ) 78 | 79 | if "%1" == "singlehtml" ( 80 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 81 | if errorlevel 1 exit /b 1 82 | echo. 83 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 84 | goto end 85 | ) 86 | 87 | if "%1" == "pickle" ( 88 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 89 | if errorlevel 1 exit /b 1 90 | echo. 91 | echo.Build finished; now you can process the pickle files. 92 | goto end 93 | ) 94 | 95 | if "%1" == "json" ( 96 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 97 | if errorlevel 1 exit /b 1 98 | echo. 99 | echo.Build finished; now you can process the JSON files. 100 | goto end 101 | ) 102 | 103 | if "%1" == "htmlhelp" ( 104 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 105 | if errorlevel 1 exit /b 1 106 | echo. 107 | echo.Build finished; now you can run HTML Help Workshop with the ^ 108 | .hhp project file in %BUILDDIR%/htmlhelp. 109 | goto end 110 | ) 111 | 112 | if "%1" == "qthelp" ( 113 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 117 | .qhcp project file in %BUILDDIR%/qthelp, like this: 118 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\twindb_table_compare.qhcp 119 | echo.To view the help file: 120 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\twindb_table_compare.ghc 121 | goto end 122 | ) 123 | 124 | if "%1" == "devhelp" ( 125 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 126 | if errorlevel 1 exit /b 1 127 | echo. 128 | echo.Build finished. 129 | goto end 130 | ) 131 | 132 | if "%1" == "epub" ( 133 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 134 | if errorlevel 1 exit /b 1 135 | echo. 136 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 137 | goto end 138 | ) 139 | 140 | if "%1" == "latex" ( 141 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 142 | if errorlevel 1 exit /b 1 143 | echo. 144 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 145 | goto end 146 | ) 147 | 148 | if "%1" == "latexpdf" ( 149 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 150 | cd %BUILDDIR%/latex 151 | make all-pdf 152 | cd %BUILDDIR%/.. 153 | echo. 154 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 155 | goto end 156 | ) 157 | 158 | if "%1" == "latexpdfja" ( 159 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 160 | cd %BUILDDIR%/latex 161 | make all-pdf-ja 162 | cd %BUILDDIR%/.. 163 | echo. 164 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 165 | goto end 166 | ) 167 | 168 | if "%1" == "text" ( 169 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 170 | if errorlevel 1 exit /b 1 171 | echo. 172 | echo.Build finished. The text files are in %BUILDDIR%/text. 173 | goto end 174 | ) 175 | 176 | if "%1" == "man" ( 177 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 178 | if errorlevel 1 exit /b 1 179 | echo. 180 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 181 | goto end 182 | ) 183 | 184 | if "%1" == "texinfo" ( 185 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 186 | if errorlevel 1 exit /b 1 187 | echo. 188 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 189 | goto end 190 | ) 191 | 192 | if "%1" == "gettext" ( 193 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 194 | if errorlevel 1 exit /b 1 195 | echo. 196 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 197 | goto end 198 | ) 199 | 200 | if "%1" == "changes" ( 201 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 202 | if errorlevel 1 exit /b 1 203 | echo. 204 | echo.The overview file is in %BUILDDIR%/changes. 205 | goto end 206 | ) 207 | 208 | if "%1" == "linkcheck" ( 209 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 210 | if errorlevel 1 exit /b 1 211 | echo. 212 | echo.Link check complete; look for any errors in the above output ^ 213 | or in %BUILDDIR%/linkcheck/output.txt. 214 | goto end 215 | ) 216 | 217 | if "%1" == "doctest" ( 218 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 219 | if errorlevel 1 exit /b 1 220 | echo. 221 | echo.Testing of doctests in the sources finished, look at the ^ 222 | results in %BUILDDIR%/doctest/output.txt. 223 | goto end 224 | ) 225 | 226 | if "%1" == "xml" ( 227 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 228 | if errorlevel 1 exit /b 1 229 | echo. 230 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 231 | goto end 232 | ) 233 | 234 | if "%1" == "pseudoxml" ( 235 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 236 | if errorlevel 1 exit /b 1 237 | echo. 238 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 239 | goto end 240 | ) 241 | 242 | :end 243 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | twindb_table_compare 2 | ==================== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | twindb_table_compare 8 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/twindb_table_compare.rst: -------------------------------------------------------------------------------- 1 | twindb\_table\_compare package 2 | ============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_table\_compare.cli module 8 | --------------------------------- 9 | 10 | .. automodule:: twindb_table_compare.cli 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_table\_compare.compare module 16 | ------------------------------------- 17 | 18 | .. automodule:: twindb_table_compare.compare 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: twindb_table_compare 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ----- 3 | 4 | TwinDB Table Compare should be used in the command line. 5 | 6 | 7 | .. note:: 8 | 9 | Before running TwinDB Table Compare tool, you should run ``pt-table-checksum`` first. 10 | ``twindb-table-compare`` uses ``percona.checksums``. If you don't run ``pt-table-checksum``, 11 | ``percona.checksums`` will be empty and thus ``twindb-table-compare`` won't show any differences. 12 | 13 | .. note:: 14 | 15 | ``twindb-table-compare`` should run agains the slave. 16 | 17 | This will show differences in data between a *slave* and its master. 18 | 19 | .. code-block:: shell 20 | 21 | twindb-table-compare slave 22 | 23 | 24 | where *slave* is a hostname of a MySQL slave. 25 | 26 | :: 27 | 28 | # twindb-table-compare --user=dba --password=qwerty 192.168.35.251 29 | 2016-09-03 22:48:01,732: INFO: twindb_table_compare.get_inconsistencies():127: Executing: SELECT chunk FROM `percona`.`checksums` WHERE (this_crc<>master_crc OR this_cnt<>master_cnt) AND db='mysql' AND tbl='proxies_priv' 30 | 2016-09-03 22:48:01,734: INFO: twindb_table_compare.get_inconsistencies():138: Found 1 inconsistent chunk 31 | 2016-09-03 22:48:01,734: INFO: twindb_table_compare.get_inconsistencies():141: # mysql.proxies_priv, chunk 1 32 | 2016-09-03 22:48:01,736: INFO: twindb_table_compare.get_inconsistencies():143: # chunk index: None 33 | 2016-09-03 22:48:01,736: INFO: twindb_table_compare.get_inconsistencies():215: Executing: SELECT * FROM `mysql`.`proxies_priv` WHERE 1 34 | 2016-09-03 22:48:01,743: INFO: twindb_table_compare.get_inconsistencies():257: Differences between slave 192.168.35.251 and its master: 35 | --- /tmp/master.GZ8S7V 2016-09-03 22:48:01.737762174 +0000 36 | +++ /tmp/slave.9t4HhV 2016-09-03 22:48:01.738761674 +0000 37 | @@ -1,2 +1,2 @@ 38 | -localhost root 1 2016-09-03 20:02:28 39 | -master.box root 1 2016-09-03 20:02:28 40 | +localhost root 1 2016-09-03 20:10:04 41 | +slave.box root 1 2016-09-03 20:10:04 42 | 43 | 2016-09-03 22:48:01,746: INFO: twindb_table_compare.get_inconsistencies():127: Executing: SELECT chunk FROM `percona`.`checksums` WHERE (this_crc<>master_crc OR this_cnt<>master_cnt) AND db='mysql' AND tbl='user' 44 | 2016-09-03 22:48:01,747: INFO: twindb_table_compare.get_inconsistencies():138: Found 1 inconsistent chunk 45 | 2016-09-03 22:48:01,747: INFO: twindb_table_compare.get_inconsistencies():141: # mysql.user, chunk 1 46 | 2016-09-03 22:48:01,747: INFO: twindb_table_compare.get_inconsistencies():143: # chunk index: None 47 | 2016-09-03 22:48:01,748: INFO: twindb_table_compare.get_inconsistencies():215: Executing: SELECT * FROM `mysql`.`user` WHERE 1 48 | 2016-09-03 22:48:01,757: INFO: twindb_table_compare.get_inconsistencies():257: Differences between slave 192.168.35.251 and its master: 49 | --- /tmp/master.l_zYw7 2016-09-03 22:48:01.749756174 +0000 50 | +++ /tmp/slave.39qG9N 2016-09-03 22:48:01.752754674 +0000 51 | @@ -1,9 +1,9 @@ 52 | localhost root Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0 mysql_native_password N 53 | -master.box root Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0 mysql_native_password N 54 | +slave.box root Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0 mysql_native_password N 55 | 127.0.0.1 root Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0 mysql_native_password N 56 | ::1 root Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0 mysql_native_password N 57 | localhost N N N N N N N N N N N N N N N N N N N N N N N N N N N N N 0 0 0 0 mysql_native_password None N 58 | -master.box N N N N N N N N N N N N N N N N N N N N N N N N N N N N N 0 0 0 0 mysql_native_password None N 59 | +slave.box N N N N N N N N N N N N N N N N N N N N N N N N N N N N N 0 0 0 0 mysql_native_password None N 60 | % dba *AA1420F182E88B9E5F874F6FBE7459291E8F4601 Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0 mysql_native_password N 61 | localhost dba *AA1420F182E88B9E5F874F6FBE7459291E8F4601 Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y 0 0 0 0mysql_native_password N 62 | % repl *809534247D21AC735802078139D8A854F45C31F3 N N N N N N N N N N N N N N N N N N N Y N N N N N N N N N 0 0 0 0 mysql_native_password N 63 | 64 | Run ``twindb-table-compare --help`` for other options. 65 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Add files or directories to the blacklist. They should be base names, not 11 | # paths. 12 | ignore=CVS 13 | 14 | # Add files or directories matching the regex patterns to the blacklist. The 15 | # regex matches against base names, not paths. 16 | ignore-patterns= 17 | 18 | # Pickle collected data for later comparisons. 19 | persistent=yes 20 | 21 | # List of plugins (as comma separated values of python modules names) to load, 22 | # usually to register additional checkers. 23 | load-plugins= 24 | 25 | # Use multiple processes to speed up Pylint. 26 | jobs=1 27 | 28 | # Allow loading of arbitrary C extensions. Extensions are imported into the 29 | # active Python interpreter and may run arbitrary code. 30 | unsafe-load-any-extension=no 31 | 32 | # A comma-separated list of package or module names from where C extensions may 33 | # be loaded. Extensions are loading into the active Python interpreter and may 34 | # run arbitrary code 35 | extension-pkg-whitelist= 36 | 37 | # Allow optimization of some AST trees. This will activate a peephole AST 38 | # optimizer, which will apply various small optimizations. For instance, it can 39 | # be used to obtain the result of joining multiple strings with the addition 40 | # operator. Joining a lot of strings can lead to a maximum recursion error in 41 | # Pylint and this flag can prevent that. It has one side effect, the resulting 42 | # AST will be different than the one from reality. This option is deprecated 43 | # and it will be removed in Pylint 2.0. 44 | optimize-ast=no 45 | 46 | 47 | [MESSAGES CONTROL] 48 | 49 | # Only show warnings with the listed confidence levels. Leave empty to show 50 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 51 | confidence= 52 | 53 | # Enable the message, report, category or checker with the given id(s). You can 54 | # either give multiple identifier separated by comma (,) or put this option 55 | # multiple time (only on the command line, not in the configuration file where 56 | # it should appear only once). See also the "--disable" option for examples. 57 | #enable= 58 | 59 | # Disable the message, report, category or checker with the given id(s). You 60 | # can either give multiple identifiers separated by comma (,) or put this 61 | # option multiple times (only on the command line, not in the configuration 62 | # file where it should appear only once).You can also use "--disable=all" to 63 | # disable everything first and then reenable specific checks. For example, if 64 | # you want to run only the similarities checker, you can use "--disable=all 65 | # --enable=similarities". If you want to run only the classes checker, but have 66 | # no Warning level messages displayed, use"--disable=all --enable=classes 67 | # --disable=W" 68 | disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating 69 | 70 | 71 | [REPORTS] 72 | 73 | # Set the output format. Available formats are text, parseable, colorized, msvs 74 | # (visual studio) and html. You can also give a reporter class, eg 75 | # mypackage.mymodule.MyReporterClass. 76 | output-format=text 77 | 78 | # Put messages in a separate file for each module / package specified on the 79 | # command line instead of printing them on stdout. Reports (if any) will be 80 | # written in a file name "pylint_global.[txt|html]". This option is deprecated 81 | # and it will be removed in Pylint 2.0. 82 | files-output=no 83 | 84 | # Tells whether to display a full report or only the messages 85 | reports=yes 86 | 87 | # Python expression which should return a note less than 10 (10 is the highest 88 | # note). You have access to the variables errors warning, statement which 89 | # respectively contain the number of errors / warnings messages and the total 90 | # number of statements analyzed. This is used by the global evaluation report 91 | # (RP0004). 92 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 93 | 94 | # Template used to display messages. This is a python new-style format string 95 | # used to format the message information. See doc for all details 96 | #msg-template= 97 | 98 | 99 | [BASIC] 100 | 101 | # Good variable names which should always be accepted, separated by a comma 102 | good-names=i,j,k,ex,Run,_ 103 | 104 | # Bad variable names which should always be refused, separated by a comma 105 | bad-names=foo,bar,baz,toto,tutu,tata 106 | 107 | # Colon-delimited sets of names that determine each other's naming style when 108 | # the name regexes allow several styles. 109 | name-group= 110 | 111 | # Include a hint for the correct naming format with invalid-name 112 | include-naming-hint=no 113 | 114 | # List of decorators that produce properties, such as abc.abstractproperty. Add 115 | # to this list to register other decorators that produce valid properties. 116 | property-classes=abc.abstractproperty 117 | 118 | # Regular expression matching correct function names 119 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 120 | 121 | # Naming hint for function names 122 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 123 | 124 | # Regular expression matching correct variable names 125 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 126 | 127 | # Naming hint for variable names 128 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 129 | 130 | # Regular expression matching correct constant names 131 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 132 | 133 | # Naming hint for constant names 134 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 135 | 136 | # Regular expression matching correct attribute names 137 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 138 | 139 | # Naming hint for attribute names 140 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 141 | 142 | # Regular expression matching correct argument names 143 | argument-rgx=[a-z_][a-z0-9_]{2,30}$|db$ 144 | 145 | # Naming hint for argument names 146 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 147 | 148 | # Regular expression matching correct class attribute names 149 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 150 | 151 | # Naming hint for class attribute names 152 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 153 | 154 | # Regular expression matching correct inline iteration names 155 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 156 | 157 | # Naming hint for inline iteration names 158 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 159 | 160 | # Regular expression matching correct class names 161 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 162 | 163 | # Naming hint for class names 164 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 165 | 166 | # Regular expression matching correct module names 167 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 168 | 169 | # Naming hint for module names 170 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 171 | 172 | # Regular expression matching correct method names 173 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 174 | 175 | # Naming hint for method names 176 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 177 | 178 | # Regular expression which should only match function or class names that do 179 | # not require a docstring. 180 | no-docstring-rgx=^_ 181 | 182 | # Minimum line length for functions/classes that require docstrings, shorter 183 | # ones are exempt. 184 | docstring-min-length=-1 185 | 186 | 187 | [ELIF] 188 | 189 | # Maximum number of nested blocks for function / method body 190 | max-nested-blocks=5 191 | 192 | 193 | [FORMAT] 194 | 195 | # Maximum number of characters on a single line. 196 | max-line-length=88 197 | 198 | # Regexp for a line that is allowed to be longer than the limit. 199 | ignore-long-lines=^\s*(# )??$ 200 | 201 | # Allow the body of an if to be on the same line as the test if there is no 202 | # else. 203 | single-line-if-stmt=no 204 | 205 | # List of optional constructs for which whitespace checking is disabled. `dict- 206 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 207 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 208 | # `empty-line` allows space-only lines. 209 | no-space-check=trailing-comma,dict-separator 210 | 211 | # Maximum number of lines in a module 212 | max-module-lines=1000 213 | 214 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 215 | # tab). 216 | indent-string=' ' 217 | 218 | # Number of spaces of indent required inside a hanging or continued line. 219 | indent-after-paren=4 220 | 221 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 222 | expected-line-ending-format= 223 | 224 | 225 | [LOGGING] 226 | 227 | # Logging modules to check that the string format arguments are in logging 228 | # function parameter format 229 | logging-modules=logging 230 | 231 | 232 | [MISCELLANEOUS] 233 | 234 | # List of note tags to take in consideration, separated by a comma. 235 | notes=FIXME,XXX,TODO 236 | 237 | 238 | [SIMILARITIES] 239 | 240 | # Minimum lines number of a similarity. 241 | min-similarity-lines=4 242 | 243 | # Ignore comments when computing similarities. 244 | ignore-comments=yes 245 | 246 | # Ignore docstrings when computing similarities. 247 | ignore-docstrings=yes 248 | 249 | # Ignore imports when computing similarities. 250 | ignore-imports=no 251 | 252 | 253 | [SPELLING] 254 | 255 | # Spelling dictionary name. Available dictionaries: none. To make it working 256 | # install python-enchant package. 257 | spelling-dict= 258 | 259 | # List of comma separated words that should not be checked. 260 | spelling-ignore-words= 261 | 262 | # A path to a file that contains private dictionary; one word per line. 263 | spelling-private-dict-file= 264 | 265 | # Tells whether to store unknown words to indicated private dictionary in 266 | # --spelling-private-dict-file option instead of raising a message. 267 | spelling-store-unknown-words=no 268 | 269 | 270 | [TYPECHECK] 271 | 272 | # Tells whether missing members accessed in mixin class should be ignored. A 273 | # mixin class is detected if its name ends with "mixin" (case insensitive). 274 | ignore-mixin-members=yes 275 | 276 | # List of module names for which member attributes should not be checked 277 | # (useful for modules/projects where namespaces are manipulated during runtime 278 | # and thus existing member attributes cannot be deduced by static analysis. It 279 | # supports qualified module names, as well as Unix pattern matching. 280 | ignored-modules= 281 | 282 | # List of class names for which member attributes should not be checked (useful 283 | # for classes with dynamically set attributes). This supports the use of 284 | # qualified names. 285 | ignored-classes=optparse.Values,thread._local,_thread._local 286 | 287 | # List of members which are set dynamically and missed by pylint inference 288 | # system, and so shouldn't trigger E1101 when accessed. Python regular 289 | # expressions are accepted. 290 | generated-members= 291 | 292 | # List of decorators that produce context managers, such as 293 | # contextlib.contextmanager. Add to this list to register other decorators that 294 | # produce valid context managers. 295 | contextmanager-decorators=contextlib.contextmanager 296 | 297 | 298 | [VARIABLES] 299 | 300 | # Tells whether we should check for unused import in __init__ files. 301 | init-import=no 302 | 303 | # A regular expression matching the name of dummy variables (i.e. expectedly 304 | # not used). 305 | dummy-variables-rgx=(_+[a-zA-Z0-9]*?$)|dummy 306 | 307 | # List of additional names supposed to be defined in builtins. Remember that 308 | # you should avoid to define new builtins when possible. 309 | additional-builtins= 310 | 311 | # List of strings which can identify a callback function by name. A callback 312 | # name must start or end with one of those strings. 313 | callbacks=cb_,_cb 314 | 315 | # List of qualified module names which can have objects that can redefine 316 | # builtins. 317 | redefining-builtins-modules=six.moves,future.builtins 318 | 319 | 320 | [CLASSES] 321 | 322 | # List of method names used to declare (i.e. assign) instance attributes. 323 | defining-attr-methods=__init__,__new__,setUp 324 | 325 | # List of valid names for the first argument in a class method. 326 | valid-classmethod-first-arg=cls 327 | 328 | # List of valid names for the first argument in a metaclass class method. 329 | valid-metaclass-classmethod-first-arg=mcs 330 | 331 | # List of member names, which should be excluded from the protected access 332 | # warning. 333 | exclude-protected=_asdict,_fields,_replace,_source,_make 334 | 335 | 336 | [DESIGN] 337 | 338 | # Maximum number of arguments for function / method 339 | max-args=5 340 | 341 | # Argument names that match this expression will be ignored. Default to name 342 | # with leading underscore 343 | ignored-argument-names=_.* 344 | 345 | # Maximum number of locals for function / method body 346 | max-locals=15 347 | 348 | # Maximum number of return / yield for function / method body 349 | max-returns=6 350 | 351 | # Maximum number of branch for function / method body 352 | max-branches=12 353 | 354 | # Maximum number of statements in function / method body 355 | max-statements=50 356 | 357 | # Maximum number of parents for a class (see R0901). 358 | max-parents=7 359 | 360 | # Maximum number of attributes for a class (see R0902). 361 | max-attributes=7 362 | 363 | # Minimum number of public methods for a class (see R0903). 364 | min-public-methods=2 365 | 366 | # Maximum number of public methods for a class (see R0904). 367 | max-public-methods=20 368 | 369 | # Maximum number of boolean expressions in a if statement 370 | max-bool-expr=5 371 | 372 | 373 | [IMPORTS] 374 | 375 | # Deprecated modules which should not be used, separated by a comma 376 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 377 | 378 | # Create a graph of every (i.e. internal and external) dependencies in the 379 | # given file (report RP0402 must not be disabled) 380 | import-graph= 381 | 382 | # Create a graph of external dependencies in the given file (report RP0402 must 383 | # not be disabled) 384 | ext-import-graph= 385 | 386 | # Create a graph of internal dependencies in the given file (report RP0402 must 387 | # not be disabled) 388 | int-import-graph= 389 | 390 | # Force import order to recognize a module as part of the standard 391 | # compatibility libraries. 392 | known-standard-library= 393 | 394 | # Force import order to recognize a module as part of a third party library. 395 | known-third-party=enchant 396 | 397 | # Analyse import fallback blocks. This can be used to support both Python 2 and 398 | # 3 compatible code, which means that the block might have code that exists 399 | # only in one or another interpreter, leading to false positives when analysed. 400 | analyse-fallback-blocks=no 401 | 402 | 403 | [EXCEPTIONS] 404 | 405 | # Exceptions that will emit a warning when being caught. Defaults to 406 | # "Exception" 407 | overgeneral-exceptions=Exception 408 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | click ~= 7.0 2 | logutils ~= 0.3 3 | pymysql ~= 0.9 4 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | black ~= 19.10b0 2 | bumpversion ~= 0.5.0 3 | codecov ~= 2.1 4 | coverage ~= 5.1 5 | mock ~= 4.0 6 | pylint ~= 2.5 7 | pytest ~= 5.4 8 | pytest-cov ~= 2.9 9 | Sphinx ~= 3.0 10 | tox ~= 3.15 11 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 3.0.3 3 | commit = True 4 | tag = False 5 | 6 | [bumpversion:file:setup.py] 7 | search = version="{current_version}" 8 | replace = version="{new_version}" 9 | 10 | [bumpversion:file:twindb_table_compare/__init__.py] 11 | search = __version__ = "{current_version}" 12 | replace = __version__ = "{new_version}" 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [flake8] 18 | exclude = docs 19 | 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | with open("README.rst") as readme_file: 4 | readme = readme_file.read() 5 | 6 | with open("HISTORY.rst") as history_file: 7 | history = history_file.read() 8 | 9 | 10 | with open("requirements.txt") as f: 11 | requirements = f.read().strip().split("\n") 12 | 13 | with open("requirements_dev.txt") as f: 14 | test_requirements = f.read().strip().split("\n") 15 | 16 | setup( 17 | name="twindb_table_compare", 18 | version="3.0.3", 19 | description=( 20 | "TwinDB Table Compare reads percona.checksums from the master and slave " 21 | "and shows what records are difference if there are any inconsistencies." 22 | ), 23 | long_description=readme + "\n\n" + history, 24 | long_description_content_type="text/x-rst", 25 | author="Aleksandr Kuzminsky", 26 | author_email="aleks@twindb.com", 27 | url="https://github.com/twindb/twindb-table-compare", 28 | packages=["twindb_table_compare"], 29 | package_dir={"twindb_table_compare": "twindb_table_compare"}, 30 | entry_points={ 31 | "console_scripts": ["twindb-table-compare=twindb_table_compare.cli:main"] 32 | }, 33 | include_package_data=True, 34 | install_requires=requirements, 35 | license="Apache Software License 2.0", 36 | zip_safe=False, 37 | keywords="twindb_table_compare", 38 | classifiers=[ 39 | "Development Status :: 5 - Production/Stable", 40 | "Environment :: Console", 41 | "Intended Audience :: Developers", 42 | "License :: OSI Approved :: Apache Software License", 43 | "Natural Language :: English", 44 | "Programming Language :: Python :: 3", 45 | "Programming Language :: Python :: 3.6", 46 | "Programming Language :: Python :: 3.7", 47 | "Programming Language :: Python :: 3.8", 48 | "Topic :: Database", 49 | "Topic :: Database :: Database Engines/Servers", 50 | ], 51 | test_suite="tests", 52 | tests_require=test_requirements, 53 | ) 54 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/functional/conftest.py: -------------------------------------------------------------------------------- 1 | from twindb_table_compare import LOG, setup_logging 2 | 3 | setup_logging(LOG, debug=True) 4 | -------------------------------------------------------------------------------- /tests/functional/test_twindb_table_compare_functional.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_twindb_table_compare 6 | ---------------------------------- 7 | 8 | Tests for `twindb_table_compare` module. 9 | """ 10 | import binascii 11 | import MySQLdb 12 | import pytest 13 | 14 | from twindb_table_compare.compare import is_printable, \ 15 | get_chunk_index, get_index_fields, get_boundary, get_master, \ 16 | build_chunk_query 17 | 18 | 19 | @pytest.fixture 20 | def mysql_cred(): 21 | return { 22 | 'user': 'dba', 23 | 'password': 'qwerty' 24 | } 25 | 26 | 27 | @pytest.fixture 28 | def master_connection(mysql_cred): 29 | return MySQLdb.connect(host='192.168.35.250', 30 | user=mysql_cred['user'], 31 | passwd=mysql_cred['password']) 32 | 33 | 34 | @pytest.fixture 35 | def slave_connection(mysql_cred): 36 | return MySQLdb.connect(host='192.168.35.251', 37 | user=mysql_cred['user'], 38 | passwd=mysql_cred['password']) 39 | 40 | 41 | @pytest.mark.parametrize('input_str,result', [ 42 | ( 43 | 'foo', 44 | True 45 | ), 46 | ( 47 | binascii.a2b_hex('AA'), 48 | False 49 | ) 50 | ]) 51 | def test_is_printable(input_str, result): 52 | assert is_printable(input_str) == result 53 | 54 | 55 | def test_get_chunk_index(master_connection): 56 | assert get_chunk_index(master_connection, 'test', 't1', 1) == 'PRIMARY' 57 | assert not get_chunk_index(master_connection, 'mysql', 'user', 1) 58 | 59 | 60 | def test_get_index_fields(master_connection): 61 | assert get_index_fields(master_connection, 62 | 'test', 't1', 'PRIMARY') == ['id'] 63 | 64 | 65 | def test_get_boundary(master_connection): 66 | boundary = get_boundary(master_connection, 'test', 't1', 1) 67 | assert int(boundary[0]) == 1 68 | assert int(boundary[1]) <= 393197 69 | 70 | 71 | def test_get_master(slave_connection): 72 | assert get_master(slave_connection) == '192.168.35.250' 73 | 74 | 75 | # def test_build_chunk_query(slave_connection): 76 | # assert build_chunk_query('test', 't1', 1, slave_connection) == "" 77 | -------------------------------------------------------------------------------- /tests/unit/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_table_compare import setup_logging, LOG 4 | 5 | setup_logging(LOG, debug=True) 6 | 7 | 8 | @pytest.fixture 9 | def out_master(): 10 | return """*************************** 1. row *************************** 11 | Host: localhost 12 | User: root 13 | Password: 14 | Select_priv: Y 15 | Insert_priv: Y 16 | Update_priv: Y 17 | Delete_priv: Y 18 | Create_priv: Y 19 | Drop_priv: Y 20 | Reload_priv: Y 21 | Shutdown_priv: Y 22 | Process_priv: Y 23 | File_priv: Y 24 | Grant_priv: Y 25 | References_priv: Y 26 | Index_priv: Y 27 | Alter_priv: Y 28 | Show_db_priv: Y 29 | Super_priv: Y 30 | Create_tmp_table_priv: Y 31 | Lock_tables_priv: Y 32 | Execute_priv: Y 33 | Repl_slave_priv: Y 34 | Repl_client_priv: Y 35 | Create_view_priv: Y 36 | Show_view_priv: Y 37 | Create_routine_priv: Y 38 | Alter_routine_priv: Y 39 | Create_user_priv: Y 40 | Event_priv: Y 41 | Trigger_priv: Y 42 | Create_tablespace_priv: Y 43 | ssl_type: 44 | ssl_cipher: 45 | x509_issuer: 46 | x509_subject: 47 | max_questions: 0 48 | max_updates: 0 49 | max_connections: 0 50 | max_user_connections: 0 51 | plugin: mysql_native_password 52 | authentication_string: 53 | password_expired: N 54 | *************************** 2. row *************************** 55 | Host: master.box 56 | User: root 57 | Password: 58 | Select_priv: Y 59 | Insert_priv: Y 60 | Update_priv: Y 61 | Delete_priv: Y 62 | Create_priv: Y 63 | Drop_priv: Y 64 | Reload_priv: Y 65 | Shutdown_priv: Y 66 | Process_priv: Y 67 | File_priv: Y 68 | Grant_priv: Y 69 | References_priv: Y 70 | Index_priv: Y 71 | Alter_priv: Y 72 | Show_db_priv: Y 73 | Super_priv: Y 74 | Create_tmp_table_priv: Y 75 | Lock_tables_priv: Y 76 | Execute_priv: Y 77 | Repl_slave_priv: Y 78 | Repl_client_priv: Y 79 | Create_view_priv: Y 80 | Show_view_priv: Y 81 | Create_routine_priv: Y 82 | Alter_routine_priv: Y 83 | Create_user_priv: Y 84 | Event_priv: Y 85 | Trigger_priv: Y 86 | Create_tablespace_priv: Y 87 | ssl_type: 88 | ssl_cipher: 89 | x509_issuer: 90 | x509_subject: 91 | max_questions: 0 92 | max_updates: 0 93 | max_connections: 0 94 | max_user_connections: 0 95 | plugin: mysql_native_password 96 | authentication_string: 97 | password_expired: N 98 | *************************** 3. row *************************** 99 | Host: 127.0.0.1 100 | User: root 101 | Password: 102 | Select_priv: Y 103 | Insert_priv: Y 104 | Update_priv: Y 105 | Delete_priv: Y 106 | Create_priv: Y 107 | Drop_priv: Y 108 | Reload_priv: Y 109 | Shutdown_priv: Y 110 | Process_priv: Y 111 | File_priv: Y 112 | Grant_priv: Y 113 | References_priv: Y 114 | Index_priv: Y 115 | Alter_priv: Y 116 | Show_db_priv: Y 117 | Super_priv: Y 118 | Create_tmp_table_priv: Y 119 | Lock_tables_priv: Y 120 | Execute_priv: Y 121 | Repl_slave_priv: Y 122 | Repl_client_priv: Y 123 | Create_view_priv: Y 124 | Show_view_priv: Y 125 | Create_routine_priv: Y 126 | Alter_routine_priv: Y 127 | Create_user_priv: Y 128 | Event_priv: Y 129 | Trigger_priv: Y 130 | Create_tablespace_priv: Y 131 | ssl_type: 132 | ssl_cipher: 133 | x509_issuer: 134 | x509_subject: 135 | max_questions: 0 136 | max_updates: 0 137 | max_connections: 0 138 | max_user_connections: 0 139 | plugin: mysql_native_password 140 | authentication_string: 141 | password_expired: N 142 | *************************** 4. row *************************** 143 | Host: ::1 144 | User: root 145 | Password: 146 | Select_priv: Y 147 | Insert_priv: Y 148 | Update_priv: Y 149 | Delete_priv: Y 150 | Create_priv: Y 151 | Drop_priv: Y 152 | Reload_priv: Y 153 | Shutdown_priv: Y 154 | Process_priv: Y 155 | File_priv: Y 156 | Grant_priv: Y 157 | References_priv: Y 158 | Index_priv: Y 159 | Alter_priv: Y 160 | Show_db_priv: Y 161 | Super_priv: Y 162 | Create_tmp_table_priv: Y 163 | Lock_tables_priv: Y 164 | Execute_priv: Y 165 | Repl_slave_priv: Y 166 | Repl_client_priv: Y 167 | Create_view_priv: Y 168 | Show_view_priv: Y 169 | Create_routine_priv: Y 170 | Alter_routine_priv: Y 171 | Create_user_priv: Y 172 | Event_priv: Y 173 | Trigger_priv: Y 174 | Create_tablespace_priv: Y 175 | ssl_type: 176 | ssl_cipher: 177 | x509_issuer: 178 | x509_subject: 179 | max_questions: 0 180 | max_updates: 0 181 | max_connections: 0 182 | max_user_connections: 0 183 | plugin: mysql_native_password 184 | authentication_string: 185 | password_expired: N 186 | *************************** 5. row *************************** 187 | Host: localhost 188 | User: 189 | Password: 190 | Select_priv: N 191 | Insert_priv: N 192 | Update_priv: N 193 | Delete_priv: N 194 | Create_priv: N 195 | Drop_priv: N 196 | Reload_priv: N 197 | Shutdown_priv: N 198 | Process_priv: N 199 | File_priv: N 200 | Grant_priv: N 201 | References_priv: N 202 | Index_priv: N 203 | Alter_priv: N 204 | Show_db_priv: N 205 | Super_priv: N 206 | Create_tmp_table_priv: N 207 | Lock_tables_priv: N 208 | Execute_priv: N 209 | Repl_slave_priv: N 210 | Repl_client_priv: N 211 | Create_view_priv: N 212 | Show_view_priv: N 213 | Create_routine_priv: N 214 | Alter_routine_priv: N 215 | Create_user_priv: N 216 | Event_priv: N 217 | Trigger_priv: N 218 | Create_tablespace_priv: N 219 | ssl_type: 220 | ssl_cipher: 221 | x509_issuer: 222 | x509_subject: 223 | max_questions: 0 224 | max_updates: 0 225 | max_connections: 0 226 | max_user_connections: 0 227 | plugin: mysql_native_password 228 | authentication_string: NULL 229 | password_expired: N 230 | *************************** 6. row *************************** 231 | Host: master.box 232 | User: 233 | Password: 234 | Select_priv: N 235 | Insert_priv: N 236 | Update_priv: N 237 | Delete_priv: N 238 | Create_priv: N 239 | Drop_priv: N 240 | Reload_priv: N 241 | Shutdown_priv: N 242 | Process_priv: N 243 | File_priv: N 244 | Grant_priv: N 245 | References_priv: N 246 | Index_priv: N 247 | Alter_priv: N 248 | Show_db_priv: N 249 | Super_priv: N 250 | Create_tmp_table_priv: N 251 | Lock_tables_priv: N 252 | Execute_priv: N 253 | Repl_slave_priv: N 254 | Repl_client_priv: N 255 | Create_view_priv: N 256 | Show_view_priv: N 257 | Create_routine_priv: N 258 | Alter_routine_priv: N 259 | Create_user_priv: N 260 | Event_priv: N 261 | Trigger_priv: N 262 | Create_tablespace_priv: N 263 | ssl_type: 264 | ssl_cipher: 265 | x509_issuer: 266 | x509_subject: 267 | max_questions: 0 268 | max_updates: 0 269 | max_connections: 0 270 | max_user_connections: 0 271 | plugin: mysql_native_password 272 | authentication_string: NULL 273 | password_expired: N 274 | *************************** 7. row *************************** 275 | Host: % 276 | User: dba 277 | Password: *AA1420F182E88B9E5F874F6FBE7459291E8F4601 278 | Select_priv: Y 279 | Insert_priv: Y 280 | Update_priv: Y 281 | Delete_priv: Y 282 | Create_priv: Y 283 | Drop_priv: Y 284 | Reload_priv: Y 285 | Shutdown_priv: Y 286 | Process_priv: Y 287 | File_priv: Y 288 | Grant_priv: Y 289 | References_priv: Y 290 | Index_priv: Y 291 | Alter_priv: Y 292 | Show_db_priv: Y 293 | Super_priv: Y 294 | Create_tmp_table_priv: Y 295 | Lock_tables_priv: Y 296 | Execute_priv: Y 297 | Repl_slave_priv: Y 298 | Repl_client_priv: Y 299 | Create_view_priv: Y 300 | Show_view_priv: Y 301 | Create_routine_priv: Y 302 | Alter_routine_priv: Y 303 | Create_user_priv: Y 304 | Event_priv: Y 305 | Trigger_priv: Y 306 | Create_tablespace_priv: Y 307 | ssl_type: 308 | ssl_cipher: 309 | x509_issuer: 310 | x509_subject: 311 | max_questions: 0 312 | max_updates: 0 313 | max_connections: 0 314 | max_user_connections: 0 315 | plugin: mysql_native_password 316 | authentication_string: 317 | password_expired: N 318 | *************************** 8. row *************************** 319 | Host: localhost 320 | User: dba 321 | Password: *AA1420F182E88B9E5F874F6FBE7459291E8F4601 322 | Select_priv: Y 323 | Insert_priv: Y 324 | Update_priv: Y 325 | Delete_priv: Y 326 | Create_priv: Y 327 | Drop_priv: Y 328 | Reload_priv: Y 329 | Shutdown_priv: Y 330 | Process_priv: Y 331 | File_priv: Y 332 | Grant_priv: Y 333 | References_priv: Y 334 | Index_priv: Y 335 | Alter_priv: Y 336 | Show_db_priv: Y 337 | Super_priv: Y 338 | Create_tmp_table_priv: Y 339 | Lock_tables_priv: Y 340 | Execute_priv: Y 341 | Repl_slave_priv: Y 342 | Repl_client_priv: Y 343 | Create_view_priv: Y 344 | Show_view_priv: Y 345 | Create_routine_priv: Y 346 | Alter_routine_priv: Y 347 | Create_user_priv: Y 348 | Event_priv: Y 349 | Trigger_priv: Y 350 | Create_tablespace_priv: Y 351 | ssl_type: 352 | ssl_cipher: 353 | x509_issuer: 354 | x509_subject: 355 | max_questions: 0 356 | max_updates: 0 357 | max_connections: 0 358 | max_user_connections: 0 359 | plugin: mysql_native_password 360 | authentication_string: 361 | password_expired: N 362 | *************************** 9. row *************************** 363 | Host: % 364 | User: repl 365 | Password: *809534247D21AC735802078139D8A854F45C31F3 366 | Select_priv: N 367 | Insert_priv: N 368 | Update_priv: N 369 | Delete_priv: N 370 | Create_priv: N 371 | Drop_priv: N 372 | Reload_priv: N 373 | Shutdown_priv: N 374 | Process_priv: N 375 | File_priv: N 376 | Grant_priv: N 377 | References_priv: N 378 | Index_priv: N 379 | Alter_priv: N 380 | Show_db_priv: N 381 | Super_priv: N 382 | Create_tmp_table_priv: N 383 | Lock_tables_priv: N 384 | Execute_priv: N 385 | Repl_slave_priv: Y 386 | Repl_client_priv: N 387 | Create_view_priv: N 388 | Show_view_priv: N 389 | Create_routine_priv: N 390 | Alter_routine_priv: N 391 | Create_user_priv: N 392 | Event_priv: N 393 | Trigger_priv: N 394 | Create_tablespace_priv: N 395 | ssl_type: 396 | ssl_cipher: 397 | x509_issuer: 398 | x509_subject: 399 | max_questions: 0 400 | max_updates: 0 401 | max_connections: 0 402 | max_user_connections: 0 403 | plugin: mysql_native_password 404 | authentication_string: 405 | password_expired: N 406 | """, "" 407 | 408 | 409 | @pytest.fixture 410 | def out_slave(): 411 | return """*************************** 1. row *************************** 412 | Host: localhost 413 | User: root 414 | Password: 415 | Select_priv: Y 416 | Insert_priv: Y 417 | Update_priv: Y 418 | Delete_priv: Y 419 | Create_priv: Y 420 | Drop_priv: Y 421 | Reload_priv: Y 422 | Shutdown_priv: Y 423 | Process_priv: Y 424 | File_priv: Y 425 | Grant_priv: Y 426 | References_priv: Y 427 | Index_priv: Y 428 | Alter_priv: Y 429 | Show_db_priv: Y 430 | Super_priv: Y 431 | Create_tmp_table_priv: Y 432 | Lock_tables_priv: Y 433 | Execute_priv: Y 434 | Repl_slave_priv: Y 435 | Repl_client_priv: Y 436 | Create_view_priv: Y 437 | Show_view_priv: Y 438 | Create_routine_priv: Y 439 | Alter_routine_priv: Y 440 | Create_user_priv: Y 441 | Event_priv: Y 442 | Trigger_priv: Y 443 | Create_tablespace_priv: Y 444 | ssl_type: 445 | ssl_cipher: 446 | x509_issuer: 447 | x509_subject: 448 | max_questions: 0 449 | max_updates: 0 450 | max_connections: 0 451 | max_user_connections: 0 452 | plugin: mysql_native_password 453 | authentication_string: 454 | password_expired: N 455 | *************************** 2. row *************************** 456 | Host: slave.box 457 | User: root 458 | Password: 459 | Select_priv: Y 460 | Insert_priv: Y 461 | Update_priv: Y 462 | Delete_priv: Y 463 | Create_priv: Y 464 | Drop_priv: Y 465 | Reload_priv: Y 466 | Shutdown_priv: Y 467 | Process_priv: Y 468 | File_priv: Y 469 | Grant_priv: Y 470 | References_priv: Y 471 | Index_priv: Y 472 | Alter_priv: Y 473 | Show_db_priv: Y 474 | Super_priv: Y 475 | Create_tmp_table_priv: Y 476 | Lock_tables_priv: Y 477 | Execute_priv: Y 478 | Repl_slave_priv: Y 479 | Repl_client_priv: Y 480 | Create_view_priv: Y 481 | Show_view_priv: Y 482 | Create_routine_priv: Y 483 | Alter_routine_priv: Y 484 | Create_user_priv: Y 485 | Event_priv: Y 486 | Trigger_priv: Y 487 | Create_tablespace_priv: Y 488 | ssl_type: 489 | ssl_cipher: 490 | x509_issuer: 491 | x509_subject: 492 | max_questions: 0 493 | max_updates: 0 494 | max_connections: 0 495 | max_user_connections: 0 496 | plugin: mysql_native_password 497 | authentication_string: 498 | password_expired: N 499 | *************************** 3. row *************************** 500 | Host: 127.0.0.1 501 | User: root 502 | Password: 503 | Select_priv: Y 504 | Insert_priv: Y 505 | Update_priv: Y 506 | Delete_priv: Y 507 | Create_priv: Y 508 | Drop_priv: Y 509 | Reload_priv: Y 510 | Shutdown_priv: Y 511 | Process_priv: Y 512 | File_priv: Y 513 | Grant_priv: Y 514 | References_priv: Y 515 | Index_priv: Y 516 | Alter_priv: Y 517 | Show_db_priv: Y 518 | Super_priv: Y 519 | Create_tmp_table_priv: Y 520 | Lock_tables_priv: Y 521 | Execute_priv: Y 522 | Repl_slave_priv: Y 523 | Repl_client_priv: Y 524 | Create_view_priv: Y 525 | Show_view_priv: Y 526 | Create_routine_priv: Y 527 | Alter_routine_priv: Y 528 | Create_user_priv: Y 529 | Event_priv: Y 530 | Trigger_priv: Y 531 | Create_tablespace_priv: Y 532 | ssl_type: 533 | ssl_cipher: 534 | x509_issuer: 535 | x509_subject: 536 | max_questions: 0 537 | max_updates: 0 538 | max_connections: 0 539 | max_user_connections: 0 540 | plugin: mysql_native_password 541 | authentication_string: 542 | password_expired: N 543 | *************************** 4. row *************************** 544 | Host: ::1 545 | User: root 546 | Password: 547 | Select_priv: Y 548 | Insert_priv: Y 549 | Update_priv: Y 550 | Delete_priv: Y 551 | Create_priv: Y 552 | Drop_priv: Y 553 | Reload_priv: Y 554 | Shutdown_priv: Y 555 | Process_priv: Y 556 | File_priv: Y 557 | Grant_priv: Y 558 | References_priv: Y 559 | Index_priv: Y 560 | Alter_priv: Y 561 | Show_db_priv: Y 562 | Super_priv: Y 563 | Create_tmp_table_priv: Y 564 | Lock_tables_priv: Y 565 | Execute_priv: Y 566 | Repl_slave_priv: Y 567 | Repl_client_priv: Y 568 | Create_view_priv: Y 569 | Show_view_priv: Y 570 | Create_routine_priv: Y 571 | Alter_routine_priv: Y 572 | Create_user_priv: Y 573 | Event_priv: Y 574 | Trigger_priv: Y 575 | Create_tablespace_priv: Y 576 | ssl_type: 577 | ssl_cipher: 578 | x509_issuer: 579 | x509_subject: 580 | max_questions: 0 581 | max_updates: 0 582 | max_connections: 0 583 | max_user_connections: 0 584 | plugin: mysql_native_password 585 | authentication_string: 586 | password_expired: N 587 | *************************** 5. row *************************** 588 | Host: localhost 589 | User: 590 | Password: 591 | Select_priv: N 592 | Insert_priv: N 593 | Update_priv: N 594 | Delete_priv: N 595 | Create_priv: N 596 | Drop_priv: N 597 | Reload_priv: N 598 | Shutdown_priv: N 599 | Process_priv: N 600 | File_priv: N 601 | Grant_priv: N 602 | References_priv: N 603 | Index_priv: N 604 | Alter_priv: N 605 | Show_db_priv: N 606 | Super_priv: N 607 | Create_tmp_table_priv: N 608 | Lock_tables_priv: N 609 | Execute_priv: N 610 | Repl_slave_priv: N 611 | Repl_client_priv: N 612 | Create_view_priv: N 613 | Show_view_priv: N 614 | Create_routine_priv: N 615 | Alter_routine_priv: N 616 | Create_user_priv: N 617 | Event_priv: N 618 | Trigger_priv: N 619 | Create_tablespace_priv: N 620 | ssl_type: 621 | ssl_cipher: 622 | x509_issuer: 623 | x509_subject: 624 | max_questions: 0 625 | max_updates: 0 626 | max_connections: 0 627 | max_user_connections: 0 628 | plugin: mysql_native_password 629 | authentication_string: NULL 630 | password_expired: N 631 | *************************** 6. row *************************** 632 | Host: slave.box 633 | User: 634 | Password: 635 | Select_priv: N 636 | Insert_priv: N 637 | Update_priv: N 638 | Delete_priv: N 639 | Create_priv: N 640 | Drop_priv: N 641 | Reload_priv: N 642 | Shutdown_priv: N 643 | Process_priv: N 644 | File_priv: N 645 | Grant_priv: N 646 | References_priv: N 647 | Index_priv: N 648 | Alter_priv: N 649 | Show_db_priv: N 650 | Super_priv: N 651 | Create_tmp_table_priv: N 652 | Lock_tables_priv: N 653 | Execute_priv: N 654 | Repl_slave_priv: N 655 | Repl_client_priv: N 656 | Create_view_priv: N 657 | Show_view_priv: N 658 | Create_routine_priv: N 659 | Alter_routine_priv: N 660 | Create_user_priv: N 661 | Event_priv: N 662 | Trigger_priv: N 663 | Create_tablespace_priv: N 664 | ssl_type: 665 | ssl_cipher: 666 | x509_issuer: 667 | x509_subject: 668 | max_questions: 0 669 | max_updates: 0 670 | max_connections: 0 671 | max_user_connections: 0 672 | plugin: mysql_native_password 673 | authentication_string: NULL 674 | password_expired: N 675 | *************************** 7. row *************************** 676 | Host: % 677 | User: dba 678 | Password: *AA1420F182E88B9E5F874F6FBE7459291E8F4601 679 | Select_priv: Y 680 | Insert_priv: Y 681 | Update_priv: Y 682 | Delete_priv: Y 683 | Create_priv: Y 684 | Drop_priv: Y 685 | Reload_priv: Y 686 | Shutdown_priv: Y 687 | Process_priv: Y 688 | File_priv: Y 689 | Grant_priv: Y 690 | References_priv: Y 691 | Index_priv: Y 692 | Alter_priv: Y 693 | Show_db_priv: Y 694 | Super_priv: Y 695 | Create_tmp_table_priv: Y 696 | Lock_tables_priv: Y 697 | Execute_priv: Y 698 | Repl_slave_priv: Y 699 | Repl_client_priv: Y 700 | Create_view_priv: Y 701 | Show_view_priv: Y 702 | Create_routine_priv: Y 703 | Alter_routine_priv: Y 704 | Create_user_priv: Y 705 | Event_priv: Y 706 | Trigger_priv: Y 707 | Create_tablespace_priv: Y 708 | ssl_type: 709 | ssl_cipher: 710 | x509_issuer: 711 | x509_subject: 712 | max_questions: 0 713 | max_updates: 0 714 | max_connections: 0 715 | max_user_connections: 0 716 | plugin: mysql_native_password 717 | authentication_string: 718 | password_expired: N 719 | *************************** 8. row *************************** 720 | Host: localhost 721 | User: dba 722 | Password: *AA1420F182E88B9E5F874F6FBE7459291E8F4601 723 | Select_priv: Y 724 | Insert_priv: Y 725 | Update_priv: Y 726 | Delete_priv: Y 727 | Create_priv: Y 728 | Drop_priv: Y 729 | Reload_priv: Y 730 | Shutdown_priv: Y 731 | Process_priv: Y 732 | File_priv: Y 733 | Grant_priv: Y 734 | References_priv: Y 735 | Index_priv: Y 736 | Alter_priv: Y 737 | Show_db_priv: Y 738 | Super_priv: Y 739 | Create_tmp_table_priv: Y 740 | Lock_tables_priv: Y 741 | Execute_priv: Y 742 | Repl_slave_priv: Y 743 | Repl_client_priv: Y 744 | Create_view_priv: Y 745 | Show_view_priv: Y 746 | Create_routine_priv: Y 747 | Alter_routine_priv: Y 748 | Create_user_priv: Y 749 | Event_priv: Y 750 | Trigger_priv: Y 751 | Create_tablespace_priv: Y 752 | ssl_type: 753 | ssl_cipher: 754 | x509_issuer: 755 | x509_subject: 756 | max_questions: 0 757 | max_updates: 0 758 | max_connections: 0 759 | max_user_connections: 0 760 | plugin: mysql_native_password 761 | authentication_string: 762 | password_expired: N 763 | *************************** 9. row *************************** 764 | Host: % 765 | User: repl 766 | Password: *809534247D21AC735802078139D8A854F45C31F3 767 | Select_priv: N 768 | Insert_priv: N 769 | Update_priv: N 770 | Delete_priv: N 771 | Create_priv: N 772 | Drop_priv: N 773 | Reload_priv: N 774 | Shutdown_priv: N 775 | Process_priv: N 776 | File_priv: N 777 | Grant_priv: N 778 | References_priv: N 779 | Index_priv: N 780 | Alter_priv: N 781 | Show_db_priv: N 782 | Super_priv: N 783 | Create_tmp_table_priv: N 784 | Lock_tables_priv: N 785 | Execute_priv: N 786 | Repl_slave_priv: Y 787 | Repl_client_priv: N 788 | Create_view_priv: N 789 | Show_view_priv: N 790 | Create_routine_priv: N 791 | Alter_routine_priv: N 792 | Create_user_priv: N 793 | Event_priv: N 794 | Trigger_priv: N 795 | Create_tablespace_priv: N 796 | ssl_type: 797 | ssl_cipher: 798 | x509_issuer: 799 | x509_subject: 800 | max_questions: 0 801 | max_updates: 0 802 | max_connections: 0 803 | max_user_connections: 0 804 | plugin: mysql_native_password 805 | authentication_string: 806 | password_expired: N 807 | """, "" 808 | -------------------------------------------------------------------------------- /tests/unit/test_twindb_table_compare.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_twindb_table_compare 6 | ---------------------------------- 7 | 8 | Tests for `twindb_table_compare` module. 9 | """ 10 | import binascii 11 | 12 | import mock 13 | import pytest 14 | 15 | from click.testing import CliRunner 16 | 17 | from twindb_table_compare import cli, __version__ 18 | from twindb_table_compare.compare import is_printable, diff, print_vertical, \ 19 | get_fields, build_chunk_query, get_where 20 | 21 | 22 | def test_command_line_interface(): 23 | runner = CliRunner() 24 | help_result = runner.invoke(cli.main, ['--help']) 25 | assert help_result.exit_code == 0 26 | 27 | 28 | @mock.patch('twindb_table_compare.cli.get_inconsistencies') 29 | def test_version(mock_get_inconsistencies): 30 | runner = CliRunner() 31 | mock_get_inconsistencies.side_effect = Exception 32 | help_result = runner.invoke(cli.main, ['--version']) 33 | assert help_result.output.strip('\n') == __version__ 34 | assert help_result.exit_code == 0 35 | 36 | 37 | @pytest.mark.parametrize('input_str,result', [ 38 | ( 39 | 'foo', 40 | True 41 | ), 42 | ( 43 | binascii.a2b_hex('AA'), 44 | False 45 | ) 46 | ]) 47 | def test_is_printable(input_str, result): 48 | assert is_printable(input_str) == result 49 | 50 | 51 | @pytest.mark.parametrize('master_lines, slave_lines, difference', [ 52 | ( 53 | [ 54 | 'localhost\troot\t1\t2016-12-02 04:46:12\n', 55 | 'master.box\troot\t1\t2016-12-02 04:46:12\n' 56 | ], 57 | [ 58 | 'localhost\troot\t1\t2016-12-02 05:43:47\n', 59 | 'slave.box\troot\t1\t2016-12-02 05:43:47\n' 60 | ], 61 | """@@ -1,2 +1,2 @@ 62 | -localhost\troot\t1\t2016-12-02 04:46:12 63 | -master.box\troot\t1\t2016-12-02 04:46:12 64 | +localhost\troot\t1\t2016-12-02 05:43:47 65 | +slave.box\troot\t1\t2016-12-02 05:43:47 66 | """ 67 | ), 68 | ( 69 | [ 70 | '3882\t2016-04-20 14:57:31\n', 71 | '3882\t2016-04-20 14:57:31\n', 72 | '3937\t2016-05-13 14:32:53\n' 73 | ], 74 | [ 75 | '3937\t2016-05-13 14:32:53\n', 76 | '3882\t2016-04-20 14:57:31\n', 77 | '3882\t2016-04-20 14:57:31\n' 78 | ], 79 | """@@ -1,3 +1,3 @@ 80 | +3937\t2016-05-13 14:32:53 81 | 3882\t2016-04-20 14:57:31 82 | 3882\t2016-04-20 14:57:31 83 | -3937\t2016-05-13 14:32:53 84 | """ 85 | ), 86 | ( 87 | [ 88 | "*************************** 1. row ***************************\n", 89 | " Host: localhost\n", 90 | " User: root\n", 91 | " Password: \n", 92 | " Select_priv: Y\n", 93 | " Insert_priv: Y\n", 94 | " Update_priv: Y\n", 95 | " Delete_priv: Y\n", 96 | " Create_priv: Y\n", 97 | " Drop_priv: Y\n", 98 | " Reload_priv: Y\n", 99 | " Shutdown_priv: Y\n", 100 | " Process_priv: Y\n", 101 | " File_priv: Y\n", 102 | " Grant_priv: Y\n", 103 | " References_priv: Y\n", 104 | " Index_priv: Y\n", 105 | " Alter_priv: Y\n", 106 | " Show_db_priv: Y\n", 107 | " Super_priv: Y\n", 108 | " Create_tmp_table_priv: Y\n", 109 | " Lock_tables_priv: Y\n", 110 | " Execute_priv: Y\n", 111 | " Repl_slave_priv: Y\n", 112 | " Repl_client_priv: Y\n", 113 | " Create_view_priv: Y\n", 114 | " Show_view_priv: Y\n", 115 | " Create_routine_priv: Y\n", 116 | " Alter_routine_priv: Y\n", 117 | " Create_user_priv: Y\n", 118 | " Event_priv: Y\n", 119 | " Trigger_priv: Y\n", 120 | "Create_tablespace_priv: Y\n", 121 | " ssl_type: \n", 122 | " HEX(ssl_cipher): \n", 123 | " HEX(x509_issuer): \n", 124 | " HEX(x509_subject): \n", 125 | " max_questions: 0\n", 126 | " max_updates: 0\n", 127 | " max_connections: 0\n", 128 | " max_user_connections: 0\n", 129 | " plugin: mysql_native_password\n", 130 | " authentication_string: \n", 131 | " password_expired: N\n", 132 | "*************************** 2. row ***************************\n", 133 | " Host: master.box\n", 134 | " User: root\n", 135 | " Password: \n", 136 | " Select_priv: Y\n", 137 | " Insert_priv: Y\n", 138 | " Update_priv: Y\n", 139 | " Delete_priv: Y\n", 140 | " Create_priv: Y\n", 141 | " Drop_priv: Y\n", 142 | " Reload_priv: Y\n", 143 | " Shutdown_priv: Y\n", 144 | " Process_priv: Y\n", 145 | " File_priv: Y\n", 146 | " Grant_priv: Y\n", 147 | " References_priv: Y\n", 148 | " Index_priv: Y\n", 149 | " Alter_priv: Y\n", 150 | " Show_db_priv: Y\n", 151 | " Super_priv: Y\n", 152 | " Create_tmp_table_priv: Y\n", 153 | " Lock_tables_priv: Y\n", 154 | " Execute_priv: Y\n", 155 | " Repl_slave_priv: Y\n", 156 | " Repl_client_priv: Y\n", 157 | " Create_view_priv: Y\n", 158 | " Show_view_priv: Y\n", 159 | " Create_routine_priv: Y\n", 160 | " Alter_routine_priv: Y\n", 161 | " Create_user_priv: Y\n", 162 | " Event_priv: Y\n", 163 | " Trigger_priv: Y\n", 164 | "Create_tablespace_priv: Y\n", 165 | " ssl_type: \n", 166 | " HEX(ssl_cipher): \n", 167 | " HEX(x509_issuer): \n", 168 | " HEX(x509_subject): \n", 169 | " max_questions: 0\n", 170 | " max_updates: 0\n", 171 | " max_connections: 0\n", 172 | " max_user_connections: 0\n", 173 | " plugin: mysql_native_password\n", 174 | " authentication_string: \n", 175 | " password_expired: N\n", 176 | "*************************** 3. row ***************************\n", 177 | " Host: 127.0.0.1\n", 178 | " User: root\n", 179 | " Password: \n", 180 | " Select_priv: Y\n", 181 | " Insert_priv: Y\n", 182 | " Update_priv: Y\n", 183 | " Delete_priv: Y\n", 184 | " Create_priv: Y\n", 185 | " Drop_priv: Y\n", 186 | " Reload_priv: Y\n", 187 | " Shutdown_priv: Y\n", 188 | " Process_priv: Y\n", 189 | " File_priv: Y\n", 190 | " Grant_priv: Y\n", 191 | " References_priv: Y\n", 192 | " Index_priv: Y\n", 193 | " Alter_priv: Y\n", 194 | " Show_db_priv: Y\n", 195 | " Super_priv: Y\n", 196 | " Create_tmp_table_priv: Y\n", 197 | " Lock_tables_priv: Y\n", 198 | " Execute_priv: Y\n", 199 | " Repl_slave_priv: Y\n", 200 | " Repl_client_priv: Y\n", 201 | " Create_view_priv: Y\n", 202 | " Show_view_priv: Y\n", 203 | " Create_routine_priv: Y\n", 204 | " Alter_routine_priv: Y\n", 205 | " Create_user_priv: Y\n", 206 | " Event_priv: Y\n", 207 | " Trigger_priv: Y\n", 208 | "Create_tablespace_priv: Y\n", 209 | " ssl_type: \n", 210 | " HEX(ssl_cipher): \n", 211 | " HEX(x509_issuer): \n", 212 | " HEX(x509_subject): \n", 213 | " max_questions: 0\n", 214 | " max_updates: 0\n", 215 | " max_connections: 0\n", 216 | " max_user_connections: 0\n", 217 | " plugin: mysql_native_password\n", 218 | " authentication_string: \n", 219 | " password_expired: N\n", 220 | "*************************** 4. row ***************************\n", 221 | " Host: ::1\n", 222 | " User: root\n", 223 | " Password: \n", 224 | " Select_priv: Y\n", 225 | " Insert_priv: Y\n", 226 | " Update_priv: Y\n", 227 | " Delete_priv: Y\n", 228 | " Create_priv: Y\n", 229 | " Drop_priv: Y\n", 230 | " Reload_priv: Y\n", 231 | " Shutdown_priv: Y\n", 232 | " Process_priv: Y\n", 233 | " File_priv: Y\n", 234 | " Grant_priv: Y\n", 235 | " References_priv: Y\n", 236 | " Index_priv: Y\n", 237 | " Alter_priv: Y\n", 238 | " Show_db_priv: Y\n", 239 | " Super_priv: Y\n", 240 | " Create_tmp_table_priv: Y\n", 241 | " Lock_tables_priv: Y\n", 242 | " Execute_priv: Y\n", 243 | " Repl_slave_priv: Y\n", 244 | " Repl_client_priv: Y\n", 245 | " Create_view_priv: Y\n", 246 | " Show_view_priv: Y\n", 247 | " Create_routine_priv: Y\n", 248 | " Alter_routine_priv: Y\n", 249 | " Create_user_priv: Y\n", 250 | " Event_priv: Y\n", 251 | " Trigger_priv: Y\n", 252 | "Create_tablespace_priv: Y\n", 253 | " ssl_type: \n", 254 | " HEX(ssl_cipher): \n", 255 | " HEX(x509_issuer): \n", 256 | " HEX(x509_subject): \n", 257 | " max_questions: 0\n", 258 | " max_updates: 0\n", 259 | " max_connections: 0\n", 260 | " max_user_connections: 0\n", 261 | " plugin: mysql_native_password\n", 262 | " authentication_string: \n", 263 | " password_expired: N\n", 264 | "*************************** 5. row ***************************\n", 265 | " Host: localhost\n", 266 | " User: \n", 267 | " Password: \n", 268 | " Select_priv: N\n", 269 | " Insert_priv: N\n", 270 | " Update_priv: N\n", 271 | " Delete_priv: N\n", 272 | " Create_priv: N\n", 273 | " Drop_priv: N\n", 274 | " Reload_priv: N\n", 275 | " Shutdown_priv: N\n", 276 | " Process_priv: N\n", 277 | " File_priv: N\n", 278 | " Grant_priv: N\n", 279 | " References_priv: N\n", 280 | " Index_priv: N\n", 281 | " Alter_priv: N\n", 282 | " Show_db_priv: N\n", 283 | " Super_priv: N\n", 284 | " Create_tmp_table_priv: N\n", 285 | " Lock_tables_priv: N\n", 286 | " Execute_priv: N\n", 287 | " Repl_slave_priv: N\n", 288 | " Repl_client_priv: N\n", 289 | " Create_view_priv: N\n", 290 | " Show_view_priv: N\n", 291 | " Create_routine_priv: N\n", 292 | " Alter_routine_priv: N\n", 293 | " Create_user_priv: N\n", 294 | " Event_priv: N\n", 295 | " Trigger_priv: N\n", 296 | "Create_tablespace_priv: N\n", 297 | " ssl_type: \n", 298 | " HEX(ssl_cipher): \n", 299 | " HEX(x509_issuer): \n", 300 | " HEX(x509_subject): \n", 301 | " max_questions: 0\n", 302 | " max_updates: 0\n", 303 | " max_connections: 0\n", 304 | " max_user_connections: 0\n", 305 | " plugin: mysql_native_password\n", 306 | " authentication_string: NULL\n", 307 | " password_expired: N\n", 308 | "*************************** 6. row ***************************\n", 309 | " Host: master.box\n", 310 | " User: \n", 311 | " Password: \n", 312 | " Select_priv: N\n", 313 | " Insert_priv: N\n", 314 | " Update_priv: N\n", 315 | " Delete_priv: N\n", 316 | " Create_priv: N\n", 317 | " Drop_priv: N\n", 318 | " Reload_priv: N\n", 319 | " Shutdown_priv: N\n", 320 | " Process_priv: N\n", 321 | " File_priv: N\n", 322 | " Grant_priv: N\n", 323 | " References_priv: N\n", 324 | " Index_priv: N\n", 325 | " Alter_priv: N\n", 326 | " Show_db_priv: N\n", 327 | " Super_priv: N\n", 328 | " Create_tmp_table_priv: N\n", 329 | " Lock_tables_priv: N\n", 330 | " Execute_priv: N\n", 331 | " Repl_slave_priv: N\n", 332 | " Repl_client_priv: N\n", 333 | " Create_view_priv: N\n", 334 | " Show_view_priv: N\n", 335 | " Create_routine_priv: N\n", 336 | " Alter_routine_priv: N\n", 337 | " Create_user_priv: N\n", 338 | " Event_priv: N\n", 339 | " Trigger_priv: N\n", 340 | "Create_tablespace_priv: N\n", 341 | " ssl_type: \n", 342 | " HEX(ssl_cipher): \n", 343 | " HEX(x509_issuer): \n", 344 | " HEX(x509_subject): \n", 345 | " max_questions: 0\n", 346 | " max_updates: 0\n", 347 | " max_connections: 0\n", 348 | " max_user_connections: 0\n", 349 | " plugin: mysql_native_password\n", 350 | " authentication_string: NULL\n", 351 | " password_expired: N\n", 352 | ], 353 | [ 354 | "*************************** 1. row ***************************\n", 355 | " Host: localhost\n", 356 | " User: root\n", 357 | " Password: \n", 358 | " Select_priv: Y\n", 359 | " Insert_priv: Y\n", 360 | " Update_priv: Y\n", 361 | " Delete_priv: Y\n", 362 | " Create_priv: Y\n", 363 | " Drop_priv: Y\n", 364 | " Reload_priv: Y\n", 365 | " Shutdown_priv: Y\n", 366 | " Process_priv: Y\n", 367 | " File_priv: Y\n", 368 | " Grant_priv: Y\n", 369 | " References_priv: Y\n", 370 | " Index_priv: Y\n", 371 | " Alter_priv: Y\n", 372 | " Show_db_priv: Y\n", 373 | " Super_priv: Y\n", 374 | " Create_tmp_table_priv: Y\n", 375 | " Lock_tables_priv: Y\n", 376 | " Execute_priv: Y\n", 377 | " Repl_slave_priv: Y\n", 378 | " Repl_client_priv: Y\n", 379 | " Create_view_priv: Y\n", 380 | " Show_view_priv: Y\n", 381 | " Create_routine_priv: Y\n", 382 | " Alter_routine_priv: Y\n", 383 | " Create_user_priv: Y\n", 384 | " Event_priv: Y\n", 385 | " Trigger_priv: Y\n", 386 | "Create_tablespace_priv: Y\n", 387 | " ssl_type: \n", 388 | " HEX(ssl_cipher): \n", 389 | " HEX(x509_issuer): \n", 390 | " HEX(x509_subject): \n", 391 | " max_questions: 0\n", 392 | " max_updates: 0\n", 393 | " max_connections: 0\n", 394 | " max_user_connections: 0\n", 395 | " plugin: mysql_native_password\n", 396 | " authentication_string: \n", 397 | " password_expired: N\n", 398 | "*************************** 2. row ***************************\n", 399 | " Host: slave.box\n", 400 | " User: root\n", 401 | " Password: \n", 402 | " Select_priv: Y\n", 403 | " Insert_priv: Y\n", 404 | " Update_priv: Y\n", 405 | " Delete_priv: Y\n", 406 | " Create_priv: Y\n", 407 | " Drop_priv: Y\n", 408 | " Reload_priv: Y\n", 409 | " Shutdown_priv: Y\n", 410 | " Process_priv: Y\n", 411 | " File_priv: Y\n", 412 | " Grant_priv: Y\n", 413 | " References_priv: Y\n", 414 | " Index_priv: Y\n", 415 | " Alter_priv: Y\n", 416 | " Show_db_priv: Y\n", 417 | " Super_priv: Y\n", 418 | " Create_tmp_table_priv: Y\n", 419 | " Lock_tables_priv: Y\n", 420 | " Execute_priv: Y\n", 421 | " Repl_slave_priv: Y\n", 422 | " Repl_client_priv: Y\n", 423 | " Create_view_priv: Y\n", 424 | " Show_view_priv: Y\n", 425 | " Create_routine_priv: Y\n", 426 | " Alter_routine_priv: Y\n", 427 | " Create_user_priv: Y\n", 428 | " Event_priv: Y\n", 429 | " Trigger_priv: Y\n", 430 | "Create_tablespace_priv: Y\n", 431 | " ssl_type: \n", 432 | " HEX(ssl_cipher): \n", 433 | " HEX(x509_issuer): \n", 434 | " HEX(x509_subject): \n", 435 | " max_questions: 0\n", 436 | " max_updates: 0\n", 437 | " max_connections: 0\n", 438 | " max_user_connections: 0\n", 439 | " plugin: mysql_native_password\n", 440 | " authentication_string: \n", 441 | " password_expired: N\n", 442 | "*************************** 3. row ***************************\n", 443 | " Host: 127.0.0.1\n", 444 | " User: root\n", 445 | " Password: \n", 446 | " Select_priv: Y\n", 447 | " Insert_priv: Y\n", 448 | " Update_priv: Y\n", 449 | " Delete_priv: Y\n", 450 | " Create_priv: Y\n", 451 | " Drop_priv: Y\n", 452 | " Reload_priv: Y\n", 453 | " Shutdown_priv: Y\n", 454 | " Process_priv: Y\n", 455 | " File_priv: Y\n", 456 | " Grant_priv: Y\n", 457 | " References_priv: Y\n", 458 | " Index_priv: Y\n", 459 | " Alter_priv: Y\n", 460 | " Show_db_priv: Y\n", 461 | " Super_priv: Y\n", 462 | " Create_tmp_table_priv: Y\n", 463 | " Lock_tables_priv: Y\n", 464 | " Execute_priv: Y\n", 465 | " Repl_slave_priv: Y\n", 466 | " Repl_client_priv: Y\n", 467 | " Create_view_priv: Y\n", 468 | " Show_view_priv: Y\n", 469 | " Create_routine_priv: Y\n", 470 | " Alter_routine_priv: Y\n", 471 | " Create_user_priv: Y\n", 472 | " Event_priv: Y\n", 473 | " Trigger_priv: Y\n", 474 | "Create_tablespace_priv: Y\n", 475 | " ssl_type: \n", 476 | " HEX(ssl_cipher): \n", 477 | " HEX(x509_issuer): \n", 478 | " HEX(x509_subject): \n", 479 | " max_questions: 0\n", 480 | " max_updates: 0\n", 481 | " max_connections: 0\n", 482 | " max_user_connections: 0\n", 483 | " plugin: mysql_native_password\n", 484 | " authentication_string: \n", 485 | " password_expired: N\n", 486 | "*************************** 4. row ***************************\n", 487 | " Host: ::1\n", 488 | " User: root\n", 489 | " Password: \n", 490 | " Select_priv: Y\n", 491 | " Insert_priv: Y\n", 492 | " Update_priv: Y\n", 493 | " Delete_priv: Y\n", 494 | " Create_priv: Y\n", 495 | " Drop_priv: Y\n", 496 | " Reload_priv: Y\n", 497 | " Shutdown_priv: Y\n", 498 | " Process_priv: Y\n", 499 | " File_priv: Y\n", 500 | " Grant_priv: Y\n", 501 | " References_priv: Y\n", 502 | " Index_priv: Y\n", 503 | " Alter_priv: Y\n", 504 | " Show_db_priv: Y\n", 505 | " Super_priv: Y\n", 506 | " Create_tmp_table_priv: Y\n", 507 | " Lock_tables_priv: Y\n", 508 | " Execute_priv: Y\n", 509 | " Repl_slave_priv: Y\n", 510 | " Repl_client_priv: Y\n", 511 | " Create_view_priv: Y\n", 512 | " Show_view_priv: Y\n", 513 | " Create_routine_priv: Y\n", 514 | " Alter_routine_priv: Y\n", 515 | " Create_user_priv: Y\n", 516 | " Event_priv: Y\n", 517 | " Trigger_priv: Y\n", 518 | "Create_tablespace_priv: Y\n", 519 | " ssl_type: \n", 520 | " HEX(ssl_cipher): \n", 521 | " HEX(x509_issuer): \n", 522 | " HEX(x509_subject): \n", 523 | " max_questions: 0\n", 524 | " max_updates: 0\n", 525 | " max_connections: 0\n", 526 | " max_user_connections: 0\n", 527 | " plugin: mysql_native_password\n", 528 | " authentication_string: \n", 529 | " password_expired: N\n", 530 | "*************************** 5. row ***************************\n", 531 | " Host: localhost\n", 532 | " User: \n", 533 | " Password: \n", 534 | " Select_priv: N\n", 535 | " Insert_priv: N\n", 536 | " Update_priv: N\n", 537 | " Delete_priv: N\n", 538 | " Create_priv: N\n", 539 | " Drop_priv: N\n", 540 | " Reload_priv: N\n", 541 | " Shutdown_priv: N\n", 542 | " Process_priv: N\n", 543 | " File_priv: N\n", 544 | " Grant_priv: N\n", 545 | " References_priv: N\n", 546 | " Index_priv: N\n", 547 | " Alter_priv: N\n", 548 | " Show_db_priv: N\n", 549 | " Super_priv: N\n", 550 | " Create_tmp_table_priv: N\n", 551 | " Lock_tables_priv: N\n", 552 | " Execute_priv: N\n", 553 | " Repl_slave_priv: N\n", 554 | " Repl_client_priv: N\n", 555 | " Create_view_priv: N\n", 556 | " Show_view_priv: N\n", 557 | " Create_routine_priv: N\n", 558 | " Alter_routine_priv: N\n", 559 | " Create_user_priv: N\n", 560 | " Event_priv: N\n", 561 | " Trigger_priv: N\n", 562 | "Create_tablespace_priv: N\n", 563 | " ssl_type: \n", 564 | " HEX(ssl_cipher): \n", 565 | " HEX(x509_issuer): \n", 566 | " HEX(x509_subject): \n", 567 | " max_questions: 0\n", 568 | " max_updates: 0\n", 569 | " max_connections: 0\n", 570 | " max_user_connections: 0\n", 571 | " plugin: mysql_native_password\n", 572 | " authentication_string: NULL\n", 573 | " password_expired: N\n", 574 | "*************************** 6. row ***************************\n", 575 | " Host: slave.box\n", 576 | " User: \n", 577 | " Password: \n", 578 | " Select_priv: N\n", 579 | " Insert_priv: N\n", 580 | " Update_priv: N\n", 581 | " Delete_priv: N\n", 582 | " Create_priv: N\n", 583 | " Drop_priv: N\n", 584 | " Reload_priv: N\n", 585 | " Shutdown_priv: N\n", 586 | " Process_priv: N\n", 587 | " File_priv: N\n", 588 | " Grant_priv: N\n", 589 | " References_priv: N\n", 590 | " Index_priv: N\n", 591 | " Alter_priv: N\n", 592 | " Show_db_priv: N\n", 593 | " Super_priv: N\n", 594 | " Create_tmp_table_priv: N\n", 595 | " Lock_tables_priv: N\n", 596 | " Execute_priv: N\n", 597 | " Repl_slave_priv: N\n", 598 | " Repl_client_priv: N\n", 599 | " Create_view_priv: N\n", 600 | " Show_view_priv: N\n", 601 | " Create_routine_priv: N\n", 602 | " Alter_routine_priv: N\n", 603 | " Create_user_priv: N\n", 604 | " Event_priv: N\n", 605 | " Trigger_priv: N\n", 606 | "Create_tablespace_priv: N\n", 607 | " ssl_type: \n", 608 | " HEX(ssl_cipher): \n", 609 | " HEX(x509_issuer): \n", 610 | " HEX(x509_subject): \n", 611 | " max_questions: 0\n", 612 | " max_updates: 0\n", 613 | " max_connections: 0\n", 614 | " max_user_connections: 0\n", 615 | " plugin: mysql_native_password\n", 616 | " authentication_string: NULL\n", 617 | " password_expired: N\n", 618 | ], 619 | "@@ -43,7 +43,7 @@\n" 620 | " authentication_string: \n" 621 | " password_expired: N\n" 622 | " *************************** 2. row ***************************\n" 623 | "- Host: master.box\n" 624 | "+ Host: slave.box\n" 625 | " User: root\n" 626 | " Password: \n" 627 | " Select_priv: Y\n" 628 | "@@ -219,7 +219,7 @@\n" 629 | " authentication_string: NULL\n" 630 | " password_expired: N\n" 631 | " *************************** 6. row ***************************\n" 632 | "- Host: master.box\n" 633 | "+ Host: slave.box\n" 634 | " User: \n" 635 | " Password: \n" 636 | " Select_priv: N\n" 637 | ) 638 | ]) 639 | def test_diff(master_lines, slave_lines, difference): 640 | 641 | actual_diff = diff(master_lines, slave_lines) 642 | assert actual_diff == difference 643 | 644 | 645 | @mock.patch('twindb_table_compare.compare.Popen') 646 | def test_print_vertical(mock_popen, out_master, out_slave): 647 | mock_proc = mock.Mock() 648 | mock_proc.communicate.side_effect = [out_master, out_slave] 649 | mock_proc.returncode = 0 650 | 651 | mock_popen.return_value = mock_proc 652 | assert print_vertical('foo1', 'foo2', 'foo3', 'foo4', 'foo5', 653 | color=False) == """@@ -43,7 +43,7 @@ 654 | authentication_string: 655 | password_expired: N 656 | ************************************************************** 657 | - Host: master.box 658 | + Host: slave.box 659 | User: root 660 | Password: 661 | Select_priv: Y 662 | @@ -219,7 +219,7 @@ 663 | authentication_string: NULL 664 | password_expired: N 665 | ************************************************************** 666 | - Host: master.box 667 | + Host: slave.box 668 | User: 669 | Password: 670 | Select_priv: N 671 | """ 672 | 673 | 674 | @pytest.mark.parametrize('fields, result', [ 675 | ( 676 | ( 677 | ('Host', 'char'), 678 | ('User', 'char'), 679 | ('Proxied_host', 'char'), 680 | ('Proxied_user', 'char'), 681 | ('With_grant', 'tinyint'), 682 | ('Grantor', 'char'), 683 | ('Timestamp', 'timestamp') 684 | ), 685 | 'Host, User, Proxied_host, Proxied_user, With_grant, Grantor, Timestamp' 686 | ), 687 | ( 688 | ( 689 | ('f1', 'char'), 690 | ('f2', 'blob') 691 | ), 692 | 'f1, HEX(f2)' 693 | ), 694 | ( 695 | ( 696 | ('f1', 'char'), 697 | ('f2', 'BLOB') 698 | ), 699 | 'f1, HEX(f2)' 700 | ), 701 | ( 702 | ( 703 | ('f1', 'char'), 704 | ('f2', 'mediumblob') 705 | ), 706 | 'f1, HEX(f2)' 707 | ), 708 | ( 709 | ( 710 | ('f1', 'char'), 711 | ('f2', 'BINARY'), 712 | ('f2', 'VARBINARY'), 713 | ('f2', 'TINYBLOB'), 714 | ('f2', 'BLOB'), 715 | ('f2', 'MEDIUMBLOB'), 716 | ('f2', 'LONGBLOB'), 717 | ), 718 | 'f1, HEX(f2), HEX(f2), HEX(f2), HEX(f2), HEX(f2), HEX(f2)' 719 | ) 720 | ]) 721 | def test_get_fields(fields, result): 722 | mock_conn = mock.Mock() 723 | mock_cursor = mock.Mock() 724 | mock_cursor.fetchall.return_value = fields 725 | mock_conn.cursor.return_value = mock_cursor 726 | assert get_fields(mock_conn, 'foo', 'bar') == result 727 | 728 | 729 | @mock.patch('twindb_table_compare.compare.primary_exists') 730 | @mock.patch('twindb_table_compare.compare.get_fields') 731 | @mock.patch('twindb_table_compare.compare.get_boundary') 732 | @mock.patch('twindb_table_compare.compare.get_index_fields') 733 | @mock.patch('twindb_table_compare.compare.get_chunk_index') 734 | def test_build_chunk_query(mock_get_chunk_index, 735 | mock_get_index_fields, 736 | mock_get_boundary, 737 | mock_get_fields, 738 | mock_primary_exists): 739 | 740 | mock_get_chunk_index.return_value = 'PRIMARY' 741 | mock_get_index_fields.return_value = ['id'] 742 | mock_get_boundary.return_value = ('1', '186139') 743 | mock_get_fields.return_value = 'id, name' 744 | mock_primary_exists.return_value = True 745 | 746 | mock_connection = mock.Mock() 747 | 748 | assert build_chunk_query('test', 't1', 1, mock_connection) == \ 749 | "SELECT id, name FROM `test`.`t1` USE INDEX (PRIMARY) " \ 750 | "WHERE ( 0 OR ( 1 AND `id` >= '1' ) ) " \ 751 | "AND ( 0 OR ( 1 AND `id` <= '186139' ) )" 752 | 753 | 754 | @pytest.mark.parametrize('lower_boundary, upper_boundary, result', [ 755 | ( 756 | '1', 757 | '186139', 758 | "WHERE ( 0 OR ( 1 AND `id` >= '1' ) ) AND ( 0 OR ( 1 AND `id` <= '186139' ) )" 759 | ), 760 | ( 761 | None, 762 | '186139', 763 | "WHERE 1 AND ( 0 OR ( 1 AND `id` <= '186139' ) )" 764 | ), 765 | ( 766 | '1', 767 | None, 768 | "WHERE ( 0 OR ( 1 AND `id` >= '1' ) ) AND 1" 769 | ) 770 | ]) 771 | def test_get_where(lower_boundary, upper_boundary, result): 772 | assert get_where(lower_boundary, upper_boundary, ['id']) == result 773 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36, py37, py38, lint, cov 3 | 4 | [testenv:lint] 5 | basepython=python 6 | commands=pylint twindb_table_compare 7 | 8 | [testenv:cov] 9 | deps=-rrequirements_dev.txt 10 | commands=py.test --cov=twindb_table_compare --cov-report term-missing tests/unit 11 | 12 | [testenv] 13 | passenv = CI TRAVIS TRAVIS_* 14 | setenv = 15 | PYTHONPATH = {toxinidir}:{toxinidir}/twindb_table_compare 16 | deps = 17 | -r{toxinidir}/requirements_dev.txt 18 | commands = 19 | pip install -U pip 20 | py.test --cov=./twindb_table_compare tests/unit/ 21 | codecov 22 | 23 | 24 | ; If you want to make tox run the tests with the same versions, create a 25 | ; requirements.txt with the pinned versions and uncomment the following lines: 26 | ; deps = 27 | ; -r{toxinidir}/requirements.txt 28 | -------------------------------------------------------------------------------- /twindb_table_compare/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to read pr-table-checksum's result table (percona.checksums) 3 | and show user which records are actually different. 4 | """ 5 | import logging 6 | from logutils.colorize import ColorizingStreamHandler 7 | 8 | __author__ = "Aleksandr Kuzminsky" 9 | __email__ = "aleks@twindb.com" 10 | __version__ = "3.0.3" 11 | 12 | LOG = logging.getLogger(__name__) 13 | 14 | 15 | def setup_logging(logger, debug=False, color=True): 16 | """ 17 | Configure logging. 18 | 19 | :param logger: Logger to configure. 20 | :type logger: Logger 21 | :param debug: If True - print debug messages 22 | :param color: If True - print colored messages 23 | """ 24 | 25 | fmt_str = ( 26 | "%(asctime)s: %(levelname)s:" 27 | " %(module)s.%(funcName)s():%(lineno)d: %(message)s" 28 | ) 29 | 30 | logger.handlers = [] 31 | if color: 32 | colored_console_handler = ColorizingStreamHandler() 33 | colored_console_handler.level_map[logging.INFO] = (None, "cyan", False) 34 | colored_console_handler.setFormatter(logging.Formatter(fmt_str)) 35 | logger.addHandler(colored_console_handler) 36 | else: 37 | console_handler = logging.StreamHandler() 38 | console_handler.setFormatter(logging.Formatter(fmt_str)) 39 | logger.addHandler(console_handler) 40 | 41 | if debug: 42 | logger.setLevel(logging.DEBUG) 43 | else: 44 | logger.setLevel(logging.INFO) 45 | -------------------------------------------------------------------------------- /twindb_table_compare/cli.py: -------------------------------------------------------------------------------- 1 | """ 2 | Command line routines 3 | """ 4 | from __future__ import print_function 5 | import pwd 6 | import os 7 | import sys 8 | 9 | import click 10 | from pymysql import MySQLError 11 | 12 | from twindb_table_compare.compare import get_inconsistencies, get_inconsistent_tables 13 | 14 | from . import __version__, setup_logging, LOG 15 | 16 | 17 | @click.command() 18 | @click.option( 19 | "--user", 20 | default=pwd.getpwuid(os.getuid()).pw_name, 21 | help="User name to connect to MySQL", 22 | ) 23 | @click.option("--password", default="", help="Password to connect to MySQL") 24 | @click.option( 25 | "--db", default="percona", help="Database where checksums table is stored" 26 | ) 27 | @click.option("--tbl", default="checksums", help="Table with checksums") 28 | @click.option( 29 | "--vertical", 30 | default=False, 31 | is_flag=True, 32 | help="Print result vertically. " "Otherwise will print one record in one line", 33 | ) 34 | @click.option("--version", is_flag=True, help="Print version and exit", default=False) 35 | @click.option("--debug", is_flag=True, help="Print debug messages", default=False) 36 | @click.option( 37 | "--color/--no-color", is_flag=True, help="Print colored log messages", default=True 38 | ) 39 | @click.argument( 40 | "slave", default="localhost", required=False 41 | ) # pylint: disable=too-many-arguments 42 | def main(user, password, db, tbl, slave, vertical, debug, version, color): 43 | """twindb_table_compare reads percona.checksums from the master and slave 44 | and shows records that differ if there are any inconsistencies.""" 45 | if version: 46 | print(__version__) 47 | sys.exit(0) 48 | 49 | setup_logging(LOG, debug=debug, color=color) 50 | try: 51 | for database, table in get_inconsistent_tables( 52 | slave, user, password, ch_db=db, ch_tbl=tbl 53 | ): 54 | get_inconsistencies( 55 | database, 56 | table, 57 | slave, 58 | user, 59 | password, 60 | ch_db=db, 61 | ch_tbl=tbl, 62 | vertical=vertical, 63 | color=color, 64 | ) 65 | except MySQLError as err: # pylint: disable=no-member 66 | LOG.error(err) 67 | sys.exit(1) 68 | -------------------------------------------------------------------------------- /twindb_table_compare/compare.py: -------------------------------------------------------------------------------- 1 | """ 2 | Functions to find and print differences 3 | """ 4 | from __future__ import print_function 5 | import binascii 6 | from difflib import unified_diff 7 | import os 8 | import string 9 | from subprocess import Popen, PIPE 10 | import sys 11 | import tempfile 12 | 13 | from pymysql import connect 14 | from pymysql.cursors import DictCursor 15 | 16 | from . import LOG 17 | 18 | 19 | def is_printable(str_value): 20 | """ 21 | Checks if str_value is printable string. 22 | 23 | :param str_value: 24 | :return: True if str_value is printable. False otherwise 25 | """ 26 | return set(str_value).issubset(string.printable) 27 | 28 | 29 | def get_chunk_index(*args, **kwargs): 30 | """ 31 | Get index that was used to cut the chunk. 32 | 33 | :param args: Positional arguments. 34 | :type args: tuple(Connection, str, str, int) 35 | :param kwargs: Keyword arguments. 36 | :type kwargs: dict 37 | :return: index name or None if no index was used 38 | 39 | .. rubric:: Positional arguments 40 | 41 | - **connection**: MySQL connection. 42 | - **database**: database of the chunk. 43 | - **tbl**: table of the chunk. 44 | - **chunk**: chunk id. 45 | 46 | .. rubric:: Keyword arguments. 47 | 48 | - **ch_db**: Database where checksums are stored. Default percona. 49 | - **ch_tbl**: Table where checksums are stored. Default checksums. 50 | """ 51 | connection = args[0] 52 | database = args[1] 53 | tbl = args[2] 54 | chunk = args[3] 55 | ch_db = kwargs["ch_db"] 56 | ch_tbl = kwargs["ch_tbl"] 57 | 58 | cur = connection.cursor() 59 | query = ( 60 | "SELECT chunk_index FROM `%s`.`%s` WHERE db='%s' AND tbl='%s' AND chunk = %s" 61 | ) 62 | 63 | LOG.info("Executing %s", query % (ch_db, ch_tbl, database, tbl, chunk)) 64 | cur.execute(query % (ch_db, ch_tbl, database, tbl, chunk)) 65 | return cur.fetchone()[0] 66 | 67 | 68 | def get_index_fields(connection, db, tbl, index): 69 | """ 70 | Get fields of the given index 71 | 72 | :param connection: MySQLDb connection 73 | :param db: database 74 | :param tbl: table 75 | :param index: index name 76 | :return: list of field names 77 | """ 78 | cur = connection.cursor() 79 | query = ( 80 | "SELECT COLUMN_NAME FROM information_schema.STATISTICS " 81 | "WHERE TABLE_SCHEMA='%s' " 82 | "AND TABLE_NAME='%s' " 83 | "AND INDEX_NAME='%s' " 84 | "ORDER BY SEQ_IN_INDEX" 85 | ) 86 | LOG.info("Executing %s", query % (db, tbl, index)) 87 | cur.execute(query % (db, tbl, index)) 88 | cols = [] 89 | for row in cur.fetchall(): 90 | cols.append(row[0]) 91 | return cols 92 | 93 | 94 | def get_boundary(*args, **kwargs): 95 | """ 96 | Get lower and upper boundary values of a chunk. 97 | 98 | :param args: Positional arguments. 99 | :type args: tuple(Connection, str, str, int) 100 | :param kwargs: Keyword arguments. 101 | :type kwargs: dict 102 | :return: tuple with values lower_boundary and upper_boundary of 103 | percona.checksums 104 | 105 | .. rubric:: Positional arguments 106 | 107 | - **connection**: MySQL connection. 108 | - **database**: database of the chunk. 109 | - **tbl**: table of the chunk. 110 | - **chunk**: chunk id. 111 | 112 | .. rubric:: Keyword arguments. 113 | 114 | - **ch_db**: Database where checksums are stored. Default percona. 115 | - **ch_tbl**: Table where checksums are stored. Default checksums. 116 | """ 117 | connection = args[0] 118 | database = args[1] 119 | tbl = args[2] 120 | chunk = args[3] 121 | ch_db = kwargs["ch_db"] 122 | ch_tbl = kwargs["ch_tbl"] 123 | 124 | cur = connection.cursor() 125 | query = ( 126 | "SELECT lower_boundary, upper_boundary FROM `%s`.`%s` " 127 | "WHERE db='%s' AND tbl='%s' AND chunk = %s" 128 | ) 129 | LOG.info("Executing %s", query % (ch_db, ch_tbl, database, tbl, chunk)) 130 | cur.execute(query % (ch_db, ch_tbl, database, tbl, chunk)) 131 | return cur.fetchone() 132 | 133 | 134 | def get_master(connection): 135 | """ 136 | Get master host 137 | 138 | :param connection: MySQL connection 139 | :return: Master hostname 140 | """ 141 | cur = connection.cursor(DictCursor) 142 | query = "SHOW SLAVE STATUS" 143 | LOG.info("Executing %s", query) 144 | cur.execute(query) 145 | return cur.fetchone()["Master_Host"] 146 | 147 | 148 | # Colorize diff 149 | # https://goo.gl/GqSyoj 150 | def _green(line): 151 | if sys.stdout.isatty(): 152 | return "\033[92m" + line + "\033[0m" 153 | 154 | return line 155 | 156 | 157 | def _red(line): 158 | if sys.stdout.isatty(): 159 | return "\033[91m" + line + "\033[0m" 160 | 161 | return line 162 | 163 | 164 | def diff(master_lines, slave_lines, color=True): 165 | """ 166 | Find differences between two set of lines. 167 | 168 | :param master_lines: First set of lines 169 | :type master_lines: list 170 | :param slave_lines: Second set of lines 171 | :type slave_lines: list 172 | :param color: If True return colored diff 173 | :type color: bool 174 | :return: Difference between two set of lines 175 | :rtype: str 176 | """ 177 | result = "" 178 | for line in unified_diff(master_lines, slave_lines): 179 | if not line.startswith("---") and not line.startswith("+++"): 180 | if not line.endswith("\n"): 181 | # print(result) 182 | line += "\n" 183 | 184 | if line.startswith("+"): 185 | 186 | if color: 187 | result += _green(line) 188 | else: 189 | result += line 190 | 191 | elif line.startswith("-"): 192 | 193 | if color: 194 | result += _red(line) 195 | else: 196 | result += line 197 | else: 198 | result += line 199 | 200 | return result 201 | 202 | 203 | def get_fields(conn, db, tbl): 204 | """ 205 | Construct fields list string for a SELECT. 206 | If a field is a binary type (BLOB, VARBINARY) then HEX() it. 207 | 208 | :param conn: MySQL connection. 209 | :type conn: Connection 210 | :param db: Database name. 211 | :type db: str 212 | :param tbl: Table name. 213 | :type tbl: str 214 | :return: A comma separated list of fields. 215 | :rtype: str 216 | """ 217 | query = ( 218 | "SELECT COLUMN_NAME, DATA_TYPE " 219 | "FROM information_schema.COLUMNS " 220 | "WHERE TABLE_SCHEMA='{db}' AND TABLE_NAME='{tbl}' " 221 | "ORDER BY ORDINAL_POSITION".format(db=db, tbl=tbl) 222 | ) 223 | cursor = conn.cursor() 224 | cursor.execute(query) 225 | 226 | fields = [] 227 | for row in cursor.fetchall(): 228 | col_name = row[0] 229 | col_type = row[1].lower() 230 | if col_type in [ 231 | "tinyblob", 232 | "mediumblob", 233 | "blob", 234 | "longblob", 235 | "binary", 236 | "varbinary", 237 | ]: 238 | col_name = "HEX(%s)" % col_name 239 | 240 | fields.append(col_name) 241 | 242 | return ", ".join(fields) 243 | 244 | 245 | def primary_exists(conn, db, tbl): 246 | """ 247 | Check if PRIMARY index exists in table db.tbl 248 | 249 | :param conn: MySQLdb connection. 250 | :type conn: Connection 251 | :param db: Database name. 252 | :type db: str 253 | :param tbl: Table name. 254 | :type tbl: str 255 | :return: True if index PRIMARY exists in table db.tbl 256 | :rtype: bool 257 | """ 258 | query = ( 259 | "SELECT COUNT(*)" 260 | "FROM INFORMATION_SCHEMA.STATISTICS " 261 | "WHERE TABLE_SCHEMA = %s " 262 | "AND TABLE_NAME = %s" 263 | "AND INDEX_NAME = 'PRIMARY'" 264 | ) 265 | cursor = conn.cursor() 266 | cursor.execute(query, (db, tbl)) 267 | 268 | n_fields = cursor.fetchone()[0] 269 | 270 | LOG.debug("Number of fields in PRIMARY index %d", n_fields) 271 | 272 | return bool(n_fields > 0) 273 | 274 | 275 | def get_boundary_clause(oper=">", index_fields=None, boundaries=None): 276 | """ 277 | Generate a clause for the WHERE statement based on field names and values. 278 | 279 | :param oper: Can be either ``<`` for the lower boundary 280 | or ``>`` for the upper boundary. 281 | :param index_fields: list of fields in the index. 282 | :type index_fields: list 283 | :param boundaries: list of values for a boundary. 284 | :type boundaries: list 285 | :return: a clause that defines a boundary (upper or lower) of a chunk. 286 | """ 287 | # generate boundary clause 288 | index_field_last = index_fields[len(index_fields) - 1] 289 | LOG.debug("index last field: %s", index_field_last) 290 | 291 | clause_fields = [] 292 | v_num = 0 293 | where = "( 0" 294 | oper_current = oper 295 | for index_field in index_fields: 296 | clause_fields.append(index_field) 297 | where += " OR ( 1" 298 | for clause_field in clause_fields: 299 | if clause_field == clause_fields[len(clause_fields) - 1]: 300 | if clause_field == index_field_last: 301 | oper_current = oper + "=" 302 | else: 303 | oper_current = oper 304 | value = boundaries[v_num] 305 | v_num += 1 306 | if is_printable(value): 307 | where += " AND `%s` %s '%s'" % (clause_field, oper_current, value) 308 | else: 309 | value = "UNHEX('%s')" % binascii.hexlify(str(value)) 310 | where += " AND `%s` %s %s" % (clause_field, oper_current, value) 311 | oper = "=" 312 | where += " )" 313 | where += " )" 314 | return where 315 | 316 | 317 | def get_where(lower_boundary, upper_boundary, index_fields): 318 | """ 319 | Generate WHERE clause based on strings ``lower_boundary``, 320 | ``upper_boundary`` from ``percona.checksums`` table and fields in the index. 321 | 322 | :param lower_boundary: values of lower boundary. 323 | :type lower_boundary: str 324 | :param upper_boundary: values of upper boundary. 325 | :type upper_boundary: str 326 | :param index_fields: list of fields in the index that was used. 327 | to access a chunk. 328 | :return: a WHERE clause to read a chunk. 329 | :rtype: str 330 | """ 331 | 332 | try: 333 | lower_boundaries = lower_boundary.split(",") 334 | 335 | lower_clause = get_boundary_clause( 336 | oper=">", index_fields=index_fields, boundaries=lower_boundaries 337 | ) 338 | except AttributeError: 339 | lower_clause = "1" 340 | 341 | try: 342 | upper_boundaries = upper_boundary.split(",") 343 | upper_clause = get_boundary_clause( 344 | oper="<", index_fields=index_fields, boundaries=upper_boundaries 345 | ) 346 | 347 | except AttributeError: 348 | upper_clause = "1" 349 | 350 | result = "WHERE {lower} AND {upper}".format(lower=lower_clause, upper=upper_clause) 351 | return result 352 | 353 | 354 | # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements 355 | def build_chunk_query(db, tbl, chunk, conn, ch_db="percona", ch_tbl="checksums"): 356 | """For a given database, table and chunk number construct 357 | a SELECT query that would return records in this chunk. 358 | """ 359 | 360 | LOG.info("# %s.%s, chunk %d", db, tbl, chunk) 361 | chunk_index = get_chunk_index(conn, db, tbl, chunk, ch_db=ch_db, ch_tbl=ch_tbl) 362 | LOG.info("# chunk index: %s", chunk_index) 363 | 364 | if chunk_index: 365 | index_fields = get_index_fields(conn, db, tbl, chunk_index) 366 | 367 | lower_boundary, upper_boundary = get_boundary( 368 | conn, db, tbl, chunk, ch_db=ch_db, ch_tbl=ch_tbl 369 | ) 370 | 371 | where = get_where(lower_boundary, upper_boundary, index_fields) 372 | else: 373 | where = "WHERE 1" 374 | 375 | fields = get_fields(conn, db, tbl) 376 | if primary_exists(conn, db, tbl): 377 | index_hint = "USE INDEX (PRIMARY)" 378 | else: 379 | index_hint = "" 380 | 381 | query = "SELECT %s FROM `%s`.`%s` %s %s" % (fields, db, tbl, index_hint, where) 382 | 383 | return query 384 | 385 | 386 | def print_horizontal(cur_master, cur_slave, query, color=True): 387 | """ 388 | Find and return differences in horizontal format i.e. one line 389 | - one record 390 | 391 | :param cur_master: MySQLdb cursor on master 392 | :type cur_master: Cursor 393 | :param cur_slave: MySQLdb cursor on slave 394 | :type cur_slave: Cursor 395 | :param query: Query to find records in a chunk we compare 396 | :type query: str 397 | :param color: If True - produce colorful output 398 | :return: Differences in a chunk between master and slave 399 | :rtype: str 400 | """ 401 | 402 | LOG.info("Executing: %s", query) 403 | 404 | master_f, master_filename = tempfile.mkstemp(prefix="master.") 405 | slave_f, slave_filename = tempfile.mkstemp(prefix="slave.") 406 | # Now fetch records from the master and slave 407 | # and write them to temporary files 408 | # If a field contains unprintable characters print the field in HEX 409 | cur_master.execute(query) 410 | result = cur_master.fetchall() 411 | 412 | for row in result: 413 | for field in row: 414 | # print(field) 415 | if is_printable(str(field)): 416 | os.write(master_f, str(field).encode()) 417 | else: 418 | # pprint HEX-ed string 419 | os.write(master_f, binascii.hexlify(str(field).encode())) 420 | os.write(master_f, "\t".encode()) 421 | os.write(master_f, "\n".encode()) 422 | os.close(master_f) 423 | 424 | LOG.info("Executing: %s", query) 425 | cur_slave.execute(query) 426 | result = cur_slave.fetchall() 427 | for row in result: 428 | for field in row: 429 | if is_printable(str(field)): 430 | os.write(slave_f, str(field).encode()) 431 | else: 432 | # pprint HEX-ed string 433 | os.write(slave_f, binascii.hexlify(str(field).encode())) 434 | os.write(slave_f, "\t".encode()) 435 | os.write(slave_f, "\n".encode()) 436 | os.close(slave_f) 437 | 438 | diffs = diff( 439 | open(master_filename).readlines(), open(slave_filename).readlines(), color=color 440 | ) 441 | os.remove(master_filename) 442 | os.remove(slave_filename) 443 | return diffs 444 | 445 | 446 | def print_vertical(master, slave, user, passwd, query, color=True): 447 | r""" 448 | Find and return differences in vertical format. 449 | The vertical format is when you end MySQL query with '\G' 450 | 451 | :param master: Hostname of the master. 452 | :type master: str 453 | :param slave: Hostname of the slave. 454 | :type slave: str 455 | :param query: Query to find records in a chunk we compare 456 | :type query: str 457 | :param color: If True - produce colorful output 458 | :return: Differences in a chunk between master and slave 459 | :rtype: str 460 | """ 461 | LOG.info("Executing: %s", query) 462 | 463 | proc = Popen( 464 | ["mysql", "-h", master, "-u", user, "-p%s" % passwd, "-e", r"%s\G" % query], 465 | stdout=PIPE, 466 | stderr=PIPE, 467 | ) 468 | master_cout, master_cerr = proc.communicate() 469 | master_lines = [] 470 | for line in master_cout.split("\n"): 471 | if line.startswith("***************************"): 472 | master_lines.append( 473 | "*******************************" "*******************************" 474 | ) 475 | else: 476 | master_lines.append(line) 477 | 478 | if proc.returncode: 479 | LOG.error("Failed to query master.") 480 | LOG.error(master_cerr) 481 | sys.exit(1) 482 | 483 | LOG.info("Executing: %s", query) 484 | proc = Popen( 485 | ["mysql", "-h", slave, "-u", user, "-p%s" % passwd, "-e", r"%s\G" % query], 486 | stdout=PIPE, 487 | stderr=PIPE, 488 | ) 489 | slave_cout, slave_cerr = proc.communicate() 490 | 491 | slave_lines = [] 492 | for line in slave_cout.split("\n"): 493 | if line.startswith("***************************"): 494 | slave_lines.append( 495 | "*******************************" "*******************************" 496 | ) 497 | else: 498 | slave_lines.append(line) 499 | 500 | if proc.returncode: 501 | LOG.error("Failed to query slave.") 502 | LOG.error(slave_cerr) 503 | sys.exit(1) 504 | 505 | return diff(master_lines, slave_lines, color=color) 506 | 507 | 508 | def get_inconsistencies( 509 | db, 510 | tbl, 511 | slave, 512 | user, 513 | passwd, 514 | ch_db="percona", 515 | ch_tbl="checksums", 516 | vertical=False, 517 | color=True, 518 | ): 519 | r""" 520 | Print differences between slave and its master. 521 | 522 | :param db: Database name of the inconsistent table. 523 | :param tbl: Table name of the inconsistent table. 524 | :param slave: Hostname of the slave. 525 | :param user: User to connect to MySQL. 526 | :param passwd: Password to connect to MySQL. 527 | :param ch_db: Database where checksums are stored. 528 | :param ch_tbl: Table name where checksums are stored. 529 | :param vertical: If True - print result vertically (\G in MySQL) 530 | :param color: If True - print colorful output 531 | """ 532 | conn_slave = connect(host=slave, user=user, passwd=passwd) 533 | master = get_master(conn_slave) 534 | conn_master = connect(host=master, user=user, passwd=passwd) 535 | 536 | # Get chunks that are different on the slave and its master 537 | query = ( 538 | "SELECT chunk " 539 | "FROM `%s`.`%s` " 540 | "WHERE (this_crc<>master_crc OR this_cnt<>master_cnt) " 541 | "AND db='%s' AND tbl='%s'" 542 | ) 543 | LOG.info("Executing: %s", query % (ch_db, ch_tbl, db, tbl)) 544 | cur_master = conn_master.cursor() 545 | cur_slave = conn_slave.cursor() 546 | 547 | cur_slave.execute(query % (ch_db, ch_tbl, db, tbl)) 548 | chunks = cur_slave.fetchall() 549 | 550 | if len(chunks) == 1: 551 | chunks_str = "chunk" 552 | else: 553 | chunks_str = "chunks" 554 | LOG.info("Found %d inconsistent %s", len(chunks), chunks_str) 555 | # generate WHERE clause to fetch records of the chunk 556 | for (chunk,) in chunks: 557 | 558 | query = build_chunk_query( 559 | db, tbl, chunk, conn_slave, ch_db=ch_db, ch_tbl=ch_tbl 560 | ) 561 | 562 | if vertical: 563 | diffs = print_vertical(master, slave, user, passwd, query, color=color) 564 | else: 565 | diffs = print_horizontal(cur_master, cur_slave, query, color=color) 566 | LOG.info("Differences between slave %s and its master:", slave) 567 | 568 | print(diffs) 569 | 570 | 571 | def get_inconsistent_tables(host, user, password, ch_db="percona", ch_tbl="checksums"): 572 | """ 573 | On a given MySQL server find tables that are inconsistent with the master. 574 | 575 | :param host: Hostname with potentially inconsistent tables. 576 | :param user: MySQL user. 577 | :param password: MySQL password. 578 | :param ch_db: Database where checksums are stored. 579 | :param ch_tbl: Table name where checksums are stored. 580 | :return: List of tuples with inconsistent tables. 581 | Each tuple is database name, table name. 582 | :rtype: list 583 | """ 584 | conn = connect(host=host, user=user, passwd=password) 585 | cur = conn.cursor() 586 | cur.execute( 587 | "SELECT db, tbl FROM `%s`.`%s` " 588 | "WHERE this_crc <> master_crc OR this_cnt <> master_cnt" % (ch_db, ch_tbl) 589 | ) 590 | return cur.fetchall() 591 | -------------------------------------------------------------------------------- /vagrant/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | nodes = [ 4 | { :hostname => 'master', 5 | :ip => '192.168.35.250', 6 | :box => 'ubuntu/focal64', 7 | :ram => 3072 8 | }, 9 | { :hostname => 'slave', 10 | :ip => '192.168.35.251', 11 | :box => 'ubuntu/focal64', 12 | :ram => 3072 13 | } 14 | ] 15 | 16 | # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! 17 | VAGRANTFILE_API_VERSION = '2' 18 | 19 | $script = <