├── db_sync_tool ├── __init__.py ├── database │ ├── __init__.py │ ├── utility.py │ └── process.py ├── recipes │ ├── __init__.py │ ├── laravel.py │ ├── wordpress.py │ ├── drupal.py │ ├── symfony.py │ └── typo3.py ├── remote │ ├── __init__.py │ ├── system.py │ ├── utility.py │ ├── rsync.py │ ├── transfer.py │ └── client.py ├── utility │ ├── __init__.py │ ├── log.py │ ├── info.py │ ├── validation.py │ ├── output.py │ ├── parser.py │ ├── mode.py │ ├── helper.py │ └── system.py ├── info.py ├── sync.py └── __main__.py ├── docs ├── images │ ├── sm-proxy.png │ ├── sm-sender.png │ ├── sm-receiver.png │ ├── sm-dump-local.png │ ├── sm-dump-remote.png │ ├── sm-sync-local.png │ ├── sm-sync-remote.png │ ├── sync-mode-proxy.png │ ├── sync-mode-sender.png │ ├── sync-mode-receiver.png │ ├── sync-mode-dump-local.png │ ├── sync-mode-dump-remote.png │ ├── sync-mode-sync-local.png │ ├── sync-mode-sync-remote.png │ └── db-sync-tool-example-receiver.gif ├── dist │ ├── sf-db-sync.json.dist │ └── t3-db-sync.json.dist ├── quickstart │ ├── DRUPAL.md │ ├── WORDPRESS.md │ ├── START.md │ ├── SYMFONY.md │ └── TYPO3.md ├── RELEASE.md ├── MODE.md └── CONFIG.md ├── requirements.txt ├── sync.py ├── composer.json ├── .travis.yml ├── LICENSE ├── setup.py ├── CHANGELOG.md └── README.md /db_sync_tool/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_sync_tool/database/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_sync_tool/recipes/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_sync_tool/remote/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db_sync_tool/utility/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/images/sm-proxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-proxy.png -------------------------------------------------------------------------------- /docs/images/sm-sender.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-sender.png -------------------------------------------------------------------------------- /docs/images/sm-receiver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-receiver.png -------------------------------------------------------------------------------- /docs/images/sm-dump-local.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-dump-local.png -------------------------------------------------------------------------------- /docs/images/sm-dump-remote.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-dump-remote.png -------------------------------------------------------------------------------- /docs/images/sm-sync-local.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-sync-local.png -------------------------------------------------------------------------------- /docs/images/sm-sync-remote.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sm-sync-remote.png -------------------------------------------------------------------------------- /docs/images/sync-mode-proxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-proxy.png -------------------------------------------------------------------------------- /docs/images/sync-mode-sender.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-sender.png -------------------------------------------------------------------------------- /docs/images/sync-mode-receiver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-receiver.png -------------------------------------------------------------------------------- /docs/images/sync-mode-dump-local.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-dump-local.png -------------------------------------------------------------------------------- /docs/images/sync-mode-dump-remote.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-dump-remote.png -------------------------------------------------------------------------------- /docs/images/sync-mode-sync-local.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-sync-local.png -------------------------------------------------------------------------------- /docs/images/sync-mode-sync-remote.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/sync-mode-sync-remote.png -------------------------------------------------------------------------------- /docs/images/db-sync-tool-example-receiver.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackd248/db-sync-tool/HEAD/docs/images/db-sync-tool-example-receiver.gif -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | paramiko>=2.12 2 | future-fstrings>=1.2.0 3 | pyyaml>=6.0 4 | jsonschema>=4.2.1 5 | requests>=2.28.0 6 | semantic_version>=2.8.5 7 | yaspin>=2.3 -------------------------------------------------------------------------------- /db_sync_tool/info.py: -------------------------------------------------------------------------------- 1 | """ 2 | Info script 3 | """ 4 | __version__ = "2.11.12" 5 | __pypi_package_url__ = "https://pypi.org/pypi/db-sync-tool-kmi" 6 | __homepage__ = "https://github.com/jackd248/db-sync-tool" 7 | -------------------------------------------------------------------------------- /sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | from db_sync_tool import __main__ 4 | 5 | # 6 | # Legacy feature to support old module structure 7 | # 8 | if __name__ == "__main__": 9 | __main__.main() 10 | -------------------------------------------------------------------------------- /docs/dist/sf-db-sync.json.dist: -------------------------------------------------------------------------------- 1 | { 2 | "name": "project", 3 | "type": "Symfony", 4 | "target": { 5 | "path": "/var/www/html/app/.env" 6 | }, 7 | "origin": { 8 | "host": "ssh_host", 9 | "user": "ssh_user", 10 | "path": "/var/www/html/project/shared/.env" 11 | }, 12 | "ignore_table": [] 13 | } -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kmi/db-sync-tool", 3 | "description": "Python script to synchronize a database from and to client systems.", 4 | "version": "2.11.12", 5 | "license": "MIT", 6 | "homepage": "https://github.com/jackd248/db-sync-tool", 7 | "type": "project", 8 | "authors": [ 9 | { 10 | "name": "Konrad Michalik", 11 | "email": "support@konradmichalik.eu" 12 | } 13 | ], 14 | "support": { 15 | "issues": "https://github.com/jackd248/db-sync-tool/issues" 16 | }, 17 | "keywords": [ 18 | "database", 19 | "synchronisation", 20 | "import", 21 | "export", 22 | "sync", 23 | "typo3", 24 | "symfony", 25 | "wordpress", 26 | "drupal" 27 | ] 28 | } -------------------------------------------------------------------------------- /docs/dist/t3-db-sync.json.dist: -------------------------------------------------------------------------------- 1 | { 2 | "name": "project", 3 | "type": "TYPO3", 4 | "target": { 5 | "path": "/var/www/html/htdocs/typo3/web/typo3conf/LocalConfiguration.php" 6 | }, 7 | "origin": { 8 | "host": "ssh_host", 9 | "user": "ssh_user", 10 | "path": "/var/www/html/project/shared/typo3conf/LocalConfiguration.php" 11 | }, 12 | "ignore_table": [ 13 | "sys_domain", 14 | "cf_cache_hash", 15 | "cf_cache_hash_tags", 16 | "cf_cache_news_category", 17 | "cf_cache_news_category_tags", 18 | "cf_cache_pages", 19 | "cf_cache_pagesection", 20 | "cf_cache_pagesection_tags", 21 | "cf_cache_pages_tags", 22 | "cf_cache_rootline", 23 | "cf_cache_rootline_tags", 24 | "cf_extbase_datamapfactory_datamap", 25 | "cf_extbase_datamapfactory_datamap_tags", 26 | "cf_extbase_object", 27 | "cf_extbase_object_tags", 28 | "cf_extbase_reflection", 29 | "cf_extbase_reflection_tags" 30 | ] 31 | } -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | 3 | language: python 4 | 5 | cache: pip 6 | 7 | matrix: 8 | allow_failures: 9 | - python: "nightly" 10 | 11 | jobs: 12 | include: 13 | - name: "Python 3.6.0 on Xenial Linux" 14 | python: 3.6 15 | - name: "Python 3.7.0 on Xenial Linux" 16 | python: 3.7 17 | - name: "Python 3.8.0 on Xenial Linux" 18 | python: 3.8 19 | - name: "Python 3.7.4 on macOS" 20 | os: osx 21 | osx_image: xcode11.2 22 | language: shell 23 | - name: "Python 3.8.0 on Windows" 24 | os: windows 25 | language: shell 26 | before_install: 27 | - choco install python --version 3.8.0 28 | - python -m pip install --upgrade pip 29 | env: PATH=/c/Python38:/c/Python38/Scripts:$PATH 30 | 31 | before_install: 32 | - python --version 33 | - pip install -U pip 34 | - pip install -U pytest 35 | 36 | install: 37 | - pip install pipenv --upgrade-strategy=only-if-needed 38 | - pipenv install --dev 39 | 40 | script: 41 | - python3 db_sync_tool || python db_sync_tool -------------------------------------------------------------------------------- /db_sync_tool/utility/log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Log script 6 | """ 7 | 8 | import logging 9 | from db_sync_tool.utility import system 10 | 11 | # 12 | # GLOBALS 13 | # 14 | 15 | logger = None 16 | 17 | 18 | # 19 | # FUNCTIONS 20 | # 21 | 22 | 23 | def init_logger(): 24 | """ 25 | Initialize the logger instance 26 | :return: 27 | """ 28 | global logger 29 | logger = logging.getLogger('db_sync_tool') 30 | logger.setLevel(logging.DEBUG) 31 | 32 | if system.config: 33 | if 'log_file' in system.config: 34 | fh = logging.FileHandler(system.config['log_file']) 35 | fh.setLevel(logging.DEBUG) 36 | logger.addHandler(fh) 37 | formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 38 | fh.setFormatter(formatter) 39 | logger.addHandler(fh) 40 | 41 | 42 | def get_logger(): 43 | """ 44 | Return the logger instance 45 | :return: 46 | """ 47 | if logger is None: 48 | init_logger() 49 | return logger 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Konrad Michalik 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /db_sync_tool/recipes/laravel.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Laravel script 6 | """ 7 | from db_sync_tool.utility import mode, system, helper 8 | 9 | 10 | def check_configuration(client): 11 | """ 12 | Checking remote Laravel database configuration 13 | :param client: String 14 | :return: 15 | """ 16 | _path = system.config[client]['path'] 17 | 18 | system.config[client]['db'] = helper.clean_db_config({ 19 | 'name': get_database_parameter(client, 'DB_DATABASE', _path), 20 | 'host': get_database_parameter(client, 'DB_HOST', _path), 21 | 'password': get_database_parameter(client, 'DB_PASSWORD', _path), 22 | 'port': get_database_parameter(client, 'DB_PORT', _path), 23 | 'user': get_database_parameter(client, 'DB_USERNAME', _path), 24 | }) 25 | 26 | 27 | def get_database_parameter(client, name, file): 28 | """ 29 | Parsing a single database variable from the .env file 30 | https://gist.github.com/judy2k/7656bfe3b322d669ef75364a46327836 31 | :param client: String 32 | :param name: String 33 | :param file: String 34 | :return: 35 | """ 36 | return mode.run_command( 37 | helper.get_command(client, 'grep') + f' {name} {file} | cut -d \'=\' -f2', 38 | client, 39 | True 40 | ).replace('\n', '') 41 | -------------------------------------------------------------------------------- /docs/quickstart/DRUPAL.md: -------------------------------------------------------------------------------- 1 | # Quickstart Drupal 2 | 3 | The db-sync-tool can automatic detect the database credentials of a Drupal application. 4 | 5 | - [Drupal](https://www.drupal.org/) (>= v8.0) 6 | 7 | Therefore, you have to define the path to the desired drupal installation. The script uses `drush` to extract the database settings. See the [Drush documentation](https://www.drush.org/latest/commands/core_status/) for more information. 8 | 9 | ## Command line 10 | Example call for a Drupal sync in [receiver mode](../MODE.md): 11 | 12 | ```bash 13 | $ python3 db_sync_tool 14 | --type DRUPAL 15 | --origin-host 16 | --origin-user 17 | --origin-path 18 | --target-path 19 | ``` 20 | 21 | ## Configuration file 22 | For reusability reasons you can use an additional configuration file containing all necessary information about the sync. 23 | 24 | Command line call: 25 | ```bash 26 | $ python3 db_sync_tool 27 | --config-file 28 | ``` 29 | 30 | Example configuration file: 31 | ```json 32 | { 33 | "type": "DRUPAL", 34 | "target": { 35 | "path": "" 36 | }, 37 | "origin": { 38 | "host": "", 39 | "user": "", 40 | "path": "" 41 | } 42 | } 43 | ``` 44 | 45 | It is possible to extend the [configuration](../CONFIG.md). -------------------------------------------------------------------------------- /docs/quickstart/WORDPRESS.md: -------------------------------------------------------------------------------- 1 | # Quickstart Wordpress 2 | 3 | The db-sync-tool can automatic detect the database credentials of a Wordpress application. 4 | 5 | - [Wordpress](https://wordpress.org) (>= v5.0) 6 | 7 | Therefore, you have to define the file path to the `wp-config.php`, which contains the needed credentials. See the [Wordpress documentation](https://wordpress.org/support/article/editing-wp-config-php/) for more information. 8 | 9 | ## Command line 10 | Example call for a Drupal sync in [receiver mode](../MODE.md): 11 | 12 | ```bash 13 | $ python3 db_sync_tool 14 | --type WORDPRESS 15 | --origin-host 16 | --origin-user 17 | --origin-path 18 | --target-path 19 | ``` 20 | 21 | ## Configuration file 22 | For reusability reasons you can use an additional configuration file containing all necessary information about the sync. 23 | 24 | Command line call: 25 | ```bash 26 | $ python3 db_sync_tool 27 | --config-file 28 | ``` 29 | 30 | Example configuration file: 31 | ```json 32 | { 33 | "type": "WORDPRESS", 34 | "target": { 35 | "path": "" 36 | }, 37 | "origin": { 38 | "host": "", 39 | "user": "", 40 | "path": "" 41 | } 42 | } 43 | ``` 44 | 45 | It is possible to extend the [configuration](../CONFIG.md). -------------------------------------------------------------------------------- /db_sync_tool/remote/system.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | System script 6 | """ 7 | 8 | import sys 9 | from db_sync_tool.utility import mode, output, helper 10 | from db_sync_tool.remote import client as remote_client 11 | 12 | 13 | def run_ssh_command_by_client(client, command): 14 | """ 15 | Running origin ssh command 16 | :param client: String 17 | :param command: String 18 | :return: 19 | """ 20 | if client == mode.Client.ORIGIN: 21 | return run_ssh_command(command, remote_client.ssh_client_origin, client) 22 | else: 23 | return run_ssh_command(command, remote_client.ssh_client_target, client) 24 | 25 | 26 | def run_ssh_command(command, ssh_client=remote_client.ssh_client_origin, client=None): 27 | """ 28 | Running ssh command 29 | :param command: String 30 | :param ssh_client: 31 | :param client: String 32 | :return: 33 | """ 34 | stdin, stdout, stderr = ssh_client.exec_command(command) 35 | exit_status = stdout.channel.recv_exit_status() 36 | 37 | err = stderr.read().decode() 38 | 39 | if err and exit_status != 0: 40 | helper.run_script(client=client, script='error') 41 | sys.exit(output.message(output.Subject.ERROR, err, False)) 42 | elif err: 43 | output.message(output.Subject.WARNING, err, True) 44 | 45 | return stdout 46 | -------------------------------------------------------------------------------- /docs/quickstart/START.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | If you don't want to use the automatic database credential detection of a supported framework, you can define all needed credentials by your own. 4 | 5 | ## Command line 6 | Almost all sync features can be declared via the command line call. This is an example for a sync in [receiver mode](../MODE.md): 7 | 8 | ```bash 9 | $ python3 db_sync_tool 10 | --origin-host 11 | --origin-user 12 | --origin-db-name 13 | --origin-db-user 14 | --origin-db-password 15 | --target-db-name 16 | --target-db-user 17 | --target-db-password 18 | ``` 19 | 20 | ## Configuration file 21 | For reusability reasons you can use an additional configuration file containing all necessary information about the sync. 22 | 23 | Command line call: 24 | ```bash 25 | $ python3 db_sync_tool 26 | --config-file 27 | ``` 28 | 29 | Example configuration file: 30 | ```json 31 | { 32 | "target": { 33 | "db": { 34 | "name": "", 35 | "password": "", 36 | "user": "" 37 | } 38 | }, 39 | "origin": { 40 | "host": "", 41 | "user": "", 42 | "db": { 43 | "name": "", 44 | "password": "", 45 | "user": "" 46 | } 47 | } 48 | } 49 | ``` 50 | 51 | It is possible to extend the [configuration](../CONFIG.md). -------------------------------------------------------------------------------- /docs/quickstart/SYMFONY.md: -------------------------------------------------------------------------------- 1 | # Quickstart Symfony 2 | 3 | The db-sync-tool can automatic detect the database credentials of a Symfony application. 4 | 5 | - [Symfony](https://symfony.com/) (>= v2.8) 6 | 7 | Therefore, you have to define the file path to the database configuration file. 8 | 9 | For Symfony >= __3.4__ use the `.env` file containing the `DATABASE_URL` environment variable. See the [Doctrine documentation](https://symfony.com/doc/current/doctrine.html) for more information. 10 | 11 | For Symfony <= __2.8__ use the `parameters.yml` file containing the database parameters. See the [Doctrine documentation](https://symfony.com/doc/3.4/doctrine.html) for more information. 12 | 13 | ## Command line 14 | Example call for a Symfony sync in [receiver mode](../MODE.md): 15 | 16 | ```bash 17 | $ python3 db_sync_tool 18 | --type SYMFONY 19 | --origin-host 20 | --origin-user 21 | --origin-path 22 | --target-path 23 | ``` 24 | 25 | ## Configuration file 26 | For reusability reasons you can use an additional configuration file containing all necessary information about the sync. 27 | 28 | Command line call: 29 | ```bash 30 | $ python3 db_sync_tool 31 | --config-file 32 | ``` 33 | 34 | Example configuration file: 35 | ```json 36 | { 37 | "type": "SYMFONY", 38 | "target": { 39 | "path": "" 40 | }, 41 | "origin": { 42 | "host": "", 43 | "user": "", 44 | "path": "" 45 | } 46 | } 47 | ``` 48 | 49 | It is possible to extend the [configuration](../CONFIG.md). -------------------------------------------------------------------------------- /docs/RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Guide 2 | 3 | To release a new version of the script, the following steps are necessary: 4 | 5 | 1. Update the [Changelog](../CHANGELOG.md) 6 | 2. Increase the application version according the [Semantic Versioning](https://semver.org/spec/v2.0.0.html) 7 | 8 | - Update the `__version__` in [`db_sync_tool/info.py`](../db_sync_tool/info.py) 9 | - Update the `"version"` in [`composer.json`](../composer.json) 10 | 3. Generate a new distribution archive (see [python.org](https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives)) 11 | 12 | - _Optionally_: install the latest version of `setuptools` and `wheel`: 13 | ```bash 14 | $ python3 -m pip install --user --upgrade setuptools wheel 15 | ``` 16 | - Generate the archive: 17 | ```bash 18 | $ python3 setup.py sdist bdist_wheel 19 | ``` 20 | 4. Upload the distribution archive to [pypi.org](https://pypi.org/) (see [python.org](https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives)) 21 | 22 | - _Optionally_: install `twine` for the upload: 23 | ```bash 24 | $ python3 -m pip install --user --upgrade twine pkginfo 25 | ``` 26 | - Upload the archive: 27 | ```bash 28 | $ python3 -m twine upload dist/* 29 | ``` 30 | 5. Create a new Git Tag with the new version 31 | 32 | ```bash 33 | $ git tag v1.4 34 | ``` 35 | 6. Push the commit to the [github repository](https://github.com/jackd248/db-sync-tool) 36 | 7. The package is now available via [pypi.org](https://pypi.org/project/db-sync-tool-kmi/) and [packagist](https://packagist.org/packages/kmi/db-sync-tool) -------------------------------------------------------------------------------- /db_sync_tool/recipes/wordpress.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Wordpress script 6 | """ 7 | 8 | from db_sync_tool.utility import mode, system, helper 9 | 10 | 11 | def check_configuration(client): 12 | """ 13 | Checking Drupal database configuration 14 | :param client: String 15 | :return: 16 | """ 17 | _path = system.config[client]['path'] 18 | 19 | _db_config = { 20 | 'name': get_database_setting(client, 'DB_NAME', system.config[client]['path']), 21 | 'host': get_database_setting(client, 'DB_HOST', system.config[client]['path']), 22 | 'password': get_database_setting(client, 'DB_PASSWORD', system.config[client]['path']), 23 | 'port': get_database_setting(client, 'DB_PORT', system.config[client]['path']) 24 | if get_database_setting(client, 'DB_PORT', system.config[client]['path']) != '' else 3306, 25 | 'user': get_database_setting(client, 'DB_USER', system.config[client]['path']), 26 | } 27 | 28 | system.config[client]['db'] = helper.clean_db_config(_db_config) 29 | 30 | 31 | def get_database_setting(client, name, file): 32 | """ 33 | Parsing a single database variable from the wp-config.php file 34 | https://stackoverflow.com/questions/63493645/extract-database-name-from-a-wp-config-php-file 35 | :param client: String 36 | :param name: String 37 | :param file: String 38 | :return: 39 | """ 40 | return mode.run_command( 41 | helper.get_command(client, 'sed') + 42 | f' -n "s/define( *\'{name}\', *\'\([^\']*\)\'.*/\\1/p" {file}', 43 | client, 44 | True 45 | ).replace('\n', '') 46 | -------------------------------------------------------------------------------- /db_sync_tool/recipes/drupal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Drupal script 6 | """ 7 | 8 | import json 9 | from db_sync_tool.utility import mode, system, helper, output 10 | 11 | 12 | def check_configuration(client): 13 | """ 14 | Checking Drupal database configuration with Drush 15 | :param client: String 16 | :return: 17 | """ 18 | _path = system.config[client]['path'] 19 | 20 | # Check Drush version 21 | _raw_version = mode.run_command( 22 | f'{helper.get_command(client, "drush")} status --fields=drush-version --format=string ' 23 | f'-r {_path}', 24 | client, 25 | True 26 | ) 27 | 28 | output.message( 29 | output.host_to_subject(client), 30 | f'Drush version: {_raw_version}', 31 | True 32 | ) 33 | 34 | stdout = mode.run_command( 35 | f'{helper.get_command(client, "drush")} core-status --pipe ' 36 | f'--fields=db-hostname,db-username,db-password,db-name,db-port ' 37 | f'-r {_path}', 38 | client, 39 | True 40 | ) 41 | 42 | _db_config = parse_database_credentials(json.loads(stdout)) 43 | 44 | system.config[client]['db'] = helper.clean_db_config(_db_config) 45 | 46 | 47 | def parse_database_credentials(db_credentials): 48 | """ 49 | Parsing database credentials to needed format 50 | :param db_credentials: Dictionary 51 | :return: Dictionary 52 | """ 53 | _db_config = { 54 | 'name': db_credentials['db-name'], 55 | 'host': db_credentials['db-hostname'], 56 | 'password': db_credentials['db-password'], 57 | 'port': db_credentials['db-port'], 58 | 'user': db_credentials['db-username'], 59 | } 60 | 61 | return _db_config 62 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | import sys 3 | from db_sync_tool import info 4 | 5 | if sys.version_info < (3, 5): 6 | sys.exit('db_sync_tool requires Python 3.5+ to run') 7 | 8 | with open('README.md', 'r') as fh: 9 | long_description = fh.read() 10 | 11 | setuptools.setup( 12 | name='db_sync_tool-kmi', 13 | version=info.__version__, 14 | author='Konrad Michalik', 15 | author_email='support@konradmichalik.eu', 16 | description='Synchronize a database from and to host systems.', 17 | long_description=long_description, 18 | long_description_content_type='text/markdown', 19 | url=info.__homepage__, 20 | license='MIT', 21 | packages=setuptools.find_packages(), 22 | classifiers=[ 23 | 'Programming Language :: Python :: 3.5', 24 | 'Programming Language :: Python :: 3.6', 25 | 'Programming Language :: Python :: 3.7', 26 | 'Programming Language :: Python :: 3.8', 27 | 'Programming Language :: Python :: 3.9', 28 | 'Development Status :: 5 - Production/Stable', 29 | 'License :: OSI Approved :: MIT License', 30 | 'Operating System :: MacOS :: MacOS X', 31 | 'Operating System :: Microsoft :: Windows', 32 | 'Operating System :: POSIX', 33 | 'Topic :: Database', 34 | 'Intended Audience :: Developers' 35 | ], 36 | python_requires='>=3.5', 37 | install_requires=[ 38 | "paramiko>=2.11", 39 | "future-fstrings>=1.2", 40 | "pyyaml>=6.0", 41 | "jsonschema>=4.2.1", 42 | "requests>=2.26.0", 43 | "semantic_version>=2.8.5", 44 | "yaspin>=2.1" 45 | ], 46 | entry_points={ 47 | 'console_scripts': [ 48 | 'db_sync_tool = db_sync_tool.__main__:main' 49 | ] 50 | }, 51 | ) 52 | -------------------------------------------------------------------------------- /db_sync_tool/sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | """ 4 | Sync script 5 | """ 6 | 7 | from db_sync_tool.utility import system, helper, info 8 | from db_sync_tool.database import process 9 | from db_sync_tool.remote import transfer, client as remote_client 10 | 11 | 12 | class Sync: 13 | """ 14 | Synchronize a target database from an origin system 15 | """ 16 | 17 | def __init__(self, 18 | config_file=None, 19 | verbose=False, 20 | yes=False, 21 | mute=False, 22 | dry_run=False, 23 | import_file=None, 24 | dump_name=None, 25 | keep_dump=None, 26 | host_file=None, 27 | clear=False, 28 | force_password=False, 29 | use_rsync=False, 30 | use_rsync_options=None, 31 | reverse=False, 32 | config=None, 33 | args=None): 34 | """ 35 | Initialization 36 | :param config_file: 37 | :param verbose: 38 | :param yes: 39 | :param mute: 40 | :param dry_run: 41 | :param import_file: 42 | :param dump_name: 43 | :param keep_dump: 44 | :param host_file: 45 | :param clear: 46 | :param force_password: 47 | :param use_rsync: 48 | :param use_rsync_options: 49 | :param reverse: 50 | :param config: 51 | :param args: 52 | """ 53 | if config is None: 54 | config = {} 55 | 56 | info.print_header(mute) 57 | system.check_args_options( 58 | config_file, 59 | verbose, 60 | yes, 61 | mute, 62 | dry_run, 63 | import_file, 64 | dump_name, 65 | keep_dump, 66 | host_file, 67 | clear, 68 | force_password, 69 | use_rsync, 70 | use_rsync_options, 71 | reverse 72 | ) 73 | system.get_configuration(config, args) 74 | system.check_authorizations() 75 | process.create_origin_database_dump() 76 | transfer.transfer_origin_database_dump() 77 | process.import_database_dump() 78 | helper.clean_up() 79 | remote_client.close_ssh_clients() 80 | info.print_footer() 81 | -------------------------------------------------------------------------------- /db_sync_tool/recipes/symfony.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Symfony script 6 | """ 7 | 8 | import re 9 | import sys 10 | 11 | from db_sync_tool.utility import mode, system, helper, output 12 | 13 | 14 | def check_configuration(client): 15 | """ 16 | Checking remote Symfony database configuration 17 | :param client: String 18 | :return: 19 | """ 20 | _path = system.config[client]['path'] 21 | 22 | # Check for symfony 2.8 23 | if 'parameters.yml' in _path: 24 | _db_config = { 25 | 'name': get_database_parameter(client, 'database_name', _path), 26 | 'host': get_database_parameter(client, 'database_host', _path), 27 | 'password': get_database_parameter(client, 'database_password', _path), 28 | 'port': get_database_parameter(client, 'database_port', _path), 29 | 'user': get_database_parameter(client, 'database_user', _path), 30 | } 31 | # Using for symfony >=3.4 32 | else: 33 | stdout = mode.run_command( 34 | helper.get_command(client, 'grep') + ' -v "^#" ' + system.config[client][ 35 | 'path'] + ' | ' + helper.get_command(client, 'grep') + ' DATABASE_URL', 36 | client, 37 | True 38 | ) 39 | _db_config = parse_database_credentials(stdout) 40 | 41 | system.config[client]['db'] = helper.clean_db_config(_db_config) 42 | 43 | 44 | def parse_database_credentials(db_credentials): 45 | """ 46 | Parsing database credentials to needed format 47 | :param db_credentials: Dictionary 48 | :return: Dictionary 49 | """ 50 | db_credentials = str(db_credentials).replace('\\n\'','') 51 | # DATABASE_URL=mysql://db-user:1234@db-host:3306/db-name 52 | pattern = r'^DATABASE_URL=(?P\w+):\/\/(?P[^:]+):(?P[^@]+)@(?P[^:]+):(?P\d+)\/(?P[^?]+)(?:\?.*)?$' 53 | 54 | match = re.match(pattern, db_credentials) 55 | 56 | if match: 57 | db_config = match.groupdict() 58 | return db_config 59 | else: 60 | sys.exit( 61 | output.message( 62 | output.Subject.ERROR, 63 | 'Mismatch of expected database credentials', 64 | False 65 | ) 66 | ) 67 | 68 | 69 | def get_database_parameter(client, name, file): 70 | """ 71 | Parsing a single database variable from the parameters.yml file 72 | hhttps://unix.stackexchange.com/questions/84922/extract-a-part-of-one-line-from-a-file-with-sed 73 | :param client: String 74 | :param name: String 75 | :param file: String 76 | :return: 77 | """ 78 | return mode.run_command( 79 | helper.get_command(client, 'sed') + f' -n -e \'/{name}/ s/.*\\: *//p\' {file}', 80 | client, 81 | True 82 | ).replace('\n', '') 83 | -------------------------------------------------------------------------------- /docs/quickstart/TYPO3.md: -------------------------------------------------------------------------------- 1 | # Quickstart TYPO3 2 | 3 | The db-sync-tool can automatically detect the database credentials of a TYPO3 application. 4 | 5 | - [TYPO3](https://typo3.org/) (>= v7.6) 6 | 7 | Therefore, you have to define the file path to the `LocalConfiguration.php`, which contains the needed credentials. See the [TYPO3 documentation](https://docs.typo3.org/m/typo3/reference-coreapi/10.4/en-us/ApiOverview/GlobalValues/Typo3ConfVars/Index.html) for more information. 8 | 9 | ## Command line 10 | Example call for a TYPO3 sync in [receiver mode](../MODE.md): 11 | 12 | ```bash 13 | $ db_sync_tool 14 | --type TYPO3 15 | --origin-host 16 | --origin-user 17 | --origin-path 18 | --target-path 19 | ``` 20 | 21 | ## Configuration file 22 | For reusability reasons you can use an additional configuration file containing all necessary information about the sync. 23 | 24 | Command line call: 25 | ```bash 26 | $ db_sync_tool 27 | --config-file 28 | ``` 29 | 30 | The configuration file should look like: 31 | 32 | ```yaml 33 | type: TYPO3 34 | target: 35 | path: 36 | origin: 37 | host: 38 | user: 39 | path: 40 | ``` 41 | 42 | It is possible to extend the [configuration](../CONFIG.md). 43 | 44 | ## Example 45 | 46 | Here is an extended example with demo data: 47 | 48 | ```yaml 49 | type: TYPO3 50 | target: 51 | path: /var/www/html/htdocs/typo3/web/typo3conf/LocalConfiguration.php 52 | origin: 53 | host: 192.87.33.123 54 | user: ssh_demo_user 55 | path: /var/www/html/shared/typo3conf/LocalConfiguration.php 56 | name: Demo Prod 57 | ignore_table: 58 | - be_users 59 | - sys_domain 60 | - cf_cache_* 61 | ``` 62 | 63 | ## .env support 64 | 65 | Alternatively, the credentials can be parsed out of an `.env` file. If the `path` configuration points to a `.env` file, the db-sync-tool will try to parse the database credentials from it. The `.env` file should contain the following variables: 66 | 67 | ```dotenv 68 | # .env Database Default Configuration Keys 69 | TYPO3_CONF_VARS__DB__Connections__Default__host=db 70 | TYPO3_CONF_VARS__DB__Connections__Default__port=3306 71 | TYPO3_CONF_VARS__DB__Connections__Default__password=db 72 | TYPO3_CONF_VARS__DB__Connections__Default__user=db 73 | TYPO3_CONF_VARS__DB__Connections__Default__dbname=db 74 | ``` 75 | 76 | If the `.env` file contains different keys for the database credentials, you can specify them in the configuration file: 77 | 78 | ```yaml 79 | type: TYPO3 80 | target: 81 | path: /var/www/html/.env 82 | origin: 83 | name: Demo Prod 84 | host: 123.456.78.90 85 | user: ssh_demo_user 86 | path: /var/www/html/shared/.env 87 | db: 88 | name: TYPO3_DB_NAME 89 | host: TYPO3_DB_HOST 90 | user: TYPO3_DB_USER 91 | password: TYPO3_DB_PASSWORD 92 | ignore_table: 93 | - cf_cache_* 94 | ``` 95 | -------------------------------------------------------------------------------- /db_sync_tool/utility/info.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | 6 | """ 7 | import requests 8 | import semantic_version 9 | import random 10 | from db_sync_tool.utility import mode, system, output 11 | from db_sync_tool import info 12 | 13 | 14 | def print_header(mute): 15 | """ 16 | Printing console header 17 | :param mute: Boolean 18 | :return: 19 | """ 20 | # pylint: max-line-length=240 21 | if mute is False: 22 | _colors = get_random_colors() 23 | print( 24 | output.CliFormat.BLACK + '##############################################' + output.CliFormat.ENDC) 25 | print( 26 | output.CliFormat.BLACK + '# #' + output.CliFormat.ENDC) 27 | print( 28 | output.CliFormat.BLACK + '#' + output.CliFormat.ENDC + ' ' + _colors[0] + '⥣ ' + _colors[1] + '⥥ ' + output.CliFormat.ENDC + ' db sync tool ' + output.CliFormat.BLACK + '#' + output.CliFormat.ENDC) 29 | print( 30 | output.CliFormat.BLACK + '# v' + info.__version__ + ' #' + output.CliFormat.ENDC) 31 | print(output.CliFormat.BLACK + '# ' + info.__homepage__ + ' #' + output.CliFormat.ENDC) 32 | print( 33 | output.CliFormat.BLACK + '# #' + output.CliFormat.ENDC) 34 | print( 35 | output.CliFormat.BLACK + '##############################################' + output.CliFormat.ENDC) 36 | check_updates() 37 | 38 | 39 | def check_updates(): 40 | """ 41 | Check for updates of the db_sync_tool 42 | :return: 43 | """ 44 | try: 45 | response = requests.get(f'{info.__pypi_package_url__}/json') 46 | latest_version = response.json()['info']['version'] 47 | if semantic_version.Version(info.__version__) < semantic_version.Version(latest_version): 48 | output.message( 49 | output.Subject.WARNING, 50 | f'A new version {output.CliFormat.BOLD}v{latest_version}{output.CliFormat.ENDC} is ' 51 | f'available for the db-sync-tool: {info.__pypi_package_url__}', 52 | True 53 | ) 54 | finally: 55 | return 56 | 57 | 58 | def print_footer(): 59 | """ 60 | Printing console footer 61 | :return: 62 | """ 63 | if system.config['dry_run']: 64 | _message = 'Successfully executed dry run' 65 | elif not system.config['keep_dump'] and \ 66 | not system.config['is_same_client'] and \ 67 | not mode.is_import(): 68 | _message = 'Successfully synchronized databases' 69 | elif mode.is_import(): 70 | _message = 'Successfully imported database dump' 71 | else: 72 | _message = 'Successfully created database dump' 73 | 74 | output.message( 75 | output.Subject.INFO, 76 | _message, 77 | True, 78 | True 79 | ) 80 | 81 | 82 | def get_random_colors(): 83 | """ 84 | Generate a tuple of random console colors 85 | :return: 86 | """ 87 | _colors = [output.CliFormat.BEIGE, output.CliFormat.PURPLE, output.CliFormat.BLUE, output.CliFormat.YELLOW, output.CliFormat.GREEN, output.CliFormat.RED] 88 | return random.sample(_colors, 2) 89 | 90 | -------------------------------------------------------------------------------- /db_sync_tool/utility/validation.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Validation script 6 | """ 7 | 8 | import sys 9 | from jsonschema import validators 10 | from db_sync_tool.utility import output 11 | 12 | # 13 | # GLOBALS 14 | # 15 | schema = { 16 | "type": "object", 17 | "properties": { 18 | "type": {"enum": ['TYPO3', 'Symfony', 'Drupal', 'Wordpress', 'Laravel']}, 19 | "log_file": {"type": "string"}, 20 | "ignore_table": {"type": "array"}, 21 | "target": { 22 | "type": "object", 23 | "properties": { 24 | "name": {"type": "string"}, 25 | "host": {"type": "string", "format": "hostname"}, 26 | "user": {"type": "string"}, 27 | "password": {"type": "string"}, 28 | "path": {"type": "string"}, 29 | "ssh_key": {"type": "string"}, 30 | "port": {"type": "number"}, 31 | "dump_dir": {"type": "string"}, 32 | "after_dump": {"type": "string"}, 33 | "db": { 34 | "type": "object", 35 | "properties": { 36 | "name": {"type": "string"}, 37 | "host": {"type": "string", "format": "hostname"}, 38 | "user": {"type": "string"}, 39 | "password": {"type": "string"}, 40 | "port": {"type": "number"}, 41 | } 42 | }, 43 | "script": { 44 | "type": "object", 45 | "properties": { 46 | "before": {"type": "string"}, 47 | "after": {"type": "string"}, 48 | "error": {"type": "string"}, 49 | } 50 | } 51 | } 52 | }, 53 | "origin": { 54 | "type": "object", 55 | "properties": { 56 | "name": {"type": "string"}, 57 | "host": {"type": "string", "format": "hostname"}, 58 | "user": {"type": "string"}, 59 | "password": {"type": "string"}, 60 | "path": {"type": "string"}, 61 | "ssh_key": {"type": "string"}, 62 | "port": {"type": "number"}, 63 | "dump_dir": {"type": "string"}, 64 | "after_dump": {"type": "string"}, 65 | "db": { 66 | "type": "object", 67 | "properties": { 68 | "name": {"type": "string"}, 69 | "host": {"type": "string", "format": "hostname"}, 70 | "user": {"type": "string"}, 71 | "password": {"type": "string"}, 72 | "port": {"type": "number"}, 73 | } 74 | }, 75 | "script": { 76 | "type": "object", 77 | "properties": { 78 | "before": {"type": "string"}, 79 | "after": {"type": "string"}, 80 | "error": {"type": "string"}, 81 | } 82 | } 83 | } 84 | }, 85 | }, 86 | } 87 | 88 | 89 | # 90 | # FUNCTIONS 91 | # 92 | 93 | 94 | def check(config): 95 | output.message( 96 | output.Subject.LOCAL, 97 | 'Validating configuration', 98 | True 99 | ) 100 | v = validators.Draft7Validator(schema) 101 | errors = sorted(v.iter_errors(config), key=lambda e: e.path) 102 | 103 | for error in errors: 104 | output.message( 105 | output.Subject.ERROR, 106 | f'{error.message}', 107 | True 108 | ) 109 | if errors: 110 | sys.exit( 111 | output.message( 112 | output.Subject.ERROR, 113 | 'Validation error(s)', 114 | do_print=False 115 | ) 116 | ) 117 | -------------------------------------------------------------------------------- /db_sync_tool/remote/utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Utility script 6 | """ 7 | 8 | import os 9 | import paramiko 10 | from db_sync_tool.utility import mode, system, helper, output 11 | from db_sync_tool.database import utility as database_utility 12 | from db_sync_tool.remote import client as remote_client 13 | 14 | 15 | def remove_origin_database_dump(keep_compressed_file=False): 16 | """ 17 | Removing the origin database dump files 18 | :param keep_compressed_file: Boolean 19 | :return: 20 | """ 21 | output.message( 22 | output.Subject.ORIGIN, 23 | 'Cleaning up', 24 | True 25 | ) 26 | 27 | if system.config['dry_run']: 28 | return 29 | 30 | _file_path = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name 31 | if mode.is_origin_remote(): 32 | mode.run_command( 33 | helper.get_command(mode.Client.ORIGIN, 'rm') + ' ' + _file_path, 34 | mode.Client.ORIGIN 35 | ) 36 | if not keep_compressed_file: 37 | mode.run_command( 38 | helper.get_command(mode.Client.ORIGIN, 'rm') + ' ' + _file_path + '.tar.gz', 39 | mode.Client.ORIGIN 40 | ) 41 | else: 42 | os.remove(_file_path) 43 | if not keep_compressed_file: 44 | os.remove(f'{_file_path}.tar.gz') 45 | 46 | if keep_compressed_file: 47 | if 'keep_dumps' in system.config[mode.Client.ORIGIN]: 48 | helper.clean_up_dump_dir(mode.Client.ORIGIN, 49 | helper.get_dump_dir(mode.Client.ORIGIN) + '*', 50 | system.config[mode.Client.ORIGIN]['keep_dumps']) 51 | 52 | output.message( 53 | output.Subject.INFO, 54 | f'Database dump file is saved to: {_file_path}.tar.gz', 55 | True, 56 | True 57 | ) 58 | 59 | 60 | def remove_target_database_dump(): 61 | """ 62 | Removing the target database dump files 63 | :return: 64 | """ 65 | _file_path = helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name 66 | 67 | # 68 | # Move dump to specified directory 69 | # 70 | if system.config['keep_dump']: 71 | helper.create_local_temporary_data_dir() 72 | _keep_dump_path = system.default_local_sync_path + database_utility.database_dump_file_name 73 | mode.run_command( 74 | helper.get_command('target', 75 | 'cp') + ' ' + _file_path + ' ' + _keep_dump_path, 76 | mode.Client.TARGET 77 | ) 78 | output.message( 79 | output.Subject.INFO, 80 | f'Database dump file is saved to: {_keep_dump_path}', 81 | True, 82 | True 83 | ) 84 | 85 | # 86 | # Clean up 87 | # 88 | if not mode.is_dump() and not mode.is_import(): 89 | output.message( 90 | output.Subject.TARGET, 91 | 'Cleaning up', 92 | True 93 | ) 94 | 95 | if system.config['dry_run']: 96 | return 97 | 98 | if mode.is_target_remote(): 99 | mode.run_command( 100 | helper.get_command(mode.Client.TARGET, 'rm') + ' ' + _file_path, 101 | mode.Client.TARGET 102 | ) 103 | mode.run_command( 104 | helper.get_command(mode.Client.TARGET, 'rm') + ' ' + _file_path + '.tar.gz', 105 | mode.Client.TARGET 106 | ) 107 | else: 108 | if os.path.isfile(_file_path): 109 | os.remove(_file_path) 110 | if os.path.isfile(f'{_file_path}.tar.gz'): 111 | os.remove(f'{_file_path}.tar.gz') 112 | 113 | 114 | def check_keys_from_ssh_agent(): 115 | """ 116 | Check if private keys are available from an SSH agent. 117 | :return: 118 | """ 119 | agent = paramiko.Agent() 120 | agent_keys = agent.get_keys() 121 | if len(agent_keys) == 0: 122 | return False 123 | return True 124 | -------------------------------------------------------------------------------- /db_sync_tool/remote/rsync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | rsync script 6 | """ 7 | 8 | import re 9 | from db_sync_tool.utility import mode, system, output 10 | 11 | # Default options for rsync command 12 | # https://wiki.ubuntuusers.de/rsync/ 13 | default_options = [ 14 | '--delete', 15 | '-a', 16 | '-z', 17 | '--stats', 18 | '--human-readable', 19 | '--iconv=UTF-8', 20 | '--chmod=D2770,F660' 21 | ] 22 | 23 | 24 | def get_password_environment(client): 25 | """ 26 | Optionally create a password environment variable for sshpass password authentication 27 | https://www.redhat.com/sysadmin/ssh-automation-sshpass 28 | :param client: String 29 | :return: 30 | """ 31 | if not client: 32 | return '' 33 | 34 | if system.config['use_sshpass'] and not 'ssh_key' in system.config[client] and 'password' in system.config[client]: 35 | return f'SSHPASS=\'{system.config[client]["password"]}\' ' 36 | return '' 37 | 38 | 39 | def get_authorization(client): 40 | """ 41 | Define authorization arguments for rsync command 42 | :param client: String 43 | :return: String 44 | """ 45 | _ssh_key = None 46 | if not client: 47 | return '' 48 | 49 | if 'ssh_key' in system.config[client]: 50 | _ssh_key = system.config[mode.Client.ORIGIN]['ssh_key'] 51 | 52 | _ssh_port = system.config[client]['port'] if 'port' in system.config[client] else 22 53 | 54 | if _ssh_key is None: 55 | if system.config['use_sshpass'] and get_password_environment(client): 56 | # In combination with SSHPASS environment variable 57 | # https://www.redhat.com/sysadmin/ssh-automation-sshpass 58 | return f'--rsh="sshpass -e ssh -p{_ssh_port} -o StrictHostKeyChecking=no -l {system.config[client]["user"]}"' 59 | else: 60 | return f'-e "ssh -p{_ssh_port} -o StrictHostKeyChecking=no"' 61 | else: 62 | # Provide ssh key file path for ssh authentication 63 | return f'-e "ssh -i {_ssh_key} -p{_ssh_port}"' 64 | 65 | 66 | def get_host(client): 67 | """ 68 | Return user@host if client is not local 69 | :param client: String 70 | :return: String 71 | """ 72 | if mode.is_remote(client): 73 | return f'{system.config[client]["user"]}@{system.config[client]["host"]}:' 74 | return '' 75 | 76 | 77 | def get_options(): 78 | """ 79 | Prepare rsync options with stored default options and provided addtional options 80 | :return: String 81 | """ 82 | _options = f'{" ".join(default_options)}' 83 | if not system.config['use_rsync_options'] is None: 84 | _options += f'{system.config["use_rsync_options"]}' 85 | return _options 86 | 87 | 88 | def read_stats(stats): 89 | """ 90 | Read rsync stats and print a summary 91 | :param stats: String 92 | :return: 93 | """ 94 | if system.config['verbose']: 95 | print(f'{output.Subject.DEBUG}{output.CliFormat.BLACK}{stats}{output.CliFormat.ENDC}') 96 | 97 | _file_size = parse_string(stats, r'Total transferred file size:\s*([\d.]+[MKG]?)') 98 | 99 | if _file_size: 100 | output.message( 101 | output.Subject.INFO, 102 | f'Status: {unit_converter(_file_size[0])} transferred' 103 | ) 104 | 105 | 106 | def parse_string(string, regex): 107 | """ 108 | Parse string by given regex 109 | :param string: String 110 | :param regex: String 111 | :return: 112 | """ 113 | _file_size_pattern = regex 114 | _regex_matcher = re.compile(_file_size_pattern) 115 | return _regex_matcher.findall(string) 116 | 117 | 118 | def unit_converter(size_in_bytes): 119 | """ 120 | 121 | :param size_in_bytes: 122 | :return: 123 | """ 124 | units = ['Bytes', 'kB', 'MB', 'GB'] 125 | 126 | if isinstance(size_in_bytes, (int, float)): 127 | _convertedSize = float(size_in_bytes) 128 | for unit in units: 129 | if _convertedSize < 1024: 130 | return str(_convertedSize) + ' ' + unit 131 | _convertedSize = _convertedSize / 1024 132 | 133 | return _convertedSize 134 | return size_in_bytes 135 | 136 | 137 | def run_rsync_command(remote_client, origin_path, target_path, origin_ssh='', target_ssh=''): 138 | """ 139 | 140 | :param target_ssh: 141 | :param origin_ssh: 142 | :param target_path: 143 | :param origin_path: 144 | :param remote_client: 145 | :return: 146 | """ 147 | if origin_ssh != '': 148 | origin_ssh += ':' 149 | if target_ssh != '': 150 | target_ssh += ':' 151 | 152 | _output = mode.run_command( 153 | f'{get_password_environment(remote_client)}rsync {get_options()} ' 154 | f'{get_authorization(remote_client)} ' 155 | f'{origin_ssh}{origin_path} {target_ssh}{target_path}', 156 | mode.Client.LOCAL, 157 | True 158 | ) 159 | read_stats(_output) 160 | -------------------------------------------------------------------------------- /db_sync_tool/recipes/typo3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | TYPO3 script 6 | """ 7 | 8 | import json, sys 9 | 10 | from db_sync_tool.utility import mode, system, helper, output 11 | 12 | 13 | def check_configuration(client): 14 | """ 15 | Checking remote TYPO3 database configuration 16 | :param client: String 17 | :return: 18 | """ 19 | _path = system.config[client]['path'] 20 | 21 | if 'LocalConfiguration' in _path: 22 | stdout = mode.run_command( 23 | helper.get_command(client, 'php') + ' -r "echo json_encode(include \'' + 24 | system.config[client][ 25 | 'path'] + '\');"', 26 | client, 27 | True 28 | ) 29 | 30 | _db_config = parse_database_credentials(json.loads(stdout)['DB']) 31 | elif '.env' in _path: 32 | # Try to parse settings from .env file 33 | if 'db' not in system.config[client]: 34 | system.config[client]['db'] = {} 35 | 36 | _db_config = { 37 | 'name': get_database_setting_from_env(client, system.config[client]['db'].get('name', 'TYPO3_CONF_VARS__DB__Connections__Default__dbname'), system.config[client]['path']), 38 | 'host': get_database_setting_from_env(client, system.config[client]['db'].get('host', 'TYPO3_CONF_VARS__DB__Connections__Default__host'), system.config[client]['path']), 39 | 'password': get_database_setting_from_env(client, system.config[client]['db'].get('password', 'TYPO3_CONF_VARS__DB__Connections__Default__password'), system.config[client]['path']), 40 | 'port': get_database_setting_from_env(client, system.config[client]['db'].get('port', 'TYPO3_CONF_VARS__DB__Connections__Default__port'), system.config[client]['path']) 41 | if get_database_setting_from_env(client, system.config[client]['db'].get('port', 'TYPO3_CONF_VARS__DB__Connections__Default__port'), system.config[client]['path']) != '' else 3306, 42 | 'user': get_database_setting_from_env(client, system.config[client]['db'].get('user', 'TYPO3_CONF_VARS__DB__Connections__Default__user'), system.config[client]['path']), 43 | } 44 | elif 'AdditionalConfiguration.php' in _path: 45 | # Try to parse settings from AdditionalConfiguration.php file 46 | _db_config = { 47 | 'name': get_database_setting_from_additional_configuration(client, 'dbname', system.config[client]['path']), 48 | 'host': get_database_setting_from_additional_configuration(client, 'host', system.config[client]['path']), 49 | 'password': get_database_setting_from_additional_configuration(client, 'password', system.config[client]['path']), 50 | 'port': get_database_setting_from_additional_configuration(client, 'port', system.config[client]['path']) 51 | if get_database_setting_from_additional_configuration(client, 'port', 52 | system.config[client]['path']) != '' else 3306, 53 | 'user': get_database_setting_from_additional_configuration(client, 'user', system.config[client]['path']), 54 | } 55 | else: 56 | sys.exit( 57 | output.message( 58 | output.Subject.ERROR, 59 | f'Can\'t extract database information from given path {system.config[client]["path"]}. Can only extract settings from the following files: LocalConfiguration.php, AdditionalConfiguration.php, .env', 60 | False 61 | ) 62 | ) 63 | 64 | system.config[client]['db'] = helper.clean_db_config(_db_config) 65 | 66 | 67 | def parse_database_credentials(db_credentials): 68 | """ 69 | Parsing database credentials to needed format 70 | :param db_credentials: Dictionary 71 | :return: Dictionary 72 | """ 73 | # 74 | # Distinguish between database config scheme of TYPO3 v8+ and TYPO3 v7- 75 | # 76 | if 'Connections' in db_credentials: 77 | _db_config = db_credentials['Connections']['Default'] 78 | _db_config['name'] = _db_config['dbname'] 79 | else: 80 | _db_config = db_credentials 81 | _db_config['user'] = _db_config['username'] 82 | _db_config['name'] = _db_config['database'] 83 | 84 | if 'port' not in _db_config: 85 | _db_config['port'] = 3306 86 | 87 | return _db_config 88 | 89 | 90 | def get_database_setting_from_additional_configuration(client, name, file): 91 | """ 92 | Get database setting try to regex from AdditionalConfiguration 93 | sed -nE "s/'dbname'.*=>.*'(.*)'.*$/\1/p" /var/www/html/tests/files/www1/AdditionalConfiguration.php 94 | :param client: String 95 | :param name: String 96 | :param file: String 97 | :return: 98 | """ 99 | return helper.run_sed_command(client, f'"s/\'{name}\'.*=>.*\'(.*)\'.*$/\\1/p" {file}') 100 | 101 | def get_database_setting_from_env(client, name, file): 102 | """ 103 | Get database setting try to regex from .env 104 | sed -nE "s/TYPO3_CONF_VARS__DB__Connections__Default__host=(.*).*$/\1/p" /var/www/html/tests/files/www1/typo3.env 105 | :param client: String 106 | :param name: String 107 | :param file: String 108 | :return: 109 | """ 110 | return helper.run_sed_command(client, f'"s/{name}=(.*).*$/\\1/p" {file}') -------------------------------------------------------------------------------- /db_sync_tool/utility/output.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Output script 6 | """ 7 | import time 8 | from yaspin import yaspin 9 | from db_sync_tool.utility import log, mode, system 10 | 11 | 12 | class CliFormat: 13 | BEIGE = '\033[96m' 14 | PURPLE = '\033[95m' 15 | BLUE = '\033[94m' 16 | YELLOW = '\033[93m' 17 | GREEN = '\033[92m' 18 | RED = '\033[91m' 19 | BLACK = '\033[90m' 20 | ENDC = '\033[0m' 21 | BOLD = '\033[1m' 22 | UNDERLINE = '\033[4m' 23 | 24 | 25 | class Subject: 26 | INFO = CliFormat.GREEN + '[INFO]' + CliFormat.ENDC 27 | LOCAL = CliFormat.BEIGE + '[LOCAL]' + CliFormat.ENDC 28 | TARGET = CliFormat.BLUE + '[TARGET]' + CliFormat.ENDC 29 | ORIGIN = CliFormat.PURPLE + '[ORIGIN]' + CliFormat.ENDC 30 | ERROR = CliFormat.RED + '[ERROR]' + CliFormat.ENDC 31 | WARNING = CliFormat.YELLOW + '[WARNING]' + CliFormat.ENDC 32 | DEBUG = CliFormat.BLACK + '[DEBUG]' + CliFormat.ENDC 33 | 34 | 35 | def message(header, message, do_print=True, do_log=False, debug=False, verbose_only=False): 36 | """ 37 | Formatting a message for print or log 38 | :param header: String 39 | :param message: String 40 | :param do_print: Boolean 41 | :param do_log: Boolean 42 | :param debug: Boolean 43 | :param verbose_only: Boolean 44 | :return: String message 45 | """ 46 | # Logging if explicitly forced or verbose option is active 47 | if do_log or system.config['verbose']: 48 | _message = remove_multiple_elements_from_string([CliFormat.BEIGE, 49 | CliFormat.PURPLE, 50 | CliFormat.BLUE, 51 | CliFormat.YELLOW, 52 | CliFormat.GREEN, 53 | CliFormat.RED, 54 | CliFormat.BLACK, 55 | CliFormat.ENDC, 56 | CliFormat.BOLD, 57 | CliFormat.UNDERLINE], message) 58 | # @ToDo: Can this be done better? Dynamic functions? 59 | if debug: 60 | log.get_logger().debug(_message) 61 | elif header == Subject.WARNING: 62 | log.get_logger().warning(_message) 63 | elif header == Subject.ERROR: 64 | log.get_logger().error(_message) 65 | else: 66 | log.get_logger().info(_message) 67 | 68 | # Formatting message if mute option is inactive 69 | if (system.config['mute'] and header == Subject.ERROR) or (not system.config['mute']): 70 | if do_print: 71 | if not verbose_only or (verbose_only and system.config['verbose']): 72 | _message = header + extend_output_by_sync_mode(header, debug) + ' ' + message 73 | with yaspin(text=_message, color="yellow", side="right") as spinner: 74 | spinner.ok("\b\b") 75 | else: 76 | return header + extend_output_by_sync_mode(header, debug) + ' ' + message 77 | 78 | 79 | def extend_output_by_sync_mode(header, debug=False): 80 | """ 81 | Extending the output by a client information (LOCAL|REMOTE) 82 | :param header: String 83 | :return: String message 84 | """ 85 | _sync_mode = mode.get_sync_mode() 86 | _debug = '' 87 | 88 | if debug: 89 | _debug = Subject.DEBUG 90 | 91 | if header == Subject.INFO or header == Subject.LOCAL or \ 92 | header == Subject.WARNING or header == Subject.ERROR: 93 | return '' 94 | else: 95 | if mode.is_remote(subject_to_host(header)): 96 | return CliFormat.BLACK + '[REMOTE]' + CliFormat.ENDC + _debug 97 | else: 98 | if subject_to_host(header) == mode.Client.LOCAL: 99 | return _debug 100 | else: 101 | return CliFormat.BLACK + '[LOCAL]' + CliFormat.ENDC + _debug 102 | 103 | 104 | def host_to_subject(host): 105 | """ 106 | Converting the client to the according subject 107 | :param host: String 108 | :return: String subject 109 | """ 110 | if host == mode.Client.ORIGIN: 111 | return Subject.ORIGIN 112 | elif host == mode.Client.TARGET: 113 | return Subject.TARGET 114 | elif host == mode.Client.LOCAL: 115 | return Subject.LOCAL 116 | 117 | 118 | def subject_to_host(subject): 119 | """ 120 | Converting the subject to the according host 121 | :param subject: String 122 | :return: String host 123 | """ 124 | if subject == Subject.ORIGIN: 125 | return mode.Client.ORIGIN 126 | elif subject == Subject.TARGET: 127 | return mode.Client.TARGET 128 | elif subject == Subject.LOCAL: 129 | return mode.Client.LOCAL 130 | 131 | 132 | def remove_multiple_elements_from_string(elements, string): 133 | """ 134 | Removing multiple elements from a string 135 | :param elements: List 136 | :param string: String 137 | :return: String string 138 | """ 139 | for element in elements: 140 | if element in string: 141 | string = string.replace(element, '') 142 | return string 143 | 144 | -------------------------------------------------------------------------------- /db_sync_tool/remote/transfer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Transfer script 6 | """ 7 | 8 | import sys 9 | from db_sync_tool.utility import mode, system, helper, output 10 | from db_sync_tool.database import utility as database_utility 11 | from db_sync_tool.remote import utility, client, rsync 12 | 13 | 14 | def transfer_origin_database_dump(): 15 | """ 16 | Transfer the origin database dump files 17 | :return: 18 | """ 19 | if not mode.is_import(): 20 | if mode.get_sync_mode() == mode.SyncMode.RECEIVER: 21 | get_origin_database_dump(helper.get_dump_dir(mode.Client.TARGET)) 22 | system.check_target_configuration() 23 | elif mode.get_sync_mode() == mode.SyncMode.SENDER: 24 | system.check_target_configuration() 25 | put_origin_database_dump(helper.get_dump_dir(mode.Client.ORIGIN)) 26 | utility.remove_origin_database_dump() 27 | elif mode.get_sync_mode() == mode.SyncMode.PROXY: 28 | helper.create_local_temporary_data_dir() 29 | get_origin_database_dump(system.default_local_sync_path) 30 | system.check_target_configuration() 31 | put_origin_database_dump(system.default_local_sync_path) 32 | elif mode.get_sync_mode() == mode.SyncMode.SYNC_REMOTE or mode.get_sync_mode() == mode.SyncMode.SYNC_LOCAL: 33 | system.check_target_configuration() 34 | elif system.config['is_same_client']: 35 | utility.remove_origin_database_dump(True) 36 | else: 37 | system.check_target_configuration() 38 | 39 | 40 | def get_origin_database_dump(target_path): 41 | """ 42 | Downloading the origin database dump files 43 | :param target_path: String 44 | :return: 45 | """ 46 | output.message( 47 | output.Subject.ORIGIN, 48 | 'Downloading database dump', 49 | True 50 | ) 51 | if mode.get_sync_mode() != mode.SyncMode.PROXY: 52 | helper.check_and_create_dump_dir(mode.Client.TARGET, target_path) 53 | 54 | if not system.config['dry_run']: 55 | _remotepath = helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz' 56 | _localpath = target_path 57 | 58 | if system.config['use_rsync']: 59 | rsync.run_rsync_command( 60 | remote_client=mode.Client.ORIGIN, 61 | origin_path=_remotepath, 62 | target_path=_localpath, 63 | origin_ssh=system.config[mode.Client.ORIGIN]['user'] + '@' + system.config[mode.Client.ORIGIN]['host'] 64 | ) 65 | else: 66 | # 67 | # Download speed problems 68 | # https://github.com/paramiko/paramiko/issues/60 69 | # 70 | sftp = get_sftp_client(client.ssh_client_origin) 71 | sftp.get(helper.get_dump_dir(mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz', 72 | target_path + database_utility.database_dump_file_name + '.tar.gz', download_status) 73 | sftp.close() 74 | if not system.config['mute']: 75 | print('') 76 | 77 | utility.remove_origin_database_dump() 78 | 79 | 80 | def download_status(sent, size): 81 | """ 82 | Printing the download status information 83 | :param sent: Float 84 | :param size: Float 85 | :return: 86 | """ 87 | if not system.config['mute']: 88 | sent_mb = round(float(sent) / 1024 / 1024, 1) 89 | size = round(float(size) / 1024 / 1024, 1) 90 | sys.stdout.write( 91 | output.Subject.ORIGIN + output.CliFormat.BLACK + '[REMOTE]' + output.CliFormat.ENDC + " Status: {0} MB of {1} MB downloaded". 92 | format(sent_mb, size, )) 93 | sys.stdout.write('\r') 94 | 95 | 96 | def put_origin_database_dump(origin_path): 97 | """ 98 | Uploading the origin database dump file 99 | :param origin_path: String 100 | :return: 101 | """ 102 | if mode.get_sync_mode() == mode.SyncMode.PROXY: 103 | _subject = output.Subject.LOCAL 104 | else: 105 | _subject = output.Subject.ORIGIN 106 | 107 | output.message( 108 | _subject, 109 | 'Uploading database dump', 110 | True 111 | ) 112 | helper.check_and_create_dump_dir(mode.Client.TARGET, helper.get_dump_dir(mode.Client.TARGET)) 113 | 114 | if not system.config['dry_run']: 115 | _localpath = origin_path + database_utility.database_dump_file_name + '.tar.gz' 116 | _remotepath = helper.get_dump_dir(mode.Client.TARGET) + '/' 117 | 118 | if system.config['use_rsync']: 119 | rsync.run_rsync_command( 120 | remote_client=mode.Client.TARGET, 121 | origin_path=_localpath, 122 | target_path=_remotepath, 123 | target_ssh=system.config[mode.Client.TARGET]['user'] + '@' + system.config[mode.Client.TARGET]['host'] 124 | ) 125 | else: 126 | # 127 | # Download speed problems 128 | # https://github.com/paramiko/paramiko/issues/60 129 | # 130 | sftp = get_sftp_client(client.ssh_client_target) 131 | sftp.put(origin_path + database_utility.database_dump_file_name + '.tar.gz', 132 | helper.get_dump_dir(mode.Client.TARGET) + database_utility.database_dump_file_name + '.tar.gz', 133 | upload_status) 134 | sftp.close() 135 | if not system.config['mute']: 136 | print('') 137 | 138 | 139 | 140 | def upload_status(sent, size): 141 | """ 142 | Printing the upload status information 143 | :param sent: Float 144 | :param size: Float 145 | :return: 146 | """ 147 | if not system.config['mute']: 148 | sent_mb = round(float(sent) / 1024 / 1024, 1) 149 | size = round(float(size) / 1024 / 1024, 1) 150 | 151 | if (mode.get_sync_mode() == mode.SyncMode.PROXY): 152 | _subject = output.Subject.LOCAL 153 | else: 154 | _subject = output.Subject.ORIGIN + output.CliFormat.BLACK + '[LOCAL]' + output.CliFormat.ENDC 155 | 156 | sys.stdout.write( 157 | _subject + " Status: {0} MB of {1} MB uploaded". 158 | format(sent_mb, size, )) 159 | sys.stdout.write('\r') 160 | 161 | 162 | def get_sftp_client(ssh_client): 163 | """ 164 | 165 | :param ssh_client: 166 | :return: 167 | """ 168 | sftp = ssh_client.open_sftp() 169 | sftp.get_channel().settimeout(client.default_timeout) 170 | return sftp 171 | 172 | -------------------------------------------------------------------------------- /docs/MODE.md: -------------------------------------------------------------------------------- 1 | # Sync modes 2 | 3 | It is possible to enable different kind of sync modes for the database synchronisation depending on the preferred origin (which provides the database dump) and target system (which receiving the database dump). 4 | 5 | The default mode is _receiver_. 6 | 7 | - [Receiver](#sm-receiver) 8 | - [Sender](#sm-sender) 9 | - [Proxy](#sm-proxy) 10 | - [Dump Local](#sm-dump-local) 11 | - [Dump Remote](#sm-dump-remote) 12 | - [Import Local](#sm-import-local) 13 | - [Import Remote](#sm-import-remote) 14 | - [Sync Local](#sm-sync-local) 15 | - [Sync Remote](#sm-sync-remote) 16 | 17 | 18 | ## Receiver 19 | 20 | The _receiver_ mode offers the possibility to get a database dump from a remote system (origin) to your local system (target). 21 | 22 | ![Sync mode receiver](images/sm-receiver.png) 23 | 24 | This mode is enabled, if a `host` entry is __only__ be stored in the `origin` section of the `config.yaml` configuration. 25 | 26 | ```yaml 27 | target: 28 | path: 29 | origin: 30 | host: 31 | user: 32 | path: 33 | ``` 34 | 35 | 36 | ## Sender 37 | 38 | The _sender_ mode offers the possibility to provide a database dump from your local system (origin) to a remote system (target). 39 | 40 | ![Sync mode sender](images/sm-sender.png) 41 | 42 | This mode is enabled, if a `host` entry is __only__ be stored in the `target` section of the `config.yaml` configuration. 43 | 44 | ```yaml 45 | target: 46 | host: 47 | user: 48 | path: 49 | origin: 50 | path: 51 | ``` 52 | 53 | 54 | ## Proxy 55 | 56 | The _proxy_ mode offers the possibility to get a database dump from a remote system (origin) and store them temporarily on your local system. After that the stored database dump will be forwarded to another remote system (target). 57 | 58 | ![Sync mode proxy](images/sm-proxy.png) 59 | 60 | This mode can be used, when origin and target system can't or shouldn't connect directly (because of security restrictions). So your local system acts as proxy between both of them. 61 | 62 | This mode is enabled, if a `host` entry is being stored in the `origin` __and__ `target` section of the `config.yaml` configuration. 63 | 64 | ```yaml 65 | target: 66 | host: 67 | user: 68 | path: 69 | origin: 70 | host: 71 | user: 72 | path: 73 | ``` 74 | 75 | 76 | ## Dump Local 77 | 78 | The _dump local_ mode offers the possibility to only save a database dump from your local system. This is not really a synchronisation mode, just an easy way to save a database dump on your local mashine. No file transfer or database import will be performed. You can specify the dump file location with the `dump_dir` setting in your `config.yaml`. 79 | 80 | ![Sync mode sender](images/sm-dump-local.png) 81 | 82 | This mode is enabled, if _no_ `host` entry is being stored in the `target` or `origin` section of the `config.yaml` configuration. 83 | 84 | ```yaml 85 | origin: 86 | path: 87 | dump_dir: 88 | ``` 89 | 90 | 91 | ## Dump Remote 92 | 93 | The _dump local_ mode offers the possibility to only save a database dump from your local system. This is not really a synchronisation mode, just an easy way to save a database dump on a remote mashine (e.g. as backup mechanism). No file transfer or database import will be performed. You can specify the dump file location with the `dump_dir` setting in your `config.yaml`. 94 | 95 | ![Sync mode sender](images/sm-dump-remote.png) 96 | 97 | This mode is enabled, if the `host` entry are equal in the `target` and `origin` section of the `config.yaml` configuration. 98 | 99 | ```yaml 100 | target: 101 | host: 102 | user: 103 | path: 104 | origin: 105 | host: 106 | user: 107 | path: 108 | dump_dir: 109 | ``` 110 | 111 | 112 | ## Import Local 113 | 114 | The _import local_ mode offers the possibility to only import a database dump from a local file dump. This is not really a synchronisation mode, just an easy way to import a database dump on your local mashine. No file transfer will be performed. 115 | 116 | ![Sync mode sender](images/sm-dump-local.png) 117 | 118 | ```yaml 119 | target: 120 | path: 121 | ``` 122 | 123 | This mode is enabled, if you add the `-i` or `--import-file` option and specify the location of the local dump file. 124 | 125 | ```shell 126 | $ db_sync_tool -f import-local.yaml -i /path/to/file.sql 127 | ``` 128 | 129 | 130 | ## Import Remote 131 | 132 | The _import remote_ mode offers the possibility to only import a database dump on a remote file dump. This is not really a synchronisation mode, just an easy way to import a database dump on a remote maschine. No file transfer will be performed. 133 | 134 | ![Sync mode sender](images/sm-dump-remote.png) 135 | ```yaml 136 | target: 137 | host: 138 | user: 139 | path: 140 | ``` 141 | 142 | This mode is enabled, if you add the `-i` or `--import-file` option, specify the location of the remote dump file and the `target` in your `config.yaml` is a remote system. 143 | 144 | ```shell 145 | $ db_sync_tool -f import-local.yaml -i /path/to/file.sql 146 | ``` 147 | 148 | 149 | ## Sync Local 150 | 151 | The _sync local_ mode offers the possibility to sync a database within your local system (origin/target). 152 | 153 | ![Sync mode receiver](images/sm-sync-local.png) 154 | 155 | This mode is enabled, if no `host` entry is stored and the `path` entries differ in the `config.yaml` configuration. 156 | 157 | ```yaml 158 | target: 159 | path: 160 | origin: 161 | path: 162 | ``` 163 | 164 | 165 | ## Sync Remote 166 | 167 | The _sync remote_ mode offers the possibility to sync a database within a remote system (origin/target). 168 | 169 | ![Sync mode receiver](images/sm-sync-remote.png) 170 | 171 | This mode is enabled, if the `host` entries will be the same and the `path` entries differ in the `config.yaml` configuration. 172 | 173 | ```yaml 174 | target: 175 | host: 176 | user: 177 | path: 178 | origin: 179 | host: 180 | user: 181 | path: 182 | ``` -------------------------------------------------------------------------------- /db_sync_tool/remote/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Client script 6 | """ 7 | 8 | import sys 9 | import paramiko 10 | from db_sync_tool.utility import mode, system, helper, output 11 | 12 | ssh_client_origin = None 13 | ssh_client_target = None 14 | additional_ssh_clients = [] 15 | 16 | default_timeout = 600 17 | 18 | 19 | def load_ssh_client_origin(): 20 | """ 21 | Loading the origin ssh client 22 | :return: 23 | """ 24 | global ssh_client_origin 25 | ssh_client_origin = load_ssh_client(mode.Client.ORIGIN) 26 | helper.run_script(mode.Client.ORIGIN, 'before') 27 | 28 | 29 | def load_ssh_client_target(): 30 | """ 31 | Loading the target ssh client 32 | :return: 33 | """ 34 | global ssh_client_target 35 | ssh_client_target = load_ssh_client(mode.Client.TARGET) 36 | helper.run_script(mode.Client.TARGET, 'before') 37 | 38 | 39 | def load_ssh_client(ssh): 40 | """ 41 | Initializing the given ssh client 42 | :param ssh: String 43 | :return: 44 | """ 45 | _host_name = helper.get_ssh_host_name(ssh, True) 46 | _ssh_client = paramiko.SSHClient() 47 | _ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 48 | 49 | _ssh_port = system.config[ssh]['port'] if 'port' in system.config[ssh] else 22 50 | _ssh_key = None 51 | _ssh_password = None 52 | 53 | # Check authentication 54 | if 'ssh_key' in system.config[ssh]: 55 | _authentication_method = f'{output.CliFormat.BLACK} - ' \ 56 | f'(authentication: key){output.CliFormat.ENDC}' 57 | _ssh_key = system.config[ssh]['ssh_key'] 58 | elif 'password' in system.config[ssh]: 59 | _authentication_method = f'{output.CliFormat.BLACK} - ' \ 60 | f'authentication: password){output.CliFormat.ENDC}' 61 | _ssh_password = system.config[ssh]['password'] 62 | elif 'ssh_agent' in system.config: 63 | _authentication_method = f'{output.CliFormat.BLACK} - ' \ 64 | f'(authentication: key){output.CliFormat.ENDC}' 65 | else: 66 | sys.exit( 67 | output.message( 68 | output.Subject.ERROR, 69 | 'Missing SSH authentication. Neither ssh key nor ssh password given.', 70 | False 71 | ) 72 | ) 73 | 74 | # Try to connect to remote client via paramiko 75 | try: 76 | _ssh_client.connect(hostname=system.config[ssh]['host'], 77 | username=system.config[ssh]['user'], 78 | key_filename=_ssh_key, 79 | password=_ssh_password, 80 | port=_ssh_port, 81 | compress=True, 82 | timeout=default_timeout, 83 | sock=get_jump_host_channel(ssh)) 84 | # 85 | # Workaround for long-lasting requests 86 | # https://stackoverflow.com/questions/50009688/python-paramiko-ssh-session-not-active-after-being-idle-for-many-hours 87 | # 88 | _ssh_client.get_transport().set_keepalive(60) 89 | 90 | except paramiko.ssh_exception.AuthenticationException: 91 | sys.exit( 92 | output.message( 93 | output.Subject.ERROR, 94 | f'SSH authentication for {_host_name} failed', 95 | False 96 | ) 97 | ) 98 | 99 | output.message( 100 | output.host_to_subject(ssh), 101 | f'Initialize remote SSH connection {_host_name}{_authentication_method}', 102 | True 103 | ) 104 | 105 | return _ssh_client 106 | 107 | 108 | def close_ssh_clients(): 109 | """ 110 | Closing ssh client sessions 111 | :return: 112 | """ 113 | helper.run_script(mode.Client.ORIGIN, 'after') 114 | if not ssh_client_origin is None: 115 | ssh_client_origin.close() 116 | 117 | helper.run_script(mode.Client.TARGET, 'after') 118 | if not ssh_client_target is None: 119 | ssh_client_target.close() 120 | 121 | for additional_ssh_client in additional_ssh_clients: 122 | additional_ssh_client.close() 123 | 124 | helper.run_script(script='after') 125 | 126 | 127 | def get_jump_host_channel(client): 128 | """ 129 | Provide an optional transport channel for a SSH jump host client 130 | https://gist.github.com/tintoy/443c42ea3865680cd624039c4bb46219 131 | :param client: 132 | :return: 133 | """ 134 | _jump_host_channel = None 135 | if 'jump_host' in system.config[client]: 136 | # prepare jump host config 137 | _jump_host_client = paramiko.SSHClient() 138 | _jump_host_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 139 | 140 | _jump_host_host = system.config[client]['jump_host']['host'] 141 | _jump_host_user = system.config[client]['jump_host']['user'] if 'user' in system.config[client]['jump_host'] else system.config[client]['user'] 142 | 143 | if 'ssh_key' in system.config[client]['jump_host']: 144 | _jump_host_ssh_key = system.config[client]['jump_host']['ssh_key'] 145 | elif 'ssh_key' in system.config[client]: 146 | _jump_host_ssh_key = system.config[client]['ssh_key'] 147 | else: 148 | _jump_host_ssh_key = None 149 | 150 | if 'port' in system.config[client]['jump_host']: 151 | _jump_host_port = system.config[client]['jump_host']['port'] 152 | elif 'port' in system.config[client]: 153 | _jump_host_port = system.config[client]['port'] 154 | else: 155 | _jump_host_port = 22 156 | 157 | # connect to the jump host 158 | _jump_host_client.connect( 159 | hostname=_jump_host_host, 160 | username=_jump_host_user, 161 | key_filename=_jump_host_ssh_key, 162 | password=system.config[client]['jump_host']['password'] if 'password' in system.config[client]['jump_host'] else None, 163 | port=_jump_host_port, 164 | compress=True, 165 | timeout=default_timeout 166 | ) 167 | 168 | global additional_ssh_clients 169 | additional_ssh_clients.append(_jump_host_client) 170 | 171 | # open the necessary channel 172 | _jump_host_transport = _jump_host_client.get_transport() 173 | _jump_host_channel = _jump_host_transport.open_channel( 174 | 'direct-tcpip', 175 | dest_addr=(system.config[client]['host'], 22), 176 | src_addr=(system.config[client]['jump_host']['private'] if 'private' in system.config[client]['jump_host'] else system.config[client]['jump_host']['host'], 22) 177 | ) 178 | 179 | # print information 180 | _destination_client = helper.get_ssh_host_name(client, minimal=True) 181 | _jump_host_name = system.config[client]['jump_host']['name'] if 'name' in system.config[client]['jump_host'] else _jump_host_host 182 | output.message( 183 | output.host_to_subject(client), 184 | f'Initialize remote SSH jump host {output.CliFormat.BLACK}local ➔ {output.CliFormat.BOLD}{_jump_host_name}{output.CliFormat.ENDC}{output.CliFormat.BLACK} ➔ {_destination_client}{output.CliFormat.ENDC}', 185 | True 186 | ) 187 | 188 | return _jump_host_channel 189 | 190 | -------------------------------------------------------------------------------- /db_sync_tool/utility/parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Parser script 6 | """ 7 | 8 | import sys 9 | from db_sync_tool.utility import mode, system, output, helper 10 | from db_sync_tool.remote import client as remote_client 11 | 12 | 13 | class Framework: 14 | TYPO3 = 'TYPO3' 15 | SYMFONY = 'Symfony' 16 | DRUPAL = 'Drupal' 17 | WORDPRESS = 'Wordpress' 18 | LARAVEL = 'Laravel' 19 | MANUAL = 'Manual' 20 | 21 | 22 | mapping = { 23 | Framework.TYPO3: [ 24 | 'LocalConfiguration.php', 25 | 'AdditionalConfiguration.php' 26 | ], 27 | Framework.SYMFONY: [ 28 | '.env', 29 | 'parameters.yml' 30 | ], 31 | Framework.DRUPAL: [ 32 | 'settings.php' 33 | ], 34 | Framework.WORDPRESS: [ 35 | 'wp-config.php' 36 | ], 37 | Framework.LARAVEL: [ 38 | '.env' 39 | ] 40 | } 41 | 42 | 43 | def get_database_configuration(client): 44 | """ 45 | Getting database configuration of given client and defined sync base (framework type) 46 | :param client: String 47 | :return: 48 | """ 49 | system.config['db'] = {} 50 | 51 | # check framework type 52 | _base = '' 53 | 54 | automatic_type_detection() 55 | 56 | if 'type' in system.config and ( 57 | 'path' in system.config[mode.Client.ORIGIN] or 58 | 'path' in system.config[mode.Client.TARGET] 59 | ): 60 | _type = system.config['type'].lower() 61 | if _type == 'typo3': 62 | # TYPO3 sync base 63 | _base = Framework.TYPO3 64 | elif _type == 'symfony': 65 | # Symfony sync base 66 | _base = Framework.SYMFONY 67 | elif _type == 'drupal': 68 | # Drupal sync base 69 | _base = Framework.DRUPAL 70 | elif _type == 'wordpress': 71 | # Wordpress sync base 72 | _base = Framework.WORDPRESS 73 | elif _type == 'laravel': 74 | # Laravel sync base 75 | _base = Framework.LARAVEL 76 | else: 77 | sys.exit( 78 | output.message( 79 | output.Subject.ERROR, 80 | f'Framework type not supported: {_type}', 81 | False 82 | ) 83 | ) 84 | elif 'db' in system.config['origin'] or 'db' in system.config['target']: 85 | _base = Framework.MANUAL 86 | else: 87 | sys.exit( 88 | output.message( 89 | output.Subject.ERROR, 90 | f'Missing framework type or database credentials', 91 | False 92 | ) 93 | ) 94 | 95 | sys.path.append('../recipes') 96 | if _base == Framework.TYPO3: 97 | # Import TYPO3 parser 98 | from ..recipes import typo3 99 | _parser = typo3 100 | 101 | elif _base == Framework.SYMFONY: 102 | # Import Symfony parser 103 | from ..recipes import symfony 104 | _parser = symfony 105 | 106 | elif _base == Framework.DRUPAL: 107 | # Import Symfony parser 108 | from ..recipes import drupal 109 | _parser = drupal 110 | 111 | elif _base == Framework.WORDPRESS: 112 | # Import Symfony parser 113 | from ..recipes import wordpress 114 | _parser = wordpress 115 | 116 | elif _base == Framework.LARAVEL: 117 | # Import Symfony parser 118 | from ..recipes import laravel 119 | _parser = laravel 120 | 121 | if client == mode.Client.ORIGIN: 122 | output.message( 123 | output.Subject.INFO, 124 | 'Sync base: ' + _base, 125 | True 126 | ) 127 | 128 | if _base != Framework.MANUAL: 129 | load_parser(client, _parser) 130 | else: 131 | if client == mode.Client.ORIGIN and mode.is_origin_remote(): 132 | remote_client.load_ssh_client_origin() 133 | elif client == mode.Client.TARGET and mode.is_target_remote(): 134 | remote_client.load_ssh_client_target() 135 | 136 | validate_database_credentials(client) 137 | 138 | 139 | def load_parser(client, parser): 140 | """ 141 | Loading parser and checking database configuration 142 | :param client: 143 | :param parser: 144 | :return: 145 | """ 146 | _path = system.config[client]['path'] 147 | 148 | output.message( 149 | output.host_to_subject(client), 150 | f'Checking database configuration {output.CliFormat.BLACK}{_path}{output.CliFormat.ENDC}', 151 | True 152 | ) 153 | if client == mode.Client.ORIGIN: 154 | if mode.is_origin_remote(): 155 | remote_client.load_ssh_client_origin() 156 | else: 157 | helper.run_script(client, 'before') 158 | else: 159 | if mode.is_target_remote(): 160 | remote_client.load_ssh_client_target() 161 | else: 162 | helper.run_script(client, 'before') 163 | 164 | # Check only if database configuration is a file 165 | if not helper.check_file_exists(client, _path) and _path[-1] != '/': 166 | sys.exit( 167 | output.message( 168 | output.Subject.ERROR, 169 | f'Database configuration for {client} not found: {_path}', 170 | False 171 | ) 172 | ) 173 | parser.check_configuration(client) 174 | 175 | 176 | def validate_database_credentials(client): 177 | """ 178 | Validate the parsed database credentials 179 | :param client: String 180 | :return: 181 | """ 182 | output.message( 183 | output.host_to_subject(client), 184 | 'Validating database credentials', 185 | True 186 | ) 187 | _db_credential_keys = ['name', 'host', 'password', 'user'] 188 | 189 | for _key in _db_credential_keys: 190 | if _key not in system.config[client]['db']: 191 | sys.exit( 192 | output.message( 193 | output.Subject.ERROR, 194 | f'Missing database credential "{_key}" for {client} client', 195 | False 196 | ) 197 | ) 198 | if system.config[client]['db'][_key] is None or system.config[client]['db'][_key] == '': 199 | sys.exit( 200 | output.message( 201 | output.Subject.ERROR, 202 | f'Missing database credential "{_key}" for {client} client', 203 | False 204 | ) 205 | ) 206 | else: 207 | output.message( 208 | output.host_to_subject(client), 209 | f'Database credential "{_key}" valid', 210 | verbose_only=True 211 | ) 212 | 213 | 214 | def automatic_type_detection(): 215 | """ 216 | Detects the framework type by the provided path using the default mapping 217 | """ 218 | if 'type' in system.config or 'db' in system.config['origin'] or 'db' in system.config[ 219 | 'target']: 220 | return 221 | 222 | type = None 223 | file = None 224 | 225 | for _client in [mode.Client.ORIGIN, mode.Client.TARGET]: 226 | if 'path' in system.config[_client]: 227 | file = helper.get_file_from_path(system.config[_client]['path']) 228 | for _key, _files in mapping.items(): 229 | if file in _files: 230 | type = _key 231 | 232 | if type: 233 | output.message( 234 | output.Subject.LOCAL, 235 | f'Automatic framework type detection ' 236 | f'{output.CliFormat.BLACK}{file}{output.CliFormat.ENDC}', 237 | verbose_only=True 238 | ) 239 | system.config['type'] = type 240 | -------------------------------------------------------------------------------- /db_sync_tool/database/utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Utility script 6 | """ 7 | 8 | import sys 9 | import datetime 10 | import re 11 | from db_sync_tool.utility import mode, system, helper, output 12 | 13 | database_dump_file_name = None 14 | 15 | 16 | class DatabaseSystem: 17 | MYSQL = 'MySQL' 18 | MARIADB = 'MariaDB' 19 | 20 | 21 | def run_database_command(client, command, force_database_name=False): 22 | """ 23 | Run a database command using the "mysql -e" command 24 | :param client: String 25 | :param command: String database command 26 | :param force_database_name: Bool forces the database name 27 | :return: 28 | """ 29 | _database_name = ' ' + system.config[client]['db']['name'] if force_database_name else '' 30 | 31 | return mode.run_command( 32 | helper.get_command(client, 'mysql') + ' ' + generate_mysql_credentials( 33 | client) + _database_name + ' -e "' + command + '"', 34 | client, True) 35 | 36 | 37 | def generate_database_dump_filename(): 38 | """ 39 | Generate a database dump filename like "_[name]_[date].sql" or using the give filename 40 | :return: 41 | """ 42 | global database_dump_file_name 43 | 44 | if system.config['dump_name'] == '': 45 | # _project-db_2022-08-22_12-37.sql 46 | _now = datetime.datetime.now() 47 | database_dump_file_name = '_' + system.config[mode.Client.ORIGIN]['db']['name'] + '_' + _now.strftime( 48 | "%Y-%m-%d_%H-%M") + '.sql' 49 | else: 50 | database_dump_file_name = system.config['dump_name'] + '.sql' 51 | 52 | 53 | def truncate_tables(): 54 | """ 55 | Generate the ignore tables options for the mysqldump command by the given table list 56 | # ToDo: Too much conditional nesting 57 | :return: String 58 | """ 59 | # Workaround for config naming 60 | if 'truncate_table' in system.config: 61 | system.config['truncate_tables'] = system.config['truncate_table'] 62 | 63 | if 'truncate_tables' in system.config: 64 | output.message( 65 | output.Subject.TARGET, 66 | 'Truncating tables before import', 67 | True 68 | ) 69 | for _table in system.config['truncate_tables']: 70 | if '*' in _table: 71 | _wildcard_tables = get_database_tables_like(mode.Client.TARGET, 72 | _table.replace('*', '%')) 73 | if _wildcard_tables: 74 | for _wildcard_table in _wildcard_tables: 75 | _sql_command = f'TRUNCATE TABLE IF EXISTS {_wildcard_table}' 76 | run_database_command(mode.Client.TARGET, _sql_command, True) 77 | else: 78 | _sql_command = f'TRUNCATE TABLE IF EXISTS {_table}' 79 | run_database_command(mode.Client.TARGET, _sql_command, True) 80 | 81 | 82 | def generate_ignore_database_tables(): 83 | """ 84 | Generate the ignore tables options for the mysqldump command by the given table list 85 | # ToDo: Too much conditional nesting 86 | :return: String 87 | """ 88 | # Workaround for config naming 89 | if 'ignore_table' in system.config: 90 | system.config['ignore_tables'] = system.config['ignore_table'] 91 | 92 | _ignore_tables = [] 93 | if 'ignore_tables' in system.config: 94 | for table in system.config['ignore_tables']: 95 | if '*' in table: 96 | _wildcard_tables = get_database_tables_like(mode.Client.ORIGIN, 97 | table.replace('*', '%')) 98 | if _wildcard_tables: 99 | for wildcard_table in _wildcard_tables: 100 | _ignore_tables = generate_ignore_database_table(_ignore_tables, 101 | wildcard_table) 102 | else: 103 | _ignore_tables = generate_ignore_database_table(_ignore_tables, table) 104 | return ' '.join(_ignore_tables) 105 | return '' 106 | 107 | 108 | def generate_ignore_database_table(ignore_tables, table): 109 | """ 110 | :param ignore_tables: Dictionary 111 | :param table: String 112 | :return: Dictionary 113 | """ 114 | ignore_tables.append('--ignore-table=' + system.config['origin']['db']['name'] + '.' + table) 115 | return ignore_tables 116 | 117 | 118 | def get_database_tables_like(client, name): 119 | """ 120 | Get database table names like the given name 121 | :param client: String 122 | :param name: String 123 | :return: Dictionary 124 | """ 125 | _dbname = system.config[client]['db']['name'] 126 | _tables = run_database_command(client, f'SHOW TABLES FROM \`{_dbname}\` LIKE \'{name}\';').strip() 127 | if _tables != '': 128 | return _tables.split('\n')[1:] 129 | return 130 | 131 | 132 | def get_database_tables(): 133 | """ 134 | Generate specific tables for export 135 | :return: String 136 | """ 137 | if system.config['tables'] == '': 138 | return '' 139 | 140 | _result = ' ' 141 | _tables = system.config['tables'].split(',') 142 | for _table in _tables: 143 | _result += '\'' + _table + '\' ' 144 | return _result 145 | 146 | 147 | def generate_mysql_credentials(client, force_password=True): 148 | """ 149 | Generate the needed database credential information for the mysql command 150 | :param client: String 151 | :param force_password: Bool 152 | :return: 153 | """ 154 | _credentials = '-u\'' + system.config[client]['db']['user'] + '\'' 155 | if force_password: 156 | _credentials += ' -p\'' + system.config[client]['db']['password'] + '\'' 157 | if 'host' in system.config[client]['db']: 158 | _credentials += ' -h\'' + system.config[client]['db']['host'] + '\'' 159 | if 'port' in system.config[client]['db']: 160 | _credentials += ' -P\'' + str(system.config[client]['db']['port']) + '\'' 161 | return _credentials 162 | 163 | 164 | def check_database_dump(client, filepath): 165 | """ 166 | Checking the last line of the dump file if it contains "-- Dump completed on" 167 | :param client: String 168 | :param filepath: String 169 | :return: 170 | """ 171 | if system.config['check_dump']: 172 | _line = mode.run_command( 173 | helper.get_command(client, 'tail') + ' -n 1 ' + filepath, 174 | client, 175 | True, 176 | skip_dry_run=True 177 | ) 178 | 179 | if not _line: 180 | return 181 | 182 | if "-- Dump completed on" not in _line: 183 | sys.exit( 184 | output.message( 185 | output.Subject.ERROR, 186 | 'Dump file is corrupted', 187 | do_print=False 188 | ) 189 | ) 190 | else: 191 | output.message( 192 | output.host_to_subject(client), 193 | 'Dump file is valid', 194 | verbose_only=True 195 | ) 196 | 197 | 198 | def count_tables(client, filepath): 199 | """ 200 | Count the reference string in the database dump file to get the count of all exported tables 201 | :param client: String 202 | :param filepath: String 203 | :return: 204 | """ 205 | _reference = 'CREATE TABLE' 206 | _count = mode.run_command( 207 | f'{helper.get_command(client, "grep")} -ao "{_reference}" {filepath} | wc -l | xargs', 208 | client, 209 | True, 210 | skip_dry_run=True 211 | ) 212 | 213 | if _count: 214 | output.message( 215 | output.host_to_subject(client), 216 | f'{int(_count)} table(s) exported' 217 | ) 218 | 219 | 220 | def get_database_version(client): 221 | """ 222 | Check the database version and distinguish between mysql and mariadb 223 | :param client: 224 | :return: Tuple 225 | """ 226 | _database_system = None 227 | _version_number = None 228 | try: 229 | _database_version = run_database_command(client, 'SELECT VERSION();').splitlines()[1] 230 | _database_system = DatabaseSystem.MYSQL 231 | 232 | _version_number = re.search('(\d+\.)?(\d+\.)?(\*|\d+)', _database_version).group() 233 | 234 | if DatabaseSystem.MARIADB.lower() in _database_version.lower(): 235 | _database_system = DatabaseSystem.MARIADB 236 | 237 | output.message( 238 | output.host_to_subject(client), 239 | f'Database version: {_database_system} v{_version_number}', 240 | True 241 | ) 242 | finally: 243 | return _database_system, _version_number 244 | 245 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [2.11.12] - 2025-11-05 10 | 11 | - fix: set type for param --target-after-dump to str 12 | 13 | ## [2.11.11] - 2025-07-23 14 | 15 | - fix: include user configuration check in same host validation 16 | 17 | ## [2.11.10] - 2025-04-09 18 | 19 | - feat: distinguish between sed support for -E and -r option 20 | 21 | ## [2.11.9] - 2025-04-07 22 | 23 | - docs: add download count for python package 24 | - feat: add more flexible TYPO3 environment sync configuration and testing script 25 | - docs: add typo3 documentation for .env file configuration for database credentials 26 | 27 | ## [2.11.8] - 2025-01-22 28 | 29 | - fix: remove unsupported MYSQL_PWD env variable 30 | 31 | ## [2.11.7] - 2024-09-23 32 | 33 | - fix: cleanup quotes surrounding database credentials 34 | 35 | ## [2.11.6] - 2024-06-17 36 | 37 | - fix: allow import local with remote config 38 | 39 | ## [2.11.5] - 2024-04-12 40 | 41 | - fix: syntaxwarning "is not" with a literal 42 | 43 | ## [2.11.4] - 2024-03-20 44 | 45 | - fix: string contact within cleanup command 46 | 47 | ## [2.11.3] - 2024-03-20 48 | 49 | - feat: remove sftp from cleanup 50 | 51 | ## [2.11.2] - 2024-03-18 52 | 53 | - fix: unit converter for rsync stats 54 | 55 | ## [2.11.1] - 2023-12-12 56 | 57 | - feat: support additional mysqldump options 58 | 59 | ## [2.11.0] - 2023-12-04 60 | 61 | - feat: support additional where clause 62 | 63 | ## [2.10.11] - 2023-09-26 64 | 65 | - fix: symfony database url with additional parameters 66 | 67 | ## [2.10.10] - 2023-09-25 68 | 69 | - fix: symfony database url with additional parameters 70 | 71 | ## [2.10.9] - 2023-02-01 72 | 73 | - fix: missing database password configuration for clearing 74 | 75 | ## [2.10.8] - 2023-02-01 76 | 77 | - fix: missing database password configuration for clearing 78 | 79 | ## [2.10.7] - 2023-01-31 80 | 81 | - fix: missing database password configuration for clearing 82 | 83 | ## [2.10.6] - 2023-01-26 84 | 85 | - fix: missing database password configuration for clearing 86 | 87 | ## [2.10.5] - 2023-01-25 88 | 89 | - fix: wrong database password configuration 90 | - chore: update requirements 91 | 92 | ## [2.10.4] - 2023-01-05 93 | 94 | - fix: missing database password within console command 95 | 96 | ## [2.10.3] - 2023-01-05 97 | 98 | - feat: add typo3 env parse function 99 | 100 | ## [2.10.2] - 2022-07-28 101 | 102 | - fix: overwrite hosts with arguments 103 | - fix: consider user confirmation input 104 | 105 | ## [2.10.1] - 2022-06-22 106 | 107 | - [Bugfix] Console output header fixed 108 | - [Bugfix] Fix manual shell database arguments handling 109 | 110 | ## [2.10.0] - 2022-05-29 111 | 112 | - [Task] Loading spinner added 113 | - [Task] Rename database dump filename 114 | - [Task] Colored header 115 | - [Task] Released dependencies 116 | - [Test] Docker compose updated for ARM/M1 117 | 118 | ## [2.9.2] - 2022-05-11 119 | 120 | - [Bugfix] Tables option with necessary whitespace 121 | 122 | ## [2.9.1] - 2022-05-04 123 | 124 | - [Bugfix] Dictionary argument fix 125 | - [Doc] Additional documentation for the jump host feature 126 | 127 | ## [2.9.0] - 2022-04-10 128 | 129 | - [Task] Jump host feature 130 | 131 | ## [2.8.3] - 2022-04-08 132 | 133 | - [Bugfix] Ignore table wildcard 134 | - [Bugfix] Fix hosts without config 135 | 136 | ## [2.8.2] - 2022-04-04 137 | 138 | - [Bugfix] Module usage broken 139 | 140 | ## [2.8.1] - 2022-04-04 141 | 142 | - [Bugfix] Optional args argument 143 | - [Task] Changelog updated 144 | 145 | ## [2.8.0] - 2022-04-03 146 | 147 | - [Test] Test Setting adjusted 148 | - [Task] Reverse feature 149 | - [Task] Load arguments after config file 150 | - [Task] Truncate feature 151 | - [Task] Post SQL feature 152 | - [Task] Protected hosts 153 | - [Doc] Extended documentation for yml usage 154 | - [Task] Bump paramiko from 2.8.0 to 2.10.1 155 | 156 | ## [2.7.0] - 2022-03-06 157 | ### Added 158 | - [Task] Single table export 159 | - [Task] Read from TYPO3 AdditionalConfiguration.php 160 | - [Task] Hosts in config option 161 | 162 | ## [2.6.0] - 2022-02-13 163 | ### Added 164 | - [Task] Use rsync as transfer method 165 | 166 | ## [2.5.7] - 2021-11-15 167 | ### Added 168 | - [Bugfix] Python3.9 warning fixed 169 | - [Task] Requirements updated 170 | 171 | ## [2.5.6] - 2021-10-08 172 | ### Added 173 | - [Bugfix] Fix mysql command option 174 | 175 | ## [2.5.5] - 2021-10-08 176 | ### Added 177 | - [Task] mysql command fix for < v5.6 178 | - [Task] Improve version comparison 179 | - [Task] Check for database version 180 | - [Task] Check for own package update 181 | 182 | ## [2.5.4] - 2021-08-31 183 | ### Added 184 | - [Bugfix] Mask database name 185 | - [Build] Dockerfile updated 186 | 187 | ## [2.5.3] - 2021-08-31 188 | ### Added 189 | - [Task] Force password option 190 | 191 | ## [2.5.2] - 2021-08-31 192 | ### Added 193 | - [Task] SSH authentication via SSH agent 194 | 195 | ## [2.5.1] - 2021-06-21 196 | ### Added 197 | - [Task] Default Timeout for Paramiko Clients 198 | 199 | ## [2.5.0] - 2021-05-31 200 | ### Added 201 | - [Task] Laravel support 202 | - [Task] YAML support 203 | - [Task] JSON Schema 204 | - [Task] Host linking arguments 205 | - [Task] Automatic framework type detection 206 | - [CleanUp] pylint 207 | 208 | ## [2.4.3] - 2021-03-10 209 | ### Fixed 210 | - [Bugfix] Proxy mode clean up temp dir 211 | 212 | ## [2.4.2] - 2021-03-08 213 | ### Added 214 | - [Task] Keep alive for ssh client 215 | ### Fixed 216 | - [Bugfix] Ignore table wildcard fix 217 | 218 | ## [2.4.1] - 2021-03-07 219 | ### Fixed 220 | - [Bugfix] "Sync local" fix 221 | 222 | ## [2.4.0] - 2021-03-07 223 | ### Added 224 | - [Task] Exported tables info 225 | - [Task] Sync modes "Sync local" and "Sync remote" 226 | - [Task] Adjust final message for dry run mode 227 | - [Task] Simplify mode detection 228 | - [Task] "Dry run" mode 229 | 230 | ## [2.3.0] - 2021-02-19 231 | ### Added 232 | - [Task] Clear Database Feature 233 | - [Doc] Further documentation 234 | 235 | ## [2.2.6] - 2021-01-18 236 | ### Added 237 | - [Task] Extend scripts feature 238 | - [Doc] Support section 239 | - [Doc] Quickstart documentation 240 | - [Task] Improved console output 241 | - [Task] Drupal recipe uses Drush 242 | - [Doc] Release Guide added 243 | - [Doc] Advanced documentation 244 | 245 | ## [2.2.5] - 2020-12-28 246 | ### Added 247 | - [Task] Allow run command to fail 248 | - [Improvement] Adjust console header 249 | 250 | ## [2.2.4] - 2020-12-28 251 | ### Added 252 | - [Task] Refactoring 253 | - [Task] Additional output 254 | - [Task] Refactor authorization 255 | - [Task] Client "Local" 256 | 257 | ## [2.2.3] - 2020-12-23 258 | ### Added 259 | - [Task] Database port is not required 260 | - [Task] User confirmation for database import 261 | ### Fixed 262 | - [Bugfix] Nested dict for shell argument initialization 263 | - [Bugfix] Wrong dict key for ssh key 264 | 265 | ## [2.2.2] - 2020-12-09 266 | ### Fixed 267 | - [Bugfix] Requirements in setup extended 268 | 269 | ## [2.2.1] - 2020-12-06 270 | ### Added 271 | - [Task] After_dump feature added 272 | ### Fixed 273 | - [Bugfix] Fix legacy shell script call 274 | 275 | ## [2.2.0] - 2020-12-03 276 | ### Added 277 | - [Task] Added python 3.5 support 278 | - [Task] Shell arguments extended 279 | - [Task] Improved test output 280 | - [Task] Extend setup 281 | - [Task] Added Travis CI 282 | - [Task] Manual database credentials 283 | - [Task] Refactoring module structure 284 | - [Task] Simplify extensions to recipes 285 | - [Task] Further error prevention 286 | - [Task] Wildcards for ignore tables 287 | ### Fixed 288 | - [Bugfix] Drupal database settings parsing improved 289 | 290 | ## [2.1.0] - 2020-11-05 291 | ### Added 292 | - [Task] Test scenario symfony2.8 added 293 | - [Task] Support Symfony v2.8 294 | 295 | ## [2.0.1] - 2020-11-05 296 | ### Added 297 | - [Task] Validating database credentials 298 | 299 | ### Fixed 300 | - [Bugfix] SSH key authorization for dump-remote 301 | 302 | ## [2.0.0] - 2020-11-03 303 | ### Added 304 | - [Task] Support legacy script call from command line 305 | - [Task] Improved script handling 306 | - [Task] Support Wordpress 307 | - [Task] Extension Drupal improved 308 | - [Task] Run sync as module 309 | - [Task] Adding supported framework Drupal 8 310 | - [Task][!!!] Refactoring for export as python package 311 | - [Task] Connection output improved 312 | 313 | ## [1.8.0] - 2020-10-06 314 | ### Added 315 | - [Task] Error handling improved 316 | - [Task] Logging output optimized 317 | - [Task] Re-enable check dump feature 318 | - [Task] Added test environment 319 | - [Task] Host linking feature added 320 | - [Task] Added password option for hosts 321 | - [Task] Adding "no-tablespaces" option for mysqldump command 322 | - [CleanUp] Refactoring code 323 | 324 | ### Fixed 325 | - [Bugfix] Logging error bug -------------------------------------------------------------------------------- /db_sync_tool/database/process.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Process script 6 | """ 7 | import semantic_version 8 | 9 | from db_sync_tool.utility import parser, mode, system, helper, output 10 | from db_sync_tool.database import utility as database_utility 11 | 12 | 13 | def create_origin_database_dump(): 14 | """ 15 | Creating the origin database dump file 16 | :return: 17 | """ 18 | if not mode.is_import(): 19 | parser.get_database_configuration(mode.Client.ORIGIN) 20 | database_utility.generate_database_dump_filename() 21 | helper.check_and_create_dump_dir(mode.Client.ORIGIN, 22 | helper.get_dump_dir(mode.Client.ORIGIN)) 23 | 24 | _dump_file_path = helper.get_dump_dir( 25 | mode.Client.ORIGIN) + database_utility.database_dump_file_name 26 | 27 | _database_version = database_utility.get_database_version(mode.Client.ORIGIN) 28 | output.message( 29 | output.Subject.ORIGIN, 30 | f'Creating database dump {output.CliFormat.BLACK}{_dump_file_path}{output.CliFormat.ENDC}', 31 | True 32 | ) 33 | 34 | _mysqldump_options = '--no-tablespaces ' 35 | # Remove --no-tablespaces option for mysql < 5.6 36 | # @ToDo: Better option handling 37 | if not _database_version is None: 38 | if _database_version[0] == database_utility.DatabaseSystem.MYSQL and \ 39 | semantic_version.Version(_database_version[1]) < semantic_version.Version('5.6.0'): 40 | _mysqldump_options = '' 41 | 42 | # Adding additional where clause to sync only selected rows 43 | if system.config['where'] != '': 44 | _where = system.config['where'] 45 | _mysqldump_options = _mysqldump_options + f'--where=\'{_where}\' ' 46 | 47 | # Adding additional mysqldump options 48 | # see https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html#mysqldump-option-summary 49 | if system.config['additional_mysqldump_options'] != '': 50 | _additional = system.config['additional_mysqldump_options'] 51 | _mysqldump_options = _mysqldump_options + f'{_additional} ' 52 | 53 | # Run mysql dump command, e.g. 54 | # mysqldump --no-tablespaces -u'db' -p'db' -h'db1' -P'3306' 'db' > /tmp/_db_08-10-2021_07-00.sql 55 | mode.run_command( 56 | helper.get_command(mode.Client.ORIGIN, 'mysqldump') + ' ' + _mysqldump_options + 57 | database_utility.generate_mysql_credentials(mode.Client.ORIGIN) + ' \'' + 58 | system.config[mode.Client.ORIGIN]['db']['name'] + '\' ' + 59 | database_utility.generate_ignore_database_tables() + 60 | database_utility.get_database_tables() + 61 | ' > ' + _dump_file_path, 62 | mode.Client.ORIGIN, 63 | skip_dry_run=True 64 | ) 65 | 66 | database_utility.check_database_dump(mode.Client.ORIGIN, _dump_file_path) 67 | database_utility.count_tables(mode.Client.ORIGIN, _dump_file_path) 68 | prepare_origin_database_dump() 69 | 70 | 71 | def import_database_dump(): 72 | """ 73 | Importing the selected database dump file 74 | :return: 75 | """ 76 | if not system.config['is_same_client'] and not mode.is_import(): 77 | prepare_target_database_dump() 78 | 79 | if system.config['clear_database']: 80 | output.message( 81 | output.Subject.TARGET, 82 | 'Clearing database before import', 83 | True 84 | ) 85 | clear_database(mode.Client.TARGET) 86 | 87 | database_utility.truncate_tables() 88 | 89 | if not system.config['keep_dump'] and not mode.is_dump(): 90 | 91 | database_utility.get_database_version(mode.Client.TARGET) 92 | 93 | output.message( 94 | output.Subject.TARGET, 95 | 'Importing database dump', 96 | True 97 | ) 98 | 99 | if not mode.is_import(): 100 | _dump_path = helper.get_dump_dir( 101 | mode.Client.TARGET) + database_utility.database_dump_file_name 102 | else: 103 | _dump_path = system.config['import'] 104 | 105 | if not system.config['yes']: 106 | _host_name = helper.get_ssh_host_name(mode.Client.TARGET, True) if mode.is_remote( 107 | mode.Client.TARGET) else 'local' 108 | 109 | _input = helper.confirm( 110 | output.message( 111 | output.Subject.TARGET, 112 | f'Are you sure, you want to import the dump file into {_host_name} database?', 113 | False 114 | ), 115 | True 116 | ) 117 | 118 | if not _input: return 119 | 120 | database_utility.check_database_dump(mode.Client.TARGET, _dump_path) 121 | 122 | import_database_dump_file(mode.Client.TARGET, _dump_path) 123 | 124 | if 'after_dump' in system.config['target']: 125 | _after_dump = system.config['target']['after_dump'] 126 | output.message( 127 | output.Subject.TARGET, 128 | f'Importing after_dump file {output.CliFormat.BLACK}{_after_dump}{output.CliFormat.ENDC}', 129 | True 130 | ) 131 | import_database_dump_file(mode.Client.TARGET, _after_dump) 132 | 133 | if 'post_sql' in system.config['target']: 134 | output.message( 135 | output.Subject.TARGET, 136 | f'Running addition post sql commands', 137 | True 138 | ) 139 | for _sql_command in system.config['target']['post_sql']: 140 | database_utility.run_database_command(mode.Client.TARGET, _sql_command, True) 141 | 142 | 143 | def import_database_dump_file(client, filepath): 144 | """ 145 | Import a database dump file 146 | :param client: String 147 | :param filepath: String 148 | :return: 149 | """ 150 | if helper.check_file_exists(client, filepath): 151 | mode.run_command( 152 | helper.get_command(client, 'mysql') + ' ' + 153 | database_utility.generate_mysql_credentials(client) + ' \'' + 154 | system.config[client]['db']['name'] + '\' < ' + filepath, 155 | client, 156 | skip_dry_run=True 157 | ) 158 | 159 | 160 | def prepare_origin_database_dump(): 161 | """ 162 | Preparing the origin database dump file by compressing them as .tar.gz 163 | :return: 164 | """ 165 | output.message( 166 | output.Subject.ORIGIN, 167 | 'Compressing database dump', 168 | True 169 | ) 170 | mode.run_command( 171 | helper.get_command(mode.Client.ORIGIN, 'tar') + ' cfvz ' + helper.get_dump_dir( 172 | mode.Client.ORIGIN) + database_utility.database_dump_file_name + '.tar.gz -C ' + 173 | helper.get_dump_dir(mode.Client.ORIGIN) + ' ' + 174 | database_utility.database_dump_file_name + ' > /dev/null', 175 | mode.Client.ORIGIN, 176 | skip_dry_run=True 177 | ) 178 | 179 | 180 | def prepare_target_database_dump(): 181 | """ 182 | Preparing the target database dump by the unpacked .tar.gz file 183 | :return: 184 | """ 185 | output.message(output.Subject.TARGET, 'Extracting database dump', True) 186 | mode.run_command( 187 | helper.get_command('target', 'tar') + ' xzf ' + helper.get_dump_dir(mode.Client.TARGET) + 188 | database_utility.database_dump_file_name + '.tar.gz -C ' + 189 | helper.get_dump_dir(mode.Client.TARGET) + ' > /dev/null', 190 | mode.Client.TARGET, 191 | skip_dry_run=True 192 | ) 193 | 194 | 195 | def clear_database(client): 196 | """ 197 | Clearing the database by dropping all tables 198 | https://www.techawaken.com/drop-tables-mysql-database/ 199 | 200 | { mysql -hHOSTNAME -uUSERNAME -pPASSWORD -Nse 'show tables' DB_NAME; } | 201 | ( while read table; do if [ -z ${i+x} ]; then echo 'SET FOREIGN_KEY_CHECKS = 0;'; fi; i=1; 202 | echo "drop table \`$table\`;"; done; 203 | echo 'SET FOREIGN_KEY_CHECKS = 1;' ) | 204 | awk '{print}' ORS=' ' | mysql -hHOSTNAME -uUSERNAME -pPASSWORD DB_NAME; 205 | 206 | :param client: String 207 | :return: 208 | """ 209 | mode.run_command( 210 | '{ ' + helper.get_command(client, 'mysql') + ' ' + 211 | database_utility.generate_mysql_credentials(client) + 212 | ' -Nse \'show tables\' \'' + 213 | system.config[client]['db']['name'] + '\'; }' + 214 | ' | ( while read table; do if [ -z ${i+x} ]; then echo \'SET FOREIGN_KEY_CHECKS = 0;\'; fi; i=1; ' + 215 | 'echo "drop table \\`$table\\`;"; done; echo \'SET FOREIGN_KEY_CHECKS = 1;\' ) | awk \'{print}\' ORS=' ' | ' + 216 | ' ' + 217 | helper.get_command(client, 'mysql') + ' ' + 218 | database_utility.generate_mysql_credentials(client) + ' ' + 219 | system.config[client]['db']['name'], 220 | client, 221 | skip_dry_run=True 222 | ) 223 | -------------------------------------------------------------------------------- /db_sync_tool/utility/mode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Mode script 6 | """ 7 | 8 | import subprocess 9 | import sys 10 | 11 | from db_sync_tool.utility import system, output, helper 12 | from db_sync_tool.remote import system as remote_system 13 | 14 | 15 | # 16 | # GLOBALS 17 | # 18 | 19 | class Client: 20 | ORIGIN = 'origin' 21 | TARGET = 'target' 22 | LOCAL = 'local' 23 | 24 | 25 | class SyncMode: 26 | """ 27 | Sync Mode 28 | """ 29 | 30 | DUMP_LOCAL = 'DUMP_LOCAL' 31 | DUMP_REMOTE = 'DUMP_REMOTE' 32 | IMPORT_LOCAL = 'IMPORT_LOCAL' 33 | IMPORT_REMOTE = 'IMPORT_REMOTE' 34 | RECEIVER = 'RECEIVER' 35 | SENDER = 'SENDER' 36 | PROXY = 'PROXY' 37 | SYNC_REMOTE = 'SYNC_REMOTE' 38 | SYNC_LOCAL = 'SYNC_LOCAL' 39 | 40 | @staticmethod 41 | def is_dump_local(): 42 | """ 43 | 44 | :return: boolean 45 | """ 46 | return SyncMode.is_full_local() and SyncMode.is_same_host() and not SyncMode.is_sync_local() 47 | 48 | @staticmethod 49 | def is_dump_remote(): 50 | """ 51 | 52 | :return: boolean 53 | """ 54 | return SyncMode.is_full_remote() and SyncMode.is_same_host() and \ 55 | not SyncMode.is_sync_remote() 56 | 57 | @staticmethod 58 | def is_receiver(): 59 | """ 60 | 61 | :return: boolean 62 | """ 63 | return 'host' in system.config[Client.ORIGIN] and not SyncMode.is_proxy() and \ 64 | not SyncMode.is_sync_remote() 65 | 66 | @staticmethod 67 | def is_sender(): 68 | """ 69 | 70 | :return: boolean 71 | """ 72 | return 'host' in system.config[Client.TARGET] and not SyncMode.is_proxy() and \ 73 | not SyncMode.is_sync_remote() 74 | 75 | @staticmethod 76 | def is_proxy(): 77 | """ 78 | 79 | :return: boolean 80 | """ 81 | return SyncMode.is_full_remote() 82 | 83 | @staticmethod 84 | def is_import_local(): 85 | """ 86 | 87 | :return: boolean 88 | """ 89 | return system.config['import'] != '' and 'host' not in system.config[Client.TARGET] 90 | 91 | @staticmethod 92 | def is_import_remote(): 93 | """ 94 | 95 | :return: boolean 96 | """ 97 | return system.config['import'] != '' and 'host' in system.config[Client.TARGET] 98 | 99 | @staticmethod 100 | def is_sync_local(): 101 | """ 102 | 103 | :return: boolean 104 | """ 105 | return SyncMode.is_full_local() and SyncMode.is_same_host() and SyncMode.is_same_sync() 106 | 107 | @staticmethod 108 | def is_sync_remote(): 109 | """ 110 | 111 | :return: boolean 112 | """ 113 | return SyncMode.is_full_remote() and SyncMode.is_same_host() and SyncMode.is_same_sync() 114 | 115 | @staticmethod 116 | def is_same_sync(): 117 | """ 118 | 119 | :return: boolean 120 | """ 121 | return ((SyncMode.is_available_configuration('path') and 122 | not SyncMode.is_same_configuration('path')) or 123 | (SyncMode.is_available_configuration('db') and 124 | not SyncMode.is_same_configuration('db'))) 125 | 126 | @staticmethod 127 | def is_full_remote(): 128 | """ 129 | 130 | :return: boolean 131 | """ 132 | return SyncMode.is_available_configuration('host') 133 | 134 | @staticmethod 135 | def is_full_local(): 136 | """ 137 | 138 | :return: boolean 139 | """ 140 | return SyncMode.is_unavailable_configuration('host') 141 | 142 | @staticmethod 143 | def is_same_host(): 144 | """ 145 | 146 | :return: boolean 147 | """ 148 | return SyncMode.is_same_configuration('host') and SyncMode.is_same_configuration('port') and SyncMode.is_same_configuration('user') 149 | 150 | @staticmethod 151 | def is_available_configuration(key): 152 | """ 153 | 154 | :return: boolean 155 | """ 156 | return key in system.config[Client.ORIGIN] and key in system.config[Client.TARGET] 157 | 158 | @staticmethod 159 | def is_unavailable_configuration(key): 160 | """ 161 | 162 | :return: boolean 163 | """ 164 | return key not in system.config[Client.ORIGIN] and key not in system.config[Client.TARGET] 165 | 166 | @staticmethod 167 | def is_same_configuration(key): 168 | """ 169 | 170 | :return: boolean 171 | """ 172 | return (SyncMode.is_available_configuration(key) and 173 | system.config[Client.ORIGIN][key] == system.config[Client.TARGET][key]) or \ 174 | SyncMode.is_unavailable_configuration(key) 175 | 176 | 177 | # Default sync mode 178 | sync_mode = SyncMode.RECEIVER 179 | 180 | 181 | # 182 | # FUNCTIONS 183 | # 184 | def get_sync_mode(): 185 | """ 186 | Returning the sync mode 187 | :return: String sync_mode 188 | """ 189 | return sync_mode 190 | 191 | 192 | def check_sync_mode(): 193 | """ 194 | Checking the sync_mode based on the given configuration 195 | :return: String subject 196 | """ 197 | global sync_mode 198 | _description = '' 199 | 200 | _modes = { 201 | SyncMode.RECEIVER: '(REMOTE ➔ LOCAL)', 202 | SyncMode.SENDER: '(LOCAL ➔ REMOTE)', 203 | SyncMode.PROXY: '(REMOTE ➔ LOCAL ➔ REMOTE)', 204 | SyncMode.DUMP_LOCAL: '(LOCAL, ONLY EXPORT)', 205 | SyncMode.DUMP_REMOTE: '(REMOTE, ONLY EXPORT)', 206 | SyncMode.IMPORT_LOCAL: '(REMOTE, ONLY IMPORT)', 207 | SyncMode.IMPORT_REMOTE: '(LOCAL, ONLY IMPORT)', 208 | SyncMode.SYNC_LOCAL: '(LOCAL ➔ LOCAL)', 209 | SyncMode.SYNC_REMOTE: '(REMOTE ➔ REMOTE)' 210 | } 211 | 212 | for _mode, _desc in _modes.items(): 213 | if getattr(SyncMode, 'is_' + _mode.lower())(): 214 | sync_mode = _mode 215 | _description = _desc 216 | 217 | if is_import(): 218 | output.message( 219 | output.Subject.INFO, 220 | f'Import file {output.CliFormat.BLACK}{system.config["import"]}{output.CliFormat.ENDC}', 221 | True 222 | ) 223 | 224 | system.config['is_same_client'] = SyncMode.is_same_host() 225 | 226 | output.message( 227 | output.Subject.INFO, 228 | f'Sync mode: {sync_mode} {output.CliFormat.BLACK}{_description}{output.CliFormat.ENDC}', 229 | True 230 | ) 231 | 232 | check_for_protection() 233 | 234 | 235 | def is_remote(client): 236 | """ 237 | Check if given client is remote client 238 | :param client: String 239 | :return: Boolean 240 | """ 241 | if client == Client.ORIGIN: 242 | return is_origin_remote() 243 | elif client == Client.TARGET: 244 | return is_target_remote() 245 | elif client == Client.LOCAL: 246 | return False 247 | else: 248 | return False 249 | 250 | 251 | def is_target_remote(): 252 | """ 253 | Check if target is remote client 254 | :return: Boolean 255 | """ 256 | return sync_mode in (SyncMode.SENDER, SyncMode.PROXY, SyncMode.DUMP_REMOTE, 257 | SyncMode.IMPORT_REMOTE, SyncMode.SYNC_REMOTE) 258 | 259 | 260 | def is_origin_remote(): 261 | """ 262 | Check if origin is remote client 263 | :return: Boolean 264 | """ 265 | return sync_mode in (SyncMode.RECEIVER, SyncMode.PROXY, SyncMode.DUMP_REMOTE, 266 | SyncMode.IMPORT_REMOTE, SyncMode.SYNC_REMOTE) 267 | 268 | 269 | def is_import(): 270 | """ 271 | Check if sync mode is import 272 | :return: Boolean 273 | """ 274 | return sync_mode in (SyncMode.IMPORT_LOCAL, SyncMode.IMPORT_REMOTE) 275 | 276 | 277 | def is_dump(): 278 | """ 279 | Check if sync mode is import 280 | :return: Boolean 281 | """ 282 | return sync_mode in (SyncMode.DUMP_LOCAL, SyncMode.DUMP_REMOTE) 283 | 284 | 285 | def run_command(command, client, force_output=False, allow_fail=False, skip_dry_run=False): 286 | """ 287 | Run command depending on the given client 288 | :param command: String 289 | :param client: String 290 | :param force_output: Boolean 291 | :param allow_fail: Boolean 292 | :param skip_dry_run: Boolean 293 | :return: 294 | """ 295 | if system.config['verbose']: 296 | output.message( 297 | output.host_to_subject(client), 298 | output.CliFormat.BLACK + command + output.CliFormat.ENDC, 299 | debug=True 300 | ) 301 | 302 | if system.config['dry_run'] and skip_dry_run: 303 | return 304 | 305 | if is_remote(client): 306 | if force_output: 307 | return ''.join(remote_system.run_ssh_command_by_client(client, command).readlines()).strip() 308 | else: 309 | return remote_system.run_ssh_command_by_client(client, command) 310 | else: 311 | res = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) 312 | # Wait for the process end and print error in case of failure 313 | out, err = res.communicate() 314 | 315 | if res.wait() != 0 and err.decode() != '' and not allow_fail: 316 | helper.run_script(script='error') 317 | sys.exit(output.message(output.Subject.ERROR, err.decode(), False)) 318 | 319 | if force_output: 320 | return out.decode().strip() 321 | 322 | 323 | def check_for_protection(): 324 | """ 325 | Check if the target system is protected 326 | :return: Boolean 327 | """ 328 | if sync_mode in (SyncMode.RECEIVER, SyncMode.SENDER, SyncMode.PROXY, SyncMode.SYNC_LOCAL, 329 | SyncMode.SYNC_REMOTE, SyncMode.IMPORT_LOCAL, SyncMode.IMPORT_REMOTE) and \ 330 | 'protect' in system.config[Client.TARGET]: 331 | _host = helper.get_ssh_host_name(Client.TARGET) 332 | sys.exit(output.message(output.Subject.ERROR, 333 | f'The host {_host} is protected against the import of a database dump. Please ' 334 | 'check synchronisation target or adjust the host configuration.', False)) 335 | 336 | -------------------------------------------------------------------------------- /db_sync_tool/utility/helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Helper script 6 | """ 7 | 8 | import shutil 9 | import os 10 | import re 11 | from db_sync_tool.utility import mode, system, output 12 | from db_sync_tool.remote import utility as remote_utility 13 | 14 | 15 | def clean_up(): 16 | """ 17 | Clean up 18 | :return: 19 | """ 20 | if not mode.is_import(): 21 | remote_utility.remove_target_database_dump() 22 | if mode.get_sync_mode() == mode.SyncMode.PROXY: 23 | remove_temporary_data_dir() 24 | 25 | 26 | def remove_temporary_data_dir(): 27 | """ 28 | Remove temporary data directory for storing database dump files 29 | :return: 30 | """ 31 | if os.path.exists(system.default_local_sync_path): 32 | output.message( 33 | output.Subject.LOCAL, 34 | 'Cleaning up', 35 | True 36 | ) 37 | shutil.rmtree(system.default_local_sync_path) 38 | 39 | 40 | def clean_up_dump_dir(client, path, num=5): 41 | """ 42 | Clean up the dump directory from old dump files (only affect .sql and .tar.gz files) 43 | :param client: 44 | :param path: 45 | :param num: 46 | :return: 47 | """ 48 | # Distinguish stat command on os system (Darwin|Linux) 49 | if check_os(client).strip() == 'Darwin': 50 | _command = get_command(client, 'stat') + ' -f "%Sm %N" ' + path + ' | ' + get_command( 51 | client, 52 | 'sort') + ' -rn | ' + get_command( 53 | client, 'grep') + ' -E ".tar.gz|.sql"' 54 | else: 55 | _command = get_command(client, 'stat') + ' -c "%y %n" ' + path + ' | ' + \ 56 | get_command(client,'sort') + ' -rn | ' + get_command(client, 'grep') + \ 57 | ' -E ".tar.gz|.sql"' 58 | 59 | # List files in directory sorted by change date 60 | _files = mode.run_command( 61 | _command, 62 | client, 63 | True 64 | ).splitlines() 65 | 66 | for i in range(len(_files)): 67 | _filename = _files[i].rsplit(' ', 1)[-1] 68 | 69 | # Remove oldest files chosen by keep_dumps count 70 | if not i < num: 71 | mode.run_command( 72 | 'rm ' + _filename, 73 | client 74 | ) 75 | 76 | 77 | def check_os(client): 78 | """ 79 | Check which system is running (Linux|Darwin) 80 | :param client: 81 | :return: 82 | """ 83 | return mode.run_command( 84 | get_command(client, 'uname') + ' -s', 85 | client, 86 | True 87 | ) 88 | 89 | 90 | def get_command(client, command): 91 | """ 92 | Get command helper for overriding default commands on the given client 93 | :param client: 94 | :param command: 95 | :return: String command 96 | """ 97 | if 'console' in system.config[client]: 98 | if command in system.config[client]['console']: 99 | return system.config[client]['console'][command] 100 | return command 101 | 102 | 103 | def get_dump_dir(client): 104 | """ 105 | Get database dump directory by client 106 | :param client: 107 | :return: String path 108 | """ 109 | if system.config[f'default_{client}_dump_dir']: 110 | return '/tmp/' 111 | else: 112 | return system.config[client]['dump_dir'] 113 | 114 | 115 | def check_and_create_dump_dir(client, path): 116 | """ 117 | Check if a path exists on the client system and creates the given path if necessary 118 | :param client: 119 | :param path: 120 | :return: 121 | """ 122 | mode.run_command( 123 | '[ ! -d "' + path + '" ] && mkdir -p "' + path + '"', 124 | client 125 | ) 126 | 127 | 128 | def get_ssh_host_name(client, with_user=False, minimal=False): 129 | """ 130 | Format ssh host name depending on existing client name 131 | :param client: 132 | :param with_user: 133 | :param short: 134 | :return: 135 | """ 136 | if not 'user' in system.config[client] and not 'host' in system.config[client]: 137 | return '' 138 | 139 | if with_user: 140 | _host = system.config[client]['user'] + '@' + system.config[client]['host'] 141 | else: 142 | _host = system.config[client]['host'] 143 | 144 | if 'name' in system.config[client]: 145 | if minimal: 146 | return system.config[client]['name'] 147 | else: 148 | return output.CliFormat.BOLD + system.config[client][ 149 | 'name'] + output.CliFormat.ENDC + output.CliFormat.BLACK + ' (' + _host + ')' + \ 150 | output.CliFormat.ENDC 151 | else: 152 | return _host 153 | 154 | 155 | def create_local_temporary_data_dir(): 156 | """ 157 | Create local temporary data dir 158 | :return: 159 | """ 160 | # @ToDo: Combine with check_and_create_dump_dir() 161 | if not os.path.exists(system.default_local_sync_path): 162 | os.makedirs(system.default_local_sync_path) 163 | 164 | 165 | def dict_to_args(dict): 166 | """ 167 | Convert an dictionary to a args list 168 | :param dict: Dictionary 169 | :return: List 170 | """ 171 | _args = [] 172 | for key, val in dict.items(): 173 | if isinstance(val, bool): 174 | if val: 175 | _args.append(f'--{key}') 176 | else: 177 | _args.append(f'--{key}') 178 | _args.append(str(val)) 179 | if len(_args) == 0: 180 | return None 181 | return _args 182 | 183 | 184 | def check_file_exists(client, path): 185 | """ 186 | Check if a file exists 187 | :param client: String 188 | :param path: String file path 189 | :return: Boolean 190 | """ 191 | return mode.run_command(f'[ -f {path} ] && echo "1"', client, True) == '1' 192 | 193 | 194 | def run_script(client=None, script='before'): 195 | """ 196 | Executing script command 197 | :param client: String 198 | :param script: String 199 | :return: 200 | """ 201 | if client is None: 202 | _config = system.config 203 | _subject = output.Subject.LOCAL 204 | client = mode.Client.LOCAL 205 | else: 206 | _config = system.config[client] 207 | _subject = output.host_to_subject(client) 208 | 209 | if not 'scripts' in _config: 210 | return 211 | 212 | if f'{script}' in _config['scripts']: 213 | output.message( 214 | _subject, 215 | f'Running script {client}', 216 | True 217 | ) 218 | mode.run_command( 219 | _config['scripts'][script], 220 | client 221 | ) 222 | 223 | 224 | def check_rsync_version(): 225 | """ 226 | Check rsync version 227 | :return: 228 | """ 229 | _raw_version = mode.run_command( 230 | 'rsync --version', 231 | mode.Client.LOCAL, 232 | True 233 | ) 234 | _version = parse_version(_raw_version) 235 | output.message( 236 | output.Subject.LOCAL, 237 | f'rsync version {_version}' 238 | ) 239 | 240 | 241 | def check_sshpass_version(): 242 | """ 243 | Check sshpass version 244 | :return: 245 | """ 246 | _raw_version = mode.run_command( 247 | 'sshpass -V', 248 | mode.Client.LOCAL, 249 | force_output=True, 250 | allow_fail=True 251 | ) 252 | _version = parse_version(_raw_version) 253 | 254 | if _version: 255 | output.message( 256 | output.Subject.LOCAL, 257 | f'sshpass version {_version}' 258 | ) 259 | system.config['use_sshpass'] = True 260 | return True 261 | 262 | 263 | def parse_version(output): 264 | """ 265 | Parse version out of console output 266 | https://stackoverflow.com/a/60730346 267 | :param output: String 268 | :return: 269 | """ 270 | _version_pattern = r'\d+(=?\.(\d+(=?\.(\d+)*)*)*)*' 271 | _regex_matcher = re.compile(_version_pattern) 272 | _version = _regex_matcher.search(output) 273 | if _version: 274 | return _version.group(0) 275 | else: 276 | return None 277 | 278 | 279 | def get_file_from_path(path): 280 | """ 281 | Trims a path string to retrieve the file 282 | :param path: 283 | :return: file 284 | """ 285 | return path.split('/')[-1] 286 | 287 | 288 | def confirm(prompt=None, resp=False): 289 | """ 290 | https://code.activestate.com/recipes/541096-prompt-the-user-for-confirmation/ 291 | 292 | prompts for yes or no response from the user. Returns True for yes and 293 | False for no. 294 | 295 | 'resp' should be set to the default value assumed by the caller when 296 | user simply types ENTER. 297 | 298 | >>> confirm(prompt='Create Directory?', resp=True) 299 | Create Directory? [Y|n]: 300 | True 301 | >>> confirm(prompt='Create Directory?', resp=False) 302 | Create Directory? [y|N]: 303 | False 304 | 305 | """ 306 | 307 | if prompt is None: 308 | prompt = 'Confirm' 309 | 310 | if resp: 311 | prompt = '%s [%s|%s]: ' % (prompt, 'Y', 'n') 312 | else: 313 | prompt = '%s [%s|%s]: ' % (prompt, 'y', 'N') 314 | 315 | while True: 316 | ans = input(prompt) 317 | if not ans: 318 | return resp 319 | if ans not in ['y', 'Y', 'n', 'N']: 320 | print('Please enter y or n.') 321 | continue 322 | if ans == 'y' or ans == 'Y': 323 | return True 324 | if ans == 'n' or ans == 'N': 325 | return False 326 | 327 | 328 | def clean_db_config(config): 329 | """ 330 | Iterates over all entries of a dictionary and removes enclosing inverted commas 331 | from the values, if present. 332 | 333 | :param config: The dictionary to be edited 334 | :return: A new dictionary with adjusted values 335 | """ 336 | # Iterate over all entries in the dictionary and use the inverted comma function 337 | return {key: remove_surrounding_quotes(value) for key, value in config.items()} 338 | 339 | 340 | def remove_surrounding_quotes(s): 341 | """ 342 | Removes the enclosing inverted commas (single or double), 343 | if there are quotes at both the beginning and the end of the string. 344 | 345 | :param s: The string to be checked 346 | :return: The string without enclosing quotes, if available 347 | """ 348 | if isinstance(s, str): 349 | if s.startswith('"') and s.endswith('"'): 350 | return s[1:-1] 351 | elif s.startswith("'") and s.endswith("'"): 352 | return s[1:-1] 353 | return s 354 | 355 | 356 | def run_sed_command(client, command): 357 | """ 358 | Executes a sed command on the specified client, trying -E first and falling back to -r if -E fails. 359 | 360 | :param client: The client on which the sed command should be executed. 361 | :param command: The sed command to execute (excluding the sed options). 362 | :return: The result of the sed command as a cleaned string (with newlines removed). 363 | """ 364 | # Check if the client supports -E or -r option for sed 365 | option = mode.run_command( 366 | f"echo | {get_command(client, 'sed')} -E '' >/dev/null 2>&1 && echo -E || (echo | {get_command(client, 'sed')} -r '' >/dev/null 2>&1 && echo -r)", 367 | client, 368 | True 369 | ) 370 | # If neither option is supported, default to -E 371 | if option == '': 372 | option = '-E' 373 | 374 | return mode.run_command( 375 | f"{get_command(client, 'sed')} -n {option} {command}", 376 | client, 377 | True 378 | ).strip().replace('\n', '') 379 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # db sync tool 2 | 3 | ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/db_sync_tool-kmi) 4 | ![PyPI](https://img.shields.io/pypi/v/db_sync_tool-kmi) 5 | ![Pepy Total Downloads](https://img.shields.io/pepy/dt/db-sync-tool-kmi) 6 | ![PyPI - Downloads](https://img.shields.io/pypi/dm/db-sync-tool-kmi) 7 | [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/jackd248/db-sync-tool/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/jackd248/db-sync-tool/?branch=master) 8 | [![Build Status](https://scrutinizer-ci.com/g/jackd248/db-sync-tool/badges/build.png?b=master)](https://scrutinizer-ci.com/g/jackd248/db-sync-tool/build-status/master) 9 | 10 | Python script to synchronize a database from an origin to a target system with automatic database credential extraction depending on the selected framework. 11 | 12 | ## Features 13 | 14 | - __Database sync__ from and to a remote system 15 | - [MySQL](https://www.mysql.com/) (>= 5.5) 16 | - [MariaDB](https://mariadb.org/) (>= 10.0) 17 | - __Proxy mode__ between two remote systems 18 | - Several [synchronisation modes](docs/MODE.md) 19 | - Automatic database __credential extraction__ using a supported framework 20 | - [TYPO3](https://typo3.org/) (>= v7.6) 21 | - [Symfony](https://symfony.com/) (>= v2.8) 22 | - [Drupal](https://www.drupal.org/) (>= v8.0) 23 | - [Wordpress](https://wordpress.org) (>= v5.0) 24 | - [Laravel](https://laravel.com/) (>= v4.0) 25 | - Easily dump creation (database __backup__) 26 | - __Cleanup__ feature for backups 27 | - Extended __logging__ capabilities 28 | - Many more possibilities for [customization](docs/CONFIG.md) 29 | 30 | ## Installation 31 | 32 | ### Prerequisite 33 | 34 | The script needs [python](https://python.org/) __3.5__ or higher. It is necessary for some additional functionalities to have [pip](https://pypi.org/project/pip/) installed on your local machine. 35 | 36 | 37 | ### pip 38 | The library can be installed from [PyPI](https://pypi.org/project/db-sync-tool-kmi/): 39 | ```bash 40 | $ pip3 install db-sync-tool-kmi 41 | ``` 42 | 43 | 44 | ### composer 45 | While using the script within the PHP framework context, the script is available via [packagist.org](https://packagist.org/packages/kmi/db-sync-tool) using composer: 46 | 47 | ```bash 48 | $ composer require kmi/db-sync-tool 49 | ``` 50 | 51 | Additionally install the python requirements via the following pip command: 52 | 53 | ````bash 54 | $ pip3 install -e vendor/kmi/db-sync-tool/ 55 | ```` 56 | 57 | ## Quickstart 58 | 59 | Detailed instructions for: 60 | 61 | - [Manual database sync](docs/quickstart/START.md) 62 | - [TYPO3 database sync](docs/quickstart/TYPO3.md) 63 | - [Symfony database sync](docs/quickstart/SYMFONY.md) 64 | - [Drupal database sync](docs/quickstart/DRUPAL.md) 65 | - [Wordpress database sync](docs/quickstart/WORDPRESS.md) 66 | 67 | If you want to have an inside in more configuration examples, see the [test scenarios](tests/scenario). 68 | 69 | ## Usage 70 | 71 | ### Command line 72 | 73 | Run the python script via command line. 74 | 75 | Installed via [pip](#install-pip): 76 | ```bash 77 | $ db_sync_tool 78 | ``` 79 | 80 | Installed via [composer](#install-composer): 81 | ```bash 82 | $ python3 vendor/kmi/db-sync-tool/db_sync_tool 83 | ``` 84 | 85 | ![Example receiver](docs/images/db-sync-tool-example-receiver.gif) 86 | 87 | 88 | #### Shell arguments 89 | 90 | ```bash 91 | usage: db_sync_tool [-h] [-f CONFIG_FILE] [-v] [-y] [-m] [-dr] [-i IMPORT_FILE] [-dn DUMP_NAME] [-kd KEEP_DUMP] [-o HOST_FILE] [-l LOG_FILE] [-cd] [-ta TABLES] [-r] [-t TYPE] [-tp TARGET_PATH] 92 | [-tn TARGET_NAME] [-th TARGET_HOST] [-tu TARGET_USER] [-tpw TARGET_PASSWORD] [-tk TARGET_KEY] [-tpo TARGET_PORT] [-tdd TARGET_DUMP_DIR] [-tkd TARGET_KEEP_DUMPS] [-tdn TARGET_DB_NAME] 93 | [-tdh TARGET_DB_HOST] [-tdu TARGET_DB_USER] [-tdpw TARGET_DB_PASSWORD] [-tdpo TARGET_DB_PORT] [-tad TARGET_AFTER_DUMP] [-op ORIGIN_PATH] [-on ORIGIN_NAME] [-oh ORIGIN_HOST] 94 | [-ou ORIGIN_USER] [-opw ORIGIN_PASSWORD] [-ok ORIGIN_KEY] [-opo ORIGIN_PORT] [-odd ORIGIN_DUMP_DIR] [-okd ORIGIN_KEEP_DUMPS] [-odn ORIGIN_DB_NAME] [-odh ORIGIN_DB_HOST] 95 | [-odu ORIGIN_DB_USER] [-odpw ORIGIN_DB_PASSWORD] [-odpo ORIGIN_DB_PORT] [-fpw] [-ur] [-uro USE_RSYNC_OPTIONS] 96 | [origin] [target] 97 | 98 | A tool for automatic database synchronization from and to host systems. 99 | 100 | positional arguments: 101 | origin Origin database defined in host file 102 | target Target database defined in host file 103 | 104 | optional arguments: 105 | -h, --help show this help message and exit 106 | -f CONFIG_FILE, --config-file CONFIG_FILE 107 | Path to configuration file 108 | -v, --verbose Enable extended console output 109 | -y, --yes Skipping user confirmation for database import 110 | -m, --mute Mute console output 111 | -dr, --dry-run Testing process without running database export, transfer or import. 112 | -i IMPORT_FILE, --import-file IMPORT_FILE 113 | Import database from a specific file dump 114 | -dn DUMP_NAME, --dump-name DUMP_NAME 115 | Set a specific dump file name (default is "_[dbname]_[date]") 116 | -kd KEEP_DUMP, --keep-dump KEEP_DUMP 117 | Skipping target import of the database dump and saving the available dump file in the given directory 118 | -o HOST_FILE, --host-file HOST_FILE 119 | Using an additional hosts file for merging hosts information with the configuration file 120 | -l LOG_FILE, --log-file LOG_FILE 121 | File path for creating a additional log file 122 | -cd, --clear-database 123 | Dropping all tables before importing a new sync to get a clean database. 124 | -ta TABLES, --tables TABLES 125 | Defining specific tables to export, e.g. --tables=table1,table2 126 | -r, --reverse Reverse origin and target hosts 127 | -t TYPE, --type TYPE Defining the framework type [TYPO3, Symfony, Drupal, Wordpress] 128 | -tp TARGET_PATH, --target-path TARGET_PATH 129 | File path to target database credential file depending on the framework type 130 | -tn TARGET_NAME, --target-name TARGET_NAME 131 | Providing a name for the target system 132 | -th TARGET_HOST, --target-host TARGET_HOST 133 | SSH host to target system 134 | -tu TARGET_USER, --target-user TARGET_USER 135 | SSH user for target system 136 | -tpw TARGET_PASSWORD, --target-password TARGET_PASSWORD 137 | SSH password for target system 138 | -tk TARGET_KEY, --target-key TARGET_KEY 139 | File path to SSH key for target system 140 | -tpo TARGET_PORT, --target-port TARGET_PORT 141 | SSH port for target system 142 | -tdd TARGET_DUMP_DIR, --target-dump-dir TARGET_DUMP_DIR 143 | Directory path for database dump file on target system 144 | -tkd TARGET_KEEP_DUMPS, --target-keep-dumps TARGET_KEEP_DUMPS 145 | Keep dump file count for target system 146 | -tdn TARGET_DB_NAME, --target-db-name TARGET_DB_NAME 147 | Database name for target system 148 | -tdh TARGET_DB_HOST, --target-db-host TARGET_DB_HOST 149 | Database host for target system 150 | -tdu TARGET_DB_USER, --target-db-user TARGET_DB_USER 151 | Database user for target system 152 | -tdpw TARGET_DB_PASSWORD, --target-db-password TARGET_DB_PASSWORD 153 | Database password for target system 154 | -tdpo TARGET_DB_PORT, --target-db-port TARGET_DB_PORT 155 | Database port for target system 156 | -tad TARGET_AFTER_DUMP, --target-after-dump TARGET_AFTER_DUMP 157 | Additional dump file to insert after the regular database import 158 | -op ORIGIN_PATH, --origin-path ORIGIN_PATH 159 | File path to origin database credential file depending on the framework type 160 | -on ORIGIN_NAME, --origin-name ORIGIN_NAME 161 | Providing a name for the origin system 162 | -oh ORIGIN_HOST, --origin-host ORIGIN_HOST 163 | SSH host to origin system 164 | -ou ORIGIN_USER, --origin-user ORIGIN_USER 165 | SSH user for origin system 166 | -opw ORIGIN_PASSWORD, --origin-password ORIGIN_PASSWORD 167 | SSH password for origin system 168 | -ok ORIGIN_KEY, --origin-key ORIGIN_KEY 169 | File path to SSH key for origin system 170 | -opo ORIGIN_PORT, --origin-port ORIGIN_PORT 171 | SSH port for origin system 172 | -odd ORIGIN_DUMP_DIR, --origin-dump-dir ORIGIN_DUMP_DIR 173 | Directory path for database dump file on origin system 174 | -okd ORIGIN_KEEP_DUMPS, --origin-keep-dumps ORIGIN_KEEP_DUMPS 175 | Keep dump file count for origin system 176 | -odn ORIGIN_DB_NAME, --origin-db-name ORIGIN_DB_NAME 177 | Database name for origin system 178 | -odh ORIGIN_DB_HOST, --origin-db-host ORIGIN_DB_HOST 179 | Database host for origin system 180 | -odu ORIGIN_DB_USER, --origin-db-user ORIGIN_DB_USER 181 | Database user for origin system 182 | -odpw ORIGIN_DB_PASSWORD, --origin-db-password ORIGIN_DB_PASSWORD 183 | Database password for origin system 184 | -odpo ORIGIN_DB_PORT, --origin-db-port ORIGIN_DB_PORT 185 | Database port for origin system 186 | -fpw, --force-password 187 | Force password user query 188 | -ur, --use-rsync Use rsync as transfer method 189 | -uro USE_RSYNC_OPTIONS, --use-rsync-options USE_RSYNC_OPTIONS 190 | Additional rsync options 191 | -w WHERE, --where WHERE 192 | Additional where clause for mysql dump to sync only selected rows, e.g. --where="deleted=0" 193 | -amo OPTIONS, --additional-mysqldump-options OPTIONS 194 | Additional mysqldump options for creating the database dump, e.g. --additional-mysqldump-options="--where="deleted=0" 195 | ``` 196 | 197 | If you haven't declared a path to a SSH key, during the script execution you are requested to enter the SSH password for the given user in the shell argument or the `config.json` to enable a SSH connection for the remote system. 198 | 199 | ### Import 200 | 201 | You can import the python package and use them inside your project: 202 | 203 | ```python 204 | from db_sync_tool import sync 205 | 206 | if __name__ == "__main__": 207 | sync.Sync(config={}, args*) 208 | ``` 209 | 210 | ## Configuration 211 | 212 | You can configure the script with [shell arguments](#shell-arguments) or using a separate configuration file. 213 | 214 | ### Configuration File 215 | 216 | The `config.json` contains important information about the origin and the target system. In dependence on the given configuration the [synchronisation mode](docs/MODE.md) is implicitly selected. 217 | 218 | Example structure of a `config.yml` for a Symfony system in receiver mode (`path` defines the location of the Symfony database configuration file): 219 | ```yaml 220 | type: Symfony 221 | origin: 222 | host: 192.87.33.123 223 | user: ssh_demo_user 224 | path: /var/www/html/project/shared/.env 225 | target: 226 | path: /var/www/html/app/.env 227 | ``` 228 | 229 | It is possible to adjust the `config.yml` [configuration](docs/CONFIG.md). 230 | 231 | ## File sync 232 | 233 | There is an addon script available to sync files to. Use the [file-sync-tool](https://github.com/jackd248/file-sync-tool) to easily transfer files between origin and target system. 234 | 235 | ## Release Guide 236 | 237 | A detailed guide is available to release a new version. See [here](docs/RELEASE.md). 238 | 239 | ## Tests 240 | 241 | A docker container set up is available for testing purpose. See [here](tests/README.md). 242 | 243 | ## Support 244 | 245 | If you like the project, feel free to support the development. 246 | 247 | Buy Me A Coffee 248 | -------------------------------------------------------------------------------- /db_sync_tool/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | Main script 6 | """ 7 | 8 | import argparse 9 | import os 10 | import sys 11 | from collections import defaultdict 12 | 13 | # Workaround for ModuleNotFoundError 14 | sys.path.append(os.getcwd()) 15 | from db_sync_tool import sync 16 | from db_sync_tool.utility import helper 17 | 18 | 19 | def main(args=None): 20 | """ 21 | Main entry point for the command line. Parse the arguments and call to the main process. 22 | :param args: 23 | :return: 24 | """ 25 | if args is None: 26 | args = {} 27 | 28 | args = get_arguments(args) 29 | sync.Sync( 30 | config_file=args.config_file, 31 | verbose=args.verbose, 32 | yes=args.yes, 33 | mute=args.mute, 34 | dry_run=args.dry_run, 35 | import_file=args.import_file, 36 | dump_name=args.dump_name, 37 | keep_dump=args.keep_dump, 38 | host_file=args.host_file, 39 | clear=args.clear_database, 40 | force_password=args.force_password, 41 | use_rsync=args.use_rsync, 42 | use_rsync_options=args.use_rsync_options, 43 | reverse=args.reverse, 44 | args=args 45 | ) 46 | 47 | 48 | def get_arguments(args): 49 | """ 50 | Parses and returns script arguments 51 | :param args: 52 | :return: 53 | """ 54 | parser = argparse.ArgumentParser(prog='db_sync_tool', 55 | description='A tool for automatic database synchronization from ' 56 | 'and to host systems.') 57 | parser.add_argument('origin', 58 | help='Origin database defined in host file', 59 | nargs='?', 60 | type=str) 61 | parser.add_argument('target', 62 | help='Target database defined in host file', 63 | nargs='?', 64 | type=str) 65 | parser.add_argument('-f', '--config-file', 66 | help='Path to configuration file', 67 | required=False, 68 | type=str) 69 | parser.add_argument('-v', '--verbose', 70 | help='Enable extended console output', 71 | required=False, 72 | action='store_true') 73 | parser.add_argument('-y', '--yes', 74 | help='Skipping user confirmation for database import', 75 | required=False, 76 | action='store_true') 77 | parser.add_argument('-m', '--mute', 78 | help='Mute console output', 79 | required=False, 80 | action='store_true') 81 | parser.add_argument('-dr', '--dry-run', 82 | help='Testing process without running database export, transfer or import.', 83 | required=False, 84 | action='store_true') 85 | parser.add_argument('-i', '--import-file', 86 | help='Import database from a specific file dump', 87 | required=False, 88 | type=str) 89 | parser.add_argument('-dn', '--dump-name', 90 | help='Set a specific dump file name (default is "_[dbname]_[date]")', 91 | required=False, 92 | type=str) 93 | parser.add_argument('-kd', '--keep-dump', 94 | help='Skipping target import of the database dump and saving the available dump file in the ' 95 | 'given directory', 96 | required=False, 97 | type=str) 98 | parser.add_argument('-o', '--host-file', 99 | help='Using an additional hosts file for merging hosts information with the configuration file', 100 | required=False, 101 | type=str) 102 | parser.add_argument('-l', '--log-file', 103 | help='File path for creating a additional log file', 104 | required=False, 105 | type=str) 106 | parser.add_argument('-cd', '--clear-database', 107 | help='Dropping all tables before importing a new sync to get a clean database.', 108 | required=False, 109 | action='store_true') 110 | parser.add_argument('-ta', '--tables', 111 | help='Defining specific tables to export, e.g. --tables=table1,table2', 112 | required=False, 113 | type=str) 114 | parser.add_argument('-r', '--reverse', 115 | help='Reverse origin and target hosts', 116 | required=False, 117 | action='store_true') 118 | parser.add_argument('-t', '--type', 119 | help='Defining the framework type [TYPO3, Symfony, Drupal, Wordpress]', 120 | required=False, 121 | type=str) 122 | parser.add_argument('-tp', '--target-path', 123 | help='File path to target database credential file depending on the framework type', 124 | required=False, 125 | type=str) 126 | parser.add_argument('-tn', '--target-name', 127 | help='Providing a name for the target system', 128 | required=False, 129 | type=str) 130 | parser.add_argument('-th', '--target-host', 131 | help='SSH host to target system', 132 | required=False, 133 | type=str) 134 | parser.add_argument('-tu', '--target-user', 135 | help='SSH user for target system', 136 | required=False, 137 | type=str) 138 | parser.add_argument('-tpw', '--target-password', 139 | help='SSH password for target system', 140 | required=False, 141 | type=str) 142 | parser.add_argument('-tk', '--target-key', 143 | help='File path to SSH key for target system', 144 | required=False, 145 | type=str) 146 | parser.add_argument('-tpo', '--target-port', 147 | help='SSH port for target system', 148 | required=False, 149 | type=int) 150 | parser.add_argument('-tdd', '--target-dump-dir', 151 | help='Directory path for database dump file on target system', 152 | required=False, 153 | type=str) 154 | parser.add_argument('-tkd', '--target-keep-dumps', 155 | help='Keep dump file count for target system', 156 | required=False, 157 | type=int) 158 | parser.add_argument('-tdn', '--target-db-name', 159 | help='Database name for target system', 160 | required=False, 161 | type=str) 162 | parser.add_argument('-tdh', '--target-db-host', 163 | help='Database host for target system', 164 | required=False, 165 | type=str) 166 | parser.add_argument('-tdu', '--target-db-user', 167 | help='Database user for target system', 168 | required=False, 169 | type=str) 170 | parser.add_argument('-tdpw', '--target-db-password', 171 | help='Database password for target system', 172 | required=False, 173 | type=str) 174 | parser.add_argument('-tdpo', '--target-db-port', 175 | help='Database port for target system', 176 | required=False, 177 | type=int) 178 | parser.add_argument('-tad', '--target-after-dump', 179 | help='Additional dump file to insert after the regular database import', 180 | required=False, 181 | type=str) 182 | parser.add_argument('-op', '--origin-path', 183 | help='File path to origin database credential file depending on the framework type', 184 | required=False, 185 | type=str) 186 | parser.add_argument('-on', '--origin-name', 187 | help='Providing a name for the origin system', 188 | required=False, 189 | type=str) 190 | parser.add_argument('-oh', '--origin-host', 191 | help='SSH host to origin system', 192 | required=False, 193 | type=str) 194 | parser.add_argument('-ou', '--origin-user', 195 | help='SSH user for origin system', 196 | required=False, 197 | type=str) 198 | parser.add_argument('-opw', '--origin-password', 199 | help='SSH password for origin system', 200 | required=False, 201 | type=str) 202 | parser.add_argument('-ok', '--origin-key', 203 | help='File path to SSH key for origin system', 204 | required=False, 205 | type=str) 206 | parser.add_argument('-opo', '--origin-port', 207 | help='SSH port for origin system', 208 | required=False, 209 | type=int) 210 | parser.add_argument('-odd', '--origin-dump-dir', 211 | help='Directory path for database dump file on origin system', 212 | required=False, 213 | type=str) 214 | parser.add_argument('-okd', '--origin-keep-dumps', 215 | help='Keep dump file count for origin system', 216 | required=False, 217 | type=int) 218 | parser.add_argument('-odn', '--origin-db-name', 219 | help='Database name for origin system', 220 | required=False, 221 | type=str) 222 | parser.add_argument('-odh', '--origin-db-host', 223 | help='Database host for origin system', 224 | required=False, 225 | type=str) 226 | parser.add_argument('-odu', '--origin-db-user', 227 | help='Database user for origin system', 228 | required=False, 229 | type=str) 230 | parser.add_argument('-odpw', '--origin-db-password', 231 | help='Database password for origin system', 232 | required=False, 233 | type=str) 234 | parser.add_argument('-odpo', '--origin-db-port', 235 | help='Database port for origin system', 236 | required=False, 237 | type=int) 238 | parser.add_argument('-fpw', '--force-password', 239 | help='Force password user query', 240 | required=False, 241 | action='store_true') 242 | parser.add_argument('-ur', '--use-rsync', 243 | help='Use rsync as transfer method', 244 | required=False, 245 | action='store_true') 246 | parser.add_argument('-uro', '--use-rsync-options', 247 | help='Additional rsync options', 248 | required=False, 249 | type=str) 250 | parser.add_argument('-w', '--where', 251 | help='Additional where clause for mysql dump to sync only selected rows', 252 | required=False, 253 | type=str) 254 | parser.add_argument('-amo', '--additional-mysqldump-options', 255 | help='Additional mysqldump options for creating the database dump, e.g. --additional-mysqldump-options="--where="deleted=0"', 256 | required=False, 257 | type=str) 258 | 259 | return parser.parse_args(helper.dict_to_args(args)) 260 | 261 | 262 | if __name__ == "__main__": 263 | main() 264 | -------------------------------------------------------------------------------- /docs/CONFIG.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | Here you can find an overview over the possible configuration adjustments. 4 | 5 | - [Full configuration reference](#configuration_reference) 6 | - [Ignore tables](#ignore_tables) 7 | - [Authentication](#authentication) 8 | - [Linking hosts](#linking) 9 | - [SSH Port](#port) 10 | - [Console commands](#console) 11 | - [(Temporary) dump directory](#directory) 12 | - [Before and after script](#script) 13 | - [After dump](#after-dump) 14 | - [Logging](#logging) 15 | - [Cleaning up / keeping dumps count](#clean_up) 16 | - [Naming hosts](#naming) 17 | - [Check dump](#check) 18 | - [Manual database credentials](#manual) 19 | - [Clearing database](#clear) 20 | - [Protect host](#protect) 21 | - [Reverse hosts](#reverse) 22 | - [Jump host](#jump_host) 23 | 24 | 25 | ### Full configuration reference 26 | 27 | Here you can find the full configuration reference for a `config.yml`: 28 | 29 | ```yaml 30 | # Application type: TYPO3 [ Symfony | Drupal | Wordpress | Laravel 31 | # Isn't necessary if the database credentials are provided manually 32 | type: 33 | # Database source system 34 | origin: 35 | # Just informative for logging, e.g. prod 36 | name: 37 | # Full path to the application file, which contains the necessary database credentials 38 | path: 39 | # For reusability reasons you can store the host information in an additional hosts.yml file and link the needed entry here 40 | # See the section "Linking hosts" for more information 41 | # e.g. hosts.yml@prod 42 | link: 43 | # SSH host 44 | host: 45 | # SSH user 46 | user: 47 | # SSH port (default: 22) 48 | port: 49 | # SSh password (is not recommended to store the password here, use the interactive prompt or a ssh key instead) 50 | password: 51 | # SSH key path 52 | ssh_key: 53 | # Provide an optional configuration for a SSH jump host 54 | jump_host: 55 | # SSH jump host host 56 | host: 57 | # SSH jump host private ip, necessary for session channel 58 | private: 59 | # SSH jump host user (default: origin SSH user) 60 | user: 61 | # SSH jump host port (default: origin SSH port) 62 | port: 63 | # Just informative for logging, e.g. prod 64 | name: 65 | # Temporary or finally dump file directory (default: /tmp/) 66 | dump_dir: 67 | # Manual database credentials 68 | db: 69 | # Database name 70 | name: 71 | # Database host 72 | host: 73 | # Database password 74 | password: 75 | # Database user 76 | user: 77 | # Database port (default: 3306) 78 | port: 79 | # Additional console scripts 80 | script: 81 | # Script before synchronisation on origin system 82 | before: 83 | # Script after synchronisation on origin system 84 | after: 85 | # Script in failure case 86 | error: 87 | # Additional console path variables 88 | console: 89 | # If a command variable is not available via the standard path, you can define a divergent path 90 | # e.g. php: /usr/bin/php 91 | # Database target system 92 | target: 93 | # Just informative for logging, e.g. prod 94 | name: 95 | # Full path to the application file, which contains the necessary database credentials 96 | path: 97 | # For reusability reasons you can store the host information in an additional hosts.yml file and link the needed entry here 98 | # See the section "Linking hosts" for more information 99 | # e.g. hosts.yml@prod 100 | link: 101 | # SSH host 102 | host: 103 | # SSH user 104 | user: 105 | # SSH port (default: 22) 106 | port: 107 | # SSh password (is not recommended to store the password here, use the interactive prompt or a ssh key instead) 108 | password: 109 | # SSH key path 110 | ssh_key: 111 | # Provide an optional configuration for a SSH jump host 112 | jump_host: 113 | # SSH jump host host 114 | host: 115 | # SSH jump host private ip, necessary for session channel 116 | private: 117 | # SSH jump host user (default: target SSH user) 118 | user: 119 | # SSH jump host port (default: target SSH port) 120 | port: 121 | # Just informative for logging, e.g. prod 122 | name: 123 | # Temporary or finally dump file directory (default: /tmp/) 124 | dump_dir: 125 | # Lock the host against the import of a database dump 126 | protect: true 127 | # Manual database credentials 128 | db: 129 | # Database name 130 | name: 131 | # Database host 132 | host: 133 | # Database password 134 | password: 135 | # Database user 136 | user: 137 | # Database port (default: 3306) 138 | port: 139 | # Additional console scripts 140 | script: 141 | # Script before synchronisation on target system 142 | before: 143 | # Script after synchronisation on target system 144 | after: 145 | # Script in failure case 146 | error: 147 | # Additional console path variables 148 | console: 149 | # If a command variable is not available via the standard path, you can define a divergent path 150 | # e.g. php: /usr/bin/php 151 | # Define the backup clean up functionality and defines how many dumps will be keep depending on time 152 | keep_dumps: 153 | # Path to an additional dump file, which will be imported after the synchronisation finished 154 | # e.g. /path/to/dump/file.sql 155 | after_dump: 156 | # Additional sql commands after the import 157 | post_sql: 158 | - 159 | # Path to an additional log file 160 | log_file: 161 | # List of tables to ignore for the synchronisation 162 | ignore_table: [] 163 | # List of tables to truncate before the synchronisation 164 | truncate_table: [] 165 | # Disable the check dump feature, to verify the completeness of the created dump file (default: true) 166 | check_dump: 167 | # Additional console scripts 168 | script: 169 | # Script before synchronisation 170 | before: 171 | # Script after synchronisation 172 | after: 173 | # Script in failure case 174 | error: 175 | ``` 176 | 177 | 178 | > The config file can be written in `yaml` or `json`. 179 | 180 | 181 | ### Ignore/truncate tables 182 | 183 | Often it is better to exclude some tables from the sql dump for performance reasons, e.g. caching tables. Specify them as comma separated list in the `ignore_table` array. 184 | 185 | You can use wildcards to define several tables: 186 | 187 | ```yaml 188 | ignore_tables: 189 | - cache_* 190 | ``` 191 | 192 | There is also an option to truncate tables before the import with listing shown as below: 193 | ```yaml 194 | truncate_tables: 195 | - cache_* 196 | ``` 197 | 198 | 199 | ### Authentication 200 | 201 | There a different ways to authenticate against remote systems. 202 | 203 | ### SSH key 204 | 205 | Without any option, the db_sync_tool tries to authenticate with a running ssh agent. 206 | 207 | If you want to authenticate with a specific private ssh key instead of a user entered password to the server (useful for CI/CD), you can add the file path to the private key file in your `config.json`: 208 | 209 | ```json 210 | { 211 | "origin": { 212 | "ssh_key": "/home/bob/.ssh/id_rsa" 213 | } 214 | } 215 | ``` 216 | 217 | ### SSH password 218 | 219 | It's not recommended, but you can also specify the plain password inside the host configuration in the `config.json`: 220 | 221 | ```json 222 | { 223 | "origin": { 224 | "password": "1234" 225 | } 226 | } 227 | ``` 228 | 229 | If no options are provided so far (no ssh agent, ssh key, defined password), a prompt is displayed to enter the necessary password for the ssh authentication. You can also force the user input by adding the `--force-password` / `-fpw` option to the script call. 230 | 231 | 232 | ### Linking hosts 233 | 234 | For larger project setups with multiple configuration files, it's better to reuse the host configuration in every sync scenario. So you can link to predefined host in your `config.json`: 235 | 236 | ```json 237 | { 238 | "origin": { 239 | "link": "@prod" 240 | }, 241 | "target": { 242 | "link": "@dev" 243 | } 244 | } 245 | ``` 246 | 247 | You specify the path to the `hosts.json` file with the `-o` option within the script call. The `hosts.json` should look like this: 248 | 249 | ```json 250 | { 251 | "prod": { 252 | "host": "host", 253 | "user": "user", 254 | "path": "/var/www/html/project/shared/typo3conf/LocalConfiguration.php" 255 | }, 256 | "dev": { 257 | "host": "host", 258 | "user": "user", 259 | "path": "/var/www/html/project/shared/typo3conf/LocalConfiguration.php" 260 | } 261 | } 262 | ``` 263 | 264 | 265 | 266 | ### SSH Port 267 | 268 | You can also specify a different SSH port to the client in your `config.json` (the default port is `22`): 269 | 270 | ```json 271 | { 272 | "origin": { 273 | "port": "1234" 274 | } 275 | } 276 | ``` 277 | 278 | 279 | ### Console commands 280 | 281 | The script using among other things the `php`, `mysql`, `mysqldump`, `grep` commands to synchronize the databases. Sometimes these commands are not available via the path variable, so you have to specify the full path to the source in the `config.json` depending on the system: 282 | 283 | ```json 284 | { 285 | "origin": { 286 | "console": { 287 | "php": "/usr/bin/php", 288 | "mysql": "/usr/bin/mysql", 289 | "mysqldump": "/usr/bin/mysqldump" 290 | } 291 | } 292 | } 293 | ``` 294 | 295 | 296 | ### (Temporary) dump directory 297 | 298 | Normally is the script creating the sql dump in the `/tmp/` directory. If this directory is not writable or you want to export the database automatically in another directory, you can specify an alternative directory in the `config.json`, where the temporary sql dump will be saved: 299 | 300 | ```json 301 | { 302 | "origin": { 303 | "dump_dir": "/path/to/writable/dir/" 304 | } 305 | } 306 | ``` 307 | 308 | **Note:** It is recommended to use for every application another directory to avoid side effects (e.g. cleaning up feature). 309 | 310 | 311 | ### Before and after script 312 | 313 | Sometimes it is necessary to run a specific command before or after the dump creation on the origin or target system to ensure the correct synchronisation process. Therefore you can specify these commands in the `config.json`: 314 | 315 | ```json 316 | { 317 | "script": { 318 | "before": "", 319 | "after": "", 320 | "error": "" 321 | }, 322 | "origin": { 323 | "script": { 324 | "before": "", 325 | "after": "", 326 | "error": "" 327 | }, 328 | }, 329 | "target": { 330 | "script": { 331 | "before": "", 332 | "after": "", 333 | "error": "" 334 | }, 335 | } 336 | } 337 | ``` 338 | 339 | 340 | ### After dump 341 | 342 | It is possible to provide an additional dump file, which will be imported after the regular database import is finished. You can specify the path to the `after_dump` file of the target host in the `config.yml`: 343 | 344 | ```yaml 345 | target: 346 | after_dump: path/to/dump/file.sql 347 | ``` 348 | 349 | Alternatively you can define post sql commands, which will be executed in the 350 | 351 | ```yaml 352 | target: 353 | post_sql: 354 | - UPDATE sys_domain SET hidden = 1; 355 | ``` 356 | 357 | 358 | ### Logging 359 | 360 | You can enable the logging to a separate log file via the `log_file` entry in the `config.json`: 361 | 362 | ```json 363 | { 364 | "log_file": "/path/to/file/info.log" 365 | } 366 | ``` 367 | 368 | **Note**: By default only a summary of the sync actions will be logged. If you enable the verbose option (`-v`) all console output will also be logged in the given log file. 369 | 370 | 371 | ### Cleaning up / keeping dumps count 372 | 373 | With the concept of the *DUMP_REMOTE* or *DUMP_LOCAL* mode can you implement an automatic backup system. However it's a good option to clean up old dump files and only keep the newest ones. Therefore you can use the `keep_dumps` entry in the `config.json`: 374 | 375 | ```json 376 | { 377 | "origin": { 378 | "dump_dir": "/path/to/writable/dir/", 379 | "keep_dumps": 5 380 | } 381 | } 382 | ``` 383 | 384 | **Note**: Be aware of this feature. It will only keep the latest (e.g. 5) files in the `dump_dir` directory and delete all other `.sql` and `.tar.gz` files. 385 | 386 | 387 | ### Naming hosts 388 | 389 | For a better differentiation of the different host systems you can optionally provide a specific name in the `config.json`: 390 | 391 | ```json 392 | { 393 | "origin": { 394 | "name": "Prod" 395 | }, 396 | "target": { 397 | "name": "Stage" 398 | } 399 | } 400 | ``` 401 | 402 | 403 | ### Check dump 404 | 405 | The script is checking the target dump if the file is being downloaded completely. If you want to prevent this check, you can disable them in the `config.json`: 406 | 407 | ```json 408 | { 409 | "check_dump": false 410 | } 411 | ``` 412 | 413 | 414 | ### Manual database credentials 415 | 416 | It is also possible to skip the automatic database credential detection depending on the framework and provide the database credentials by your own in the `config.json` (example for RECEIVER mode): 417 | 418 | ```json 419 | { 420 | "name": "project", 421 | "target": { 422 | "db": { 423 | "name": "db", 424 | "host": "db2", 425 | "password": "db", 426 | "user": "db", 427 | "port": 3306 428 | } 429 | }, 430 | "origin": { 431 | "host": "www1", 432 | "user": "user", 433 | "password": "password", 434 | "db": { 435 | "name": "db", 436 | "host": "db1", 437 | "password": "db", 438 | "user": "db", 439 | "port": 3306 440 | } 441 | }, 442 | "ignore_table": [] 443 | } 444 | ``` 445 | 446 | 447 | ### Clearing database 448 | 449 | If you want a clean database sync, it is necessary to drop all existing tables of the target database. Use the `--clear-database` option (`-cd`) for this. 450 | 451 | 452 | ### Protect host 453 | 454 | You can declare a protected to host to prevent an unintentional import on this system. 455 | 456 | ```yaml 457 | type: TYPO3 458 | origin: 459 | host: 460 | user: 461 | path: 462 | name: Demo Prod 463 | protect: true 464 | target: 465 | path: 466 | ``` 467 | 468 | 469 | 470 | ### Reverse hosts 471 | 472 | You can easily reverse the declared origin and target hosts with the `--reverse` argument: 473 | 474 | ```yaml 475 | $ db_sync_tool -f config.yml --reverse 476 | ``` 477 | 478 | 479 | ### Jump host 480 | 481 | You can define an optionally SSH jump host, for accessing protected server. Unfortunately this can not be read out of the `.ssh/config`, so you have to specify them as follows: 482 | ```yaml 483 | origin: 484 | jump_host: 485 | host: 486 | private: 487 | user: 488 | port: 489 | name: Demo Jump Host 490 | ``` 491 | 492 | The `host` entry is the public ip address, the `private` entry is the private ip address (can be determined by `hostname -I` or `ip addr` or `ifconfig`). 493 | 494 | The entries for `user` (as default the parent/origin SSH user is used), `port` (as default the parent/origin SSH port is used) and `name` are optional. -------------------------------------------------------------------------------- /db_sync_tool/utility/system.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: future_fstrings -*- 3 | 4 | """ 5 | System module 6 | """ 7 | 8 | import sys 9 | import json 10 | import os 11 | import getpass 12 | import yaml 13 | from db_sync_tool.utility import log, parser, mode, helper, output, validation 14 | from db_sync_tool.remote import utility as remote_utility 15 | 16 | # 17 | # GLOBALS 18 | # 19 | 20 | config = { 21 | 'verbose': False, 22 | 'mute': False, 23 | 'dry_run': False, 24 | 'keep_dump': False, 25 | 'dump_name': '', 26 | 'import': '', 27 | 'link_hosts': '', 28 | 'default_origin_dump_dir': True, 29 | 'default_target_dump_dir': True, 30 | 'check_dump': True, 31 | 'is_same_client': False, 32 | 'config_file_path': None, 33 | 'clear_database': False, 34 | 'force_password': False, 35 | 'use_rsync': False, 36 | 'use_rsync_options': None, 37 | 'use_sshpass': False, 38 | 'ssh_agent': False, 39 | 'ssh_password': { 40 | mode.Client.ORIGIN: None, 41 | mode.Client.TARGET: None 42 | }, 43 | 'link_target': None, 44 | 'link_origin': None, 45 | 'tables': '', 46 | 'where': '', 47 | 'additional_mysqldump_options': '' 48 | } 49 | 50 | # 51 | # DEFAULTS 52 | # 53 | 54 | default_local_sync_path = '/tmp/db_sync_tool/' 55 | 56 | 57 | # 58 | # FUNCTIONS 59 | # 60 | 61 | def check_target_configuration(): 62 | """ 63 | Checking target database configuration 64 | :return: 65 | """ 66 | parser.get_database_configuration(mode.Client.TARGET) 67 | 68 | 69 | def get_configuration(host_config, args = {}): 70 | """ 71 | Checking configuration information by file or dictionary 72 | :param host_config: Dictionary 73 | :param args: Dictionary 74 | :return: 75 | """ 76 | global config 77 | config[mode.Client.TARGET] = {} 78 | config[mode.Client.ORIGIN] = {} 79 | 80 | if host_config: 81 | if type(host_config) is dict: 82 | config.update(__m=host_config) 83 | else: 84 | config.update(__m=json.dumps(obj=host_config)) 85 | 86 | _config_file_path = config['config_file_path'] 87 | if not _config_file_path is None: 88 | if os.path.isfile(_config_file_path): 89 | with open(_config_file_path, 'r') as read_file: 90 | if _config_file_path.endswith('.json'): 91 | config.update(json.load(read_file)) 92 | elif _config_file_path.endswith('.yaml') or _config_file_path.endswith('.yml'): 93 | config.update(yaml.safe_load(read_file)) 94 | else: 95 | sys.exit( 96 | output.message( 97 | output.Subject.ERROR, 98 | f'Unsupported configuration file type [json,yml,yaml]: ' 99 | f'{config["config_file_path"]}', 100 | False 101 | ) 102 | ) 103 | output.message( 104 | output.Subject.LOCAL, 105 | f'Loading host configuration ' 106 | f'{output.CliFormat.BLACK}{_config_file_path}{output.CliFormat.ENDC}', 107 | True 108 | ) 109 | else: 110 | sys.exit( 111 | output.message( 112 | output.Subject.ERROR, 113 | f'Local configuration not found: {config["config_file_path"]}', 114 | False 115 | ) 116 | ) 117 | 118 | # workaround for argument order handling respecting the linking feature 119 | build_config(args, True) 120 | link_configuration_with_hosts() 121 | build_config(args) 122 | 123 | validation.check(config) 124 | check_options() 125 | 126 | if not config[mode.Client.TARGET] and not config[mode.Client.ORIGIN]: 127 | sys.exit( 128 | output.message( 129 | output.Subject.ERROR, 130 | f'Configuration is missing, use a separate file or provide host parameter', 131 | False 132 | ) 133 | ) 134 | helper.run_script(script='before') 135 | log.get_logger().info('Starting db_sync_tool') 136 | 137 | 138 | def build_config(args, pre_run = False): 139 | """ 140 | ADding the provided arguments 141 | :param args: 142 | :param pre_run: 143 | :return: 144 | """ 145 | if args is None or not args: 146 | return {} 147 | 148 | if not args.type is None: 149 | config['type'] = args.type 150 | 151 | if not args.tables is None: 152 | config['tables'] = args.tables 153 | 154 | if not args.origin is None: 155 | config['link_origin'] = args.origin 156 | 157 | if not args.target is None: 158 | config['link_target'] = args.target 159 | 160 | # for order reasons check just the link arguments 161 | if pre_run: return 162 | 163 | if not args.target_path is None: 164 | config[mode.Client.TARGET]['path'] = args.target_path 165 | 166 | if not args.target_name is None: 167 | config[mode.Client.TARGET]['name'] = args.target_name 168 | 169 | if not args.target_host is None: 170 | config[mode.Client.TARGET]['host'] = args.target_host 171 | 172 | if not args.target_user is None: 173 | config[mode.Client.TARGET]['user'] = args.target_user 174 | 175 | if not args.target_password is None: 176 | config[mode.Client.TARGET]['password'] = args.target_password 177 | 178 | if not args.target_key is None: 179 | config[mode.Client.TARGET]['ssh_key'] = args.target_key 180 | 181 | if not args.target_port is None: 182 | config[mode.Client.TARGET]['port'] = args.target_port 183 | 184 | if not args.target_dump_dir is None: 185 | config[mode.Client.TARGET]['dump_dir'] = args.target_dump_dir 186 | 187 | if not args.target_db_name is None: 188 | check_config_dict_key(mode.Client.TARGET, 'db') 189 | config[mode.Client.TARGET]['db']['name'] = args.target_db_name 190 | 191 | if not args.target_db_host is None: 192 | check_config_dict_key(mode.Client.TARGET, 'db') 193 | config[mode.Client.TARGET]['db']['host'] = args.target_db_host 194 | 195 | if not args.target_db_user is None: 196 | check_config_dict_key(mode.Client.TARGET, 'db') 197 | config[mode.Client.TARGET]['db']['user'] = args.target_db_user 198 | 199 | if not args.target_db_password is None: 200 | check_config_dict_key(mode.Client.TARGET, 'db') 201 | config[mode.Client.TARGET]['db']['password'] = args.target_db_password 202 | 203 | if not args.target_db_port is None: 204 | check_config_dict_key(mode.Client.TARGET, 'db') 205 | config[mode.Client.TARGET]['db']['port'] = args.target_db_port 206 | 207 | if not args.target_after_dump is None: 208 | config[mode.Client.TARGET]['after_dump'] = args.target_after_dump 209 | 210 | if not args.origin_path is None: 211 | config[mode.Client.ORIGIN]['path'] = args.origin_path 212 | 213 | if not args.origin_name is None: 214 | config[mode.Client.ORIGIN]['name'] = args.origin_name 215 | 216 | if not args.origin_host is None: 217 | config[mode.Client.ORIGIN]['host'] = args.origin_host 218 | 219 | if not args.origin_user is None: 220 | config[mode.Client.ORIGIN]['user'] = args.origin_user 221 | 222 | if not args.origin_password is None: 223 | config[mode.Client.ORIGIN]['password'] = args.origin_password 224 | 225 | if not args.origin_key is None: 226 | config[mode.Client.ORIGIN]['ssh_key'] = args.origin_key 227 | 228 | if not args.origin_port is None: 229 | config[mode.Client.ORIGIN]['port'] = args.origin_port 230 | 231 | if not args.origin_dump_dir is None: 232 | config[mode.Client.ORIGIN]['dump_dir'] = args.origin_dump_dir 233 | 234 | if not args.origin_db_name is None: 235 | check_config_dict_key(mode.Client.ORIGIN, 'db') 236 | config[mode.Client.ORIGIN]['db']['name'] = args.origin_db_name 237 | 238 | if not args.origin_db_host is None: 239 | check_config_dict_key(mode.Client.ORIGIN, 'db') 240 | config[mode.Client.ORIGIN]['db']['host'] = args.origin_db_host 241 | 242 | if not args.origin_db_user is None: 243 | check_config_dict_key(mode.Client.ORIGIN, 'db') 244 | config[mode.Client.ORIGIN]['db']['user'] = args.origin_db_user 245 | 246 | if not args.origin_db_password is None: 247 | check_config_dict_key(mode.Client.ORIGIN, 'db') 248 | config[mode.Client.ORIGIN]['db']['password'] = args.origin_db_password 249 | 250 | if not args.origin_db_port is None: 251 | check_config_dict_key(mode.Client.ORIGIN, 'db') 252 | config[mode.Client.ORIGIN]['db']['port'] = args.origin_db_port 253 | 254 | if not args.where is None: 255 | config['where'] = args.where 256 | 257 | if not args.additional_mysqldump_options is None: 258 | config['additional_mysqldump_options'] = args.additional_mysqldump_options 259 | 260 | return config 261 | 262 | 263 | def check_options(): 264 | """ 265 | Checking configuration provided file 266 | :return: 267 | """ 268 | global config 269 | if 'dump_dir' in config[mode.Client.ORIGIN]: 270 | config['default_origin_dump_dir'] = False 271 | 272 | if 'dump_dir' in config[mode.Client.TARGET]: 273 | config['default_target_dump_dir'] = False 274 | 275 | if 'check_dump' in config: 276 | config['check_dump'] = config['check_dump'] 277 | 278 | reverse_hosts() 279 | mode.check_sync_mode() 280 | 281 | 282 | def check_authorizations(): 283 | """ 284 | Checking authorization for clients 285 | :return: 286 | """ 287 | check_authorization(mode.Client.ORIGIN) 288 | check_authorization(mode.Client.TARGET) 289 | 290 | 291 | def check_authorization(client): 292 | """ 293 | Checking arguments and fill options array 294 | :param client: String 295 | :return: 296 | """ 297 | # only need authorization if client is remote 298 | if mode.is_remote(client): 299 | # Workaround if no authorization is needed 300 | if (mode.get_sync_mode() == mode.SyncMode.DUMP_REMOTE and 301 | client == mode.Client.TARGET) or \ 302 | (mode.get_sync_mode() == mode.SyncMode.DUMP_LOCAL and 303 | client == mode.Client.ORIGIN) or \ 304 | (mode.get_sync_mode() == mode.SyncMode.IMPORT_REMOTE and 305 | client == mode.Client.ORIGIN): 306 | return 307 | 308 | # ssh key authorization 309 | if config['force_password']: 310 | config[client]['password'] = get_password_by_user(client) 311 | elif 'ssh_key' in config[client]: 312 | _ssh_key = config[client]['ssh_key'] 313 | if not os.path.isfile(_ssh_key): 314 | sys.exit( 315 | output.message( 316 | output.Subject.ERROR, 317 | f'SSH {client} private key not found: {_ssh_key}', 318 | False 319 | ) 320 | ) 321 | elif 'password' in config[client]: 322 | config[client]['password'] = config[client]['password'] 323 | elif remote_utility.check_keys_from_ssh_agent(): 324 | config['ssh_agent'] = True 325 | else: 326 | # user input authorization 327 | config[client]['password'] = get_password_by_user(client) 328 | 329 | if mode.get_sync_mode() == mode.SyncMode.DUMP_REMOTE and \ 330 | client == mode.Client.ORIGIN and 'password' in \ 331 | config[mode.Client.ORIGIN]: 332 | config[mode.Client.TARGET]['password'] = config[mode.Client.ORIGIN]['password'] 333 | 334 | 335 | def get_password_by_user(client): 336 | """ 337 | Getting password by user input 338 | :param client: String 339 | :return: String password 340 | """ 341 | _password = getpass.getpass( 342 | output.message( 343 | output.Subject.INFO, 344 | 'SSH password ' + helper.get_ssh_host_name(client, True) + ': ', 345 | False 346 | ) 347 | ) 348 | 349 | while _password.strip() == '': 350 | output.message( 351 | output.Subject.WARNING, 352 | 'Password seems to be empty. Please enter a valid password.', 353 | True 354 | ) 355 | 356 | _password = getpass.getpass( 357 | output.message( 358 | output.Subject.INFO, 359 | 'SSH password ' + helper.get_ssh_host_name(client, True) + ': ', 360 | False 361 | ) 362 | ) 363 | 364 | return _password 365 | 366 | 367 | def check_args_options(config_file=None, 368 | verbose=False, 369 | yes=False, 370 | mute=False, 371 | dry_run=False, 372 | import_file=None, 373 | dump_name=None, 374 | keep_dump=None, 375 | host_file=None, 376 | clear=False, 377 | force_password=False, 378 | use_rsync=False, 379 | use_rsync_options=None, 380 | reverse=False): 381 | """ 382 | Checking arguments and fill options array 383 | :param config_file: 384 | :param verbose: 385 | :param yes: 386 | :param mute: 387 | :param dry_run: 388 | :param import_file: 389 | :param dump_name: 390 | :param keep_dump: 391 | :param host_file: 392 | :param clear: 393 | :param force_password: 394 | :param use_rsync: 395 | :param use_rsync_options: 396 | :param reverse: 397 | :return: 398 | """ 399 | global config 400 | global default_local_sync_path 401 | 402 | if not config_file is None: 403 | config['config_file_path'] = config_file 404 | 405 | if not verbose is None: 406 | config['verbose'] = verbose 407 | 408 | if not yes is None: 409 | config['yes'] = yes 410 | 411 | if not mute is None: 412 | config['mute'] = mute 413 | 414 | if not dry_run is None: 415 | config['dry_run'] = dry_run 416 | 417 | if dry_run: 418 | output.message( 419 | output.Subject.INFO, 420 | 'Test mode: DRY RUN', 421 | True 422 | ) 423 | 424 | if not import_file is None: 425 | config['import'] = import_file 426 | 427 | if not dump_name is None: 428 | config['dump_name'] = dump_name 429 | 430 | if not host_file is None: 431 | config['link_hosts'] = host_file 432 | 433 | if not clear is None: 434 | config['clear_database'] = clear 435 | 436 | if not force_password is None: 437 | config['force_password'] = force_password 438 | 439 | if not use_rsync is None: 440 | config['use_rsync'] = use_rsync 441 | 442 | if use_rsync is True: 443 | helper.check_rsync_version() 444 | helper.check_sshpass_version() 445 | 446 | if not use_rsync_options is None: 447 | config['use_rsync_options'] = use_rsync_options 448 | 449 | if not reverse is None: 450 | config['reverse'] = reverse 451 | 452 | if not keep_dump is None: 453 | default_local_sync_path = keep_dump 454 | 455 | # Adding trailing slash if necessary 456 | if default_local_sync_path[-1] != '/': 457 | default_local_sync_path += '/' 458 | 459 | config['keep_dump'] = True 460 | output.message( 461 | output.Subject.INFO, 462 | '"Keep dump" option chosen', 463 | True 464 | ) 465 | 466 | 467 | def reverse_hosts(): 468 | """ 469 | Checking authorization for clients 470 | :return: 471 | """ 472 | if config['reverse']: 473 | _origin = config[mode.Client.ORIGIN] 474 | _target = config[mode.Client.TARGET] 475 | 476 | config[mode.Client.ORIGIN] = _target 477 | config[mode.Client.TARGET] = _origin 478 | 479 | output.message( 480 | output.Subject.INFO, 481 | 'Reverse origin and target hosts', 482 | True 483 | ) 484 | 485 | 486 | def link_configuration_with_hosts(): 487 | """ 488 | Merging the hosts definition with the given configuration file 489 | @ToDo Simplify function 490 | :return: 491 | """ 492 | if ('link' in config[mode.Client.ORIGIN] or 'link' in config[mode.Client.TARGET]) and config['link_hosts'] == '': 493 | # 494 | # Try to read host file path from link entry 495 | # 496 | _host = str(config[mode.Client.ORIGIN]['link'].split('@')[0]) if 'link' in config[mode.Client.ORIGIN] else '' 497 | _host = str(config[mode.Client.TARGET]['link'].split('@')[0]) if 'link' in config[mode.Client.TARGET] else _host 498 | 499 | config['link_hosts'] = _host 500 | 501 | if config['link_hosts'] == '': 502 | # Try to find default hosts.json file in same directory 503 | sys.exit( 504 | output.message( 505 | output.Subject.ERROR, 506 | f'Missing hosts file for linking hosts with configuration. ' 507 | f'Use the "-o" / "--hosts" argument to define the filepath for the hosts file, ' 508 | f'when using a link parameter within the configuration or define the the ' 509 | f'filepath direct in the link entry e.g. "host.yaml@entry1".', 510 | False 511 | ) 512 | ) 513 | 514 | if config['link_hosts'] != '': 515 | 516 | # Adjust filepath from relative to absolute 517 | if config['link_hosts'][0] != '/': 518 | config['link_hosts'] = os.path.dirname(os.path.abspath(config['config_file_path'])) + '/' + config['link_hosts'] 519 | 520 | if os.path.isfile(config['link_hosts']): 521 | with open(config['link_hosts'], 'r') as read_file: 522 | if config['link_hosts'].endswith('.json'): 523 | _hosts = json.load(read_file) 524 | elif config['link_hosts'].endswith('.yaml') or config['link_hosts'].endswith('.yml'): 525 | _hosts = yaml.safe_load(read_file) 526 | 527 | output.message( 528 | output.Subject.INFO, 529 | f'Linking configuration with hosts {output.CliFormat.BLACK}{config["link_hosts"]}{output.CliFormat.ENDC}', 530 | True 531 | ) 532 | if not config['config_file_path'] is None: 533 | if 'link' in config[mode.Client.ORIGIN]: 534 | _host_name = str(config[mode.Client.ORIGIN]['link']).split('@')[1] 535 | if _host_name in _hosts: 536 | config[mode.Client.ORIGIN] = {**config[mode.Client.ORIGIN], **_hosts[_host_name]} 537 | 538 | if 'link' in config[mode.Client.TARGET]: 539 | _host_name = str(config[mode.Client.TARGET]['link']).split('@')[1] 540 | if _host_name in _hosts: 541 | config[mode.Client.TARGET] = {**config[mode.Client.TARGET], **_hosts[_host_name]} 542 | else: 543 | if 'link_target' in config and 'link_origin' in config: 544 | if config['link_target'] in _hosts and config['link_origin'] in _hosts: 545 | config[mode.Client.TARGET] = _hosts[config['link_target']] 546 | config[mode.Client.ORIGIN] = _hosts[config['link_origin']] 547 | else: 548 | sys.exit( 549 | output.message( 550 | output.Subject.ERROR, 551 | f'Misconfiguration of link hosts {config["link_origin"]}, ' 552 | f'{config["link_target"]} in {config["link_hosts"]}', 553 | False 554 | ) 555 | ) 556 | else: 557 | sys.exit( 558 | output.message( 559 | output.Subject.ERROR, 560 | f'Missing link hosts for {config["link_hosts"]}', 561 | False 562 | ) 563 | ) 564 | else: 565 | sys.exit( 566 | output.message( 567 | output.Subject.ERROR, 568 | f'Local host file not found: {config["link_hosts"]}', 569 | False 570 | ) 571 | ) 572 | 573 | 574 | def check_config_dict_key(client, key): 575 | """ 576 | Create config key if is not present 577 | :param client: 578 | :param key: 579 | :return: 580 | """ 581 | if key not in config[client]: 582 | config[client][key] = {} 583 | 584 | --------------------------------------------------------------------------------