├── .editorconfig ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE.md └── workflows │ ├── intergration-tests.yml │ └── makefile.yml ├── .gitignore ├── .pylintrc ├── .python-version ├── .pyup.yml ├── .readthedocs.yml ├── .yamllint.yaml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── docs ├── Makefile ├── _static │ ├── TwinDB_Backup.png │ ├── favicon.png │ └── logo.png ├── authors.rst ├── conf.py ├── contributing.rst ├── history.rst ├── index.rst ├── installation.rst ├── make.bat ├── modules.rst ├── readme.rst ├── twindb_backup.cache.rst ├── twindb_backup.configuration.destinations.rst ├── twindb_backup.configuration.rst ├── twindb_backup.copy.rst ├── twindb_backup.destination.rst ├── twindb_backup.exporter.rst ├── twindb_backup.modifiers.rst ├── twindb_backup.rst ├── twindb_backup.source.rst ├── twindb_backup.ssh.rst ├── twindb_backup.status.rst └── usage.rst ├── omnibus ├── .gitignore ├── .kitchen.yml ├── Berksfile ├── Gemfile ├── Gemfile.lock ├── README.md ├── config │ ├── patches │ │ ├── bzip2 │ │ │ ├── makefile_no_bins.patch │ │ │ └── makefile_take_env_vars.patch │ │ └── openssl │ │ │ └── openssl-1.0.1f-do-not-build-docs.patch │ ├── projects │ │ └── twindb-backup.rb │ └── software │ │ ├── bzip2.rb │ │ ├── cacerts.rb │ │ ├── config_guess.rb │ │ ├── libedit.rb │ │ ├── libffi.rb │ │ ├── liblzma.rb │ │ ├── libpcap.rb │ │ ├── libreadline7.rb │ │ ├── libyaml.rb │ │ ├── ncurses.rb │ │ ├── openssl.rb │ │ ├── pip.rb │ │ ├── preparation.rb │ │ ├── python.rb │ │ ├── setuptools.rb │ │ ├── twindb-backup.rb │ │ ├── version-manifest.rb │ │ └── zlib.rb ├── lib │ └── ostools.rb ├── omnibus.rb ├── omnibus_build.sh └── package-scripts │ └── twindb-backup │ ├── postinst │ ├── postrm │ └── prerm ├── pyproject.toml ├── requirements.in ├── requirements.txt ├── requirements_dev.in ├── requirements_dev.txt ├── setup.cfg ├── setup.py ├── support ├── bootstrap │ ├── master │ │ ├── centos │ │ │ ├── master1.sh │ │ │ └── slave.sh │ │ └── ubuntu │ │ │ ├── master1.sh │ │ │ └── slave.sh │ └── storage_server.sh ├── make_release.py ├── twindb-backup.cfg ├── twindb-backup.cron └── twindb-backup.spec ├── tests ├── __init__.py ├── development │ ├── __init__.py │ ├── conftest.py │ ├── destination │ │ └── gcs │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── test_create_bucket.py │ │ │ ├── test_delete.py │ │ │ ├── test_delete_bucket.py │ │ │ ├── test_get_stream.py │ │ │ ├── test_gs_client.py │ │ │ ├── test_list_files.py │ │ │ ├── test_read_status.py │ │ │ ├── test_upload.py │ │ │ └── test_write_status.py │ ├── source │ │ ├── __init__.py │ │ ├── base_source │ │ │ ├── __init__.py │ │ │ └── test_suffix.py │ │ ├── binlog_source │ │ │ ├── __init__.py │ │ │ └── binlog_parser │ │ │ │ ├── __init__.py │ │ │ │ ├── test_created_at.py │ │ │ │ └── test_name.py │ │ ├── mysql_source │ │ │ ├── __init__.py │ │ │ ├── test_apply_retention_policy.py │ │ │ ├── test_disable_wsrep_desync.py │ │ │ ├── test_enable_wsrep_desync.py │ │ │ ├── test_get_binlog_coordinates.py │ │ │ ├── test_get_connection.py │ │ │ ├── test_get_lsn.py │ │ │ ├── test_get_name.py │ │ │ ├── test_init.py │ │ │ ├── test_is_galera.py │ │ │ ├── test_mysql_client.py │ │ │ └── test_wsrep_provider_version.py │ │ └── test_remote_mysql_source.py │ ├── ssh │ │ ├── __init__.py │ │ └── client │ │ │ ├── __init__.py │ │ │ └── ssh_client │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── test_get_text_content.py │ │ │ ├── test_list_files.py │ │ │ └── test_write_content.py │ ├── test_mysql_source.py │ └── test_ssh_exec_command.py ├── integration │ ├── __init__.py │ ├── backup │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── s3 │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── test_backup.py │ │ │ └── test_restore.py │ │ └── ssh │ │ │ ├── __init__.py │ │ │ ├── test_backup.py │ │ │ └── test_restore.py │ ├── clone │ │ ├── __init__.py │ │ ├── conftest.py │ │ └── test_clone.py │ ├── conftest.py │ ├── run_all.py │ └── verify │ │ ├── __init__.py │ │ └── test_verify.py └── unit │ ├── __init__.py │ ├── backup │ ├── __init__.py │ ├── test_backup_binlogs.py │ └── test_binlogs_to_backup.py │ ├── clone │ ├── __init__.py │ ├── test_clone.py │ ├── test_detect_xbstream.py │ ├── test_get_dst.py │ ├── test_get_mysql_service_name.py │ ├── test_get_src.py │ ├── test_get_src_by_vendor.py │ └── test_step_configure_replication.py │ ├── configuration │ ├── __init__.py │ ├── gpg │ │ ├── __init__.py │ │ └── test_init.py │ ├── test_compression.py │ ├── test_mysql.py │ ├── test_retention.py │ ├── test_run_intervals.py │ └── twindb_backup_config │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_backup_dirs.py │ │ ├── test_compression.py │ │ ├── test_gcs.py │ │ ├── test_keep_local_path.py │ │ ├── test_mysql.py │ │ ├── test_retention.py │ │ ├── test_run_intervals.py │ │ ├── test_s3.py │ │ ├── test_ssh.py │ │ └── test_tar_options.py │ ├── conftest.py │ ├── copy │ ├── __init__.py │ ├── base_copy │ │ ├── __init__.py │ │ ├── test_init.py │ │ └── test_key.py │ ├── binlog_copy │ │ ├── __init__.py │ │ ├── test_eq.py │ │ ├── test_init.py │ │ ├── test_key.py │ │ └── test_str.py │ ├── mysql_copy │ │ ├── __init__.py │ │ ├── test_as_dict.py │ │ ├── test_comparison.py │ │ ├── test_eq.py │ │ ├── test_init.py │ │ ├── test_key.py │ │ ├── test_str.py │ │ ├── test_xbstream_binary.py │ │ └── test_xtrabackup_binary.py │ └── periodic_copy │ │ ├── __init__.py │ │ ├── test_init.py │ │ └── test_key.py │ ├── destination │ ├── __init__.py │ ├── az │ │ ├── __init__.py │ │ ├── test_config.py │ │ ├── test_delete.py │ │ ├── test_download_to_pipe.py │ │ ├── test_get_stream.py │ │ ├── test_init.py │ │ ├── test_list_files.py │ │ ├── test_read.py │ │ ├── test_render_path.py │ │ ├── test_save.py │ │ ├── test_write.py │ │ └── util.py │ ├── base │ │ ├── __init__.py │ │ └── test_init.py │ ├── conftest.py │ ├── gcs │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_create_bucket.py │ │ ├── test_delete.py │ │ ├── test_delete_bucket.py │ │ ├── test_get_stream.py │ │ ├── test_init.py │ │ └── test_list_files.py │ ├── local │ │ └── __init__.py │ ├── s3 │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_create_bucket.py │ │ ├── test_delete.py │ │ ├── test_delete_bucket.py │ │ ├── test_get_file_content.py │ │ └── test_list_files.py │ ├── ssh │ │ ├── __init__.py │ │ ├── test_find_files.py │ │ ├── test_get_remote_stdout.py │ │ ├── test_list_files.py │ │ ├── test_mkdir_r.py │ │ ├── test_save.py │ │ ├── test_status_exists.py │ │ └── test_write_status.py │ └── test_base.py │ ├── exporter │ ├── __init__.py │ ├── test_datadog_exporter.py │ └── test_statsd_exporter.py │ ├── modifiers │ ├── __init__.py │ ├── bzip2 │ │ ├── __init__.py │ │ └── test_get_stream.py │ ├── conftest.py │ ├── gzip │ │ ├── __init__.py │ │ └── test_get_stream.py │ ├── lbzip2 │ │ ├── __init__.py │ │ └── test_get_stream.py │ ├── pigz │ │ ├── __init__.py │ │ └── test_get_stream.py │ ├── test_base.py │ ├── test_gpg.py │ └── test_keeplocal.py │ ├── restore │ ├── __init__.py │ ├── test_get_free_memory.py │ └── test_get_my_cnf.py │ ├── source │ ├── __init__.py │ ├── base_source │ │ ├── __init__.py │ │ └── test_suffix.py │ ├── binlog_source │ │ ├── __init__.py │ │ ├── binlog_parser │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── test_created_at.py │ │ │ ├── test_end_position.py │ │ │ ├── test_name.py │ │ │ └── test_start_position.py │ │ └── test_get_name.py │ ├── file_source │ │ ├── __init__.py │ │ ├── test_get_stream.py │ │ └── test_suffix.py │ ├── mariadb_source │ │ ├── __init__.py │ │ └── test_backup_tool.py │ ├── mysql_source │ │ ├── __init__.py │ │ ├── mysql_client │ │ │ ├── __init__.py │ │ │ └── test_server_vendor.py │ │ ├── mysql_flavor.py │ │ ├── test_apply_retention_policy.py │ │ ├── test_backup_tool.py │ │ ├── test_disable_wsrep_desync.py │ │ ├── test_enable_wsrep_desync.py │ │ ├── test_get_binlog_coordinates.py │ │ ├── test_get_connection.py │ │ ├── test_get_lsn.py │ │ ├── test_get_name.py │ │ ├── test_init.py │ │ ├── test_is_galera.py │ │ ├── test_suffix.py │ │ └── test_wsrep_provider_version.py │ ├── remote_mariadb_source │ │ ├── __init__.py │ │ ├── test_get_stream.py │ │ └── test_init.py │ └── test_remote_mysql_source.py │ ├── ssh │ ├── __init__.py │ └── ssh_client │ │ ├── __init__.py │ │ └── test_list_files.py │ ├── status │ ├── __init__.py │ ├── base_status │ │ ├── __init__.py │ │ ├── test_get_latest_backup.py │ │ ├── test_init.py │ │ └── test_remove.py │ ├── binlog_status │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_add.py │ │ ├── test_basename.py │ │ ├── test_eq.py │ │ ├── test_get_item.py │ │ ├── test_get_latest_backup.py │ │ ├── test_init.py │ │ ├── test_load.py │ │ ├── test_remove.py │ │ └── test_serialize.py │ ├── conftest.py │ ├── mysql_status │ │ ├── __init__.py │ │ ├── test_add.py │ │ ├── test_backup_duration.py │ │ ├── test_basename.py │ │ ├── test_candidate_parent.py │ │ ├── test_eq.py │ │ ├── test_full_copy_exists.py │ │ ├── test_get_item.py │ │ ├── test_get_latest_backup.py │ │ ├── test_init.py │ │ ├── test_remove.py │ │ ├── test_serialize.py │ │ ├── test_server_vendor.py │ │ └── test_str.py │ └── periodic_status │ │ ├── __init__.py │ │ ├── test_add.py │ │ ├── test_eq.py │ │ ├── test_init.py │ │ └── test_remove.py │ ├── test_cache.py │ ├── test_file_source.py │ ├── test_ls.py │ ├── test_share.py │ ├── test_twindb_backup │ ├── __init__.py │ ├── conftest.py │ ├── test_delete_local_files.py │ ├── test_get_timeout.py │ ├── test_run_backup_job.py │ └── test_save_measures.py │ ├── util │ ├── __init__.py │ ├── test_my_cnfs.py │ ├── test_normalize_b64_data.py │ └── test_split_host_port.py │ └── verify │ ├── __init__.py │ └── test_edit_backup_my_cnf.py ├── tox.ini ├── twindb_backup ├── __init__.py ├── backup.py ├── cache │ ├── __init__.py │ ├── cache.py │ └── exceptions.py ├── cli.py ├── clone.py ├── configuration │ ├── __init__.py │ ├── compression.py │ ├── destinations │ │ ├── __init__.py │ │ ├── az.py │ │ ├── gcs.py │ │ ├── s3.py │ │ └── ssh.py │ ├── exceptions.py │ ├── gpg.py │ ├── mysql.py │ ├── retention.py │ └── run_intervals.py ├── copy │ ├── __init__.py │ ├── base_copy.py │ ├── binlog_copy.py │ ├── exceptions.py │ ├── file_copy.py │ ├── mysql_copy.py │ └── periodic_copy.py ├── destination │ ├── __init__.py │ ├── az.py │ ├── base_destination.py │ ├── exceptions.py │ ├── gcs.py │ ├── local.py │ ├── s3.py │ └── ssh.py ├── exceptions.py ├── export.py ├── exporter │ ├── __init__.py │ ├── base_exporter.py │ ├── datadog_exporter.py │ ├── exceptions.py │ └── statsd_exporter.py ├── ls.py ├── modifiers │ ├── __init__.py │ ├── base.py │ ├── bzip2.py │ ├── exceptions.py │ ├── gpg.py │ ├── gzip.py │ ├── keeplocal.py │ ├── lbzip2.py │ ├── parallel_compressor.py │ └── pigz.py ├── restore.py ├── share.py ├── source │ ├── __init__.py │ ├── base_source.py │ ├── binlog_source.py │ ├── exceptions.py │ ├── file_source.py │ ├── mariadb_source.py │ ├── mysql_source.py │ ├── remote_mariadb_source.py │ └── remote_mysql_source.py ├── ssh │ ├── __init__.py │ ├── client.py │ └── exceptions.py ├── status │ ├── __init__.py │ ├── base_status.py │ ├── binlog_status.py │ ├── exceptions.py │ ├── mysql_status.py │ └── periodic_status.py ├── util.py └── verify.py └── vagrant ├── README.rst ├── Vagrantfile └── environment └── puppet ├── manifests └── site.pp └── modules ├── profile ├── files │ ├── .my.cnf │ ├── bashrc │ ├── id_rsa │ ├── id_rsa.pub │ ├── my-master-legacy.cnf │ ├── my-master.cnf │ ├── mysql-init │ ├── mysql_grants.sh │ ├── mysql_grants.sql │ ├── sysconfig_docker │ └── twindb-backup.cfg └── manifests │ ├── base.pp │ └── master.pp └── role └── manifests ├── base.pp └── master.pp /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | 18 | [*.{pp,yml,yaml}] 19 | indent_style = space 20 | indent_size = 2 21 | trim_trailing_whitespace = true 22 | insert_final_newline = true 23 | charset = utf-8 24 | end_of_line = lf 25 | 26 | [LICENSE] 27 | insert_final_newline = false 28 | 29 | [Makefile] 30 | indent_style = tab 31 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # These are supported funding model platforms 3 | 4 | github: "[akuzminsky]" 5 | ... 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * TwinDB Backup version: 2 | * Operating System: 3 | 4 | ### Description 5 | 6 | Describe what you were trying to get done. 7 | Tell us what happened, what went wrong, and what you expected to happen. 8 | 9 | ### What I Did 10 | 11 | ``` 12 | Paste the command(s) you ran and the output. 13 | If there was a crash, please include the traceback here. 14 | ``` 15 | -------------------------------------------------------------------------------- /.github/workflows/makefile.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Tests" 3 | 4 | on: 5 | push: 6 | branches: ["master"] 7 | pull_request: 8 | branches: ["master"] 9 | 10 | jobs: 11 | build: 12 | 13 | runs-on: "ubuntu-latest" 14 | 15 | steps: 16 | - uses: "actions/checkout@v3" 17 | - name: "Setup Python env" 18 | uses: "actions/setup-python@v2" 19 | with: 20 | python-version: 3.9 21 | 22 | - name: "Run tests" 23 | run: | 24 | pip install -U "pip ~= 23.0.0" 25 | python -m venv ~/.env 26 | source ~/.env/bin/activate 27 | make bootstrap lint test 28 | ... 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | *.iml 3 | /vagrant/.vagrant 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | pkg/ 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | **/unittests/excluded_env_config/dummy_env_vars.json 49 | sample_resources/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | 58 | # Sphinx documentation 59 | docs/_build/ 60 | 61 | # PyBuilder 62 | target/ 63 | .DS_Store 64 | /vagrant/README.html 65 | /.pytest_cache/ 66 | /.venv/ 67 | 68 | # asdf 69 | .envrc 70 | .tool-versions 71 | 72 | # vscode 73 | .vscode/ 74 | 75 | # environment 76 | .env -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.9.10 2 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | # autogenerated pyup.io config file 2 | # see https://pyup.io/docs/configuration/ for all available options 3 | --- 4 | schedule: '' 5 | update: "insecure" 6 | ... 7 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # .readthedocs.yml 3 | # Read the Docs configuration file 4 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 5 | 6 | # Optionally set the version of Python 7 | # and requirements required to build your docs 8 | python: 9 | install: 10 | - requirements: "requirements.txt" 11 | - requirements: "requirements_dev.txt" 12 | version: "3.8" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: "docs/conf.py" 17 | 18 | # Required 19 | version: 2 20 | ... 21 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | rules: 3 | braces: "enable" 4 | brackets: "enable" 5 | colons: "enable" 6 | commas: "enable" 7 | comments: 8 | level: "warning" 9 | comments-indentation: 10 | level: "warning" 11 | document-end: "enable" 12 | document-start: 13 | level: "warning" 14 | empty-lines: "enable" 15 | empty-values: "enable" 16 | hyphens: "enable" 17 | indentation: "enable" 18 | key-duplicates: "enable" 19 | key-ordering: "disable" 20 | line-length: 21 | allow-non-breakable-inline-mappings: true 22 | allow-non-breakable-words: true 23 | max: 120 24 | new-line-at-end-of-file: "enable" 25 | new-lines: "enable" 26 | octal-values: "enable" 27 | quoted-strings: "enable" 28 | trailing-spaces: "enable" 29 | truthy: 30 | level: "warning" 31 | yaml-files: 32 | - '*.yaml' 33 | - '*.yml' 34 | - '.yamllint' 35 | ... 36 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | 2 | Development Lead 3 | ---------------- 4 | 5 | Aleksandr Kuzminsky 6 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | See `GitHub commits history`_. 6 | 7 | .. _GitHub commits history: https://github.com/twindb/backup/commits/master 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache Software License 2.0 3 | 4 | Copyright (c) 2016, TwinDB Development Team 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | 2 | include AUTHORS.rst 3 | 4 | include CONTRIBUTING.rst 5 | include HISTORY.rst 6 | include LICENSE 7 | include README.rst 8 | include Makefile 9 | include requirements_dev.txt 10 | include requirements.txt 11 | include support/twindb-backup.cron 12 | include support/twindb-backup.cfg 13 | 14 | recursive-include tests * 15 | recursive-exclude * __pycache__ 16 | recursive-exclude * *.py[co] 17 | 18 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 19 | -------------------------------------------------------------------------------- /docs/_static/TwinDB_Backup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/docs/_static/TwinDB_Backup.png -------------------------------------------------------------------------------- /docs/_static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/docs/_static/favicon.png -------------------------------------------------------------------------------- /docs/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/docs/_static/logo.png -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. twindb_backup documentation master file, created by 2 | sphinx-quickstart on Tue Jul 9 22:26:36 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to TwinDB Backup's documentation! 7 | ========================================= 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | readme 15 | installation 16 | usage 17 | modules 18 | contributing 19 | authors 20 | history 21 | 22 | Indices and tables 23 | ================== 24 | 25 | * :ref:`genindex` 26 | * :ref:`modindex` 27 | * :ref:`search` 28 | -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | Source Code Documentation 2 | ========================= 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | twindb_backup 8 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/twindb_backup.cache.rst: -------------------------------------------------------------------------------- 1 | twindb\_backup.cache package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_backup.cache.cache module 8 | --------------------------------- 9 | 10 | .. automodule:: twindb_backup.cache.cache 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_backup.cache.exceptions module 16 | -------------------------------------- 17 | 18 | .. automodule:: twindb_backup.cache.exceptions 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: twindb_backup.cache 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/twindb_backup.configuration.destinations.rst: -------------------------------------------------------------------------------- 1 | twindb\_backup.configuration.destinations package 2 | ================================================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_backup.configuration.destinations.az module 8 | ---------------------------------------------------- 9 | 10 | .. automodule:: twindb_backup.configuration.destinations.az 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_backup.configuration.destinations.gcs module 16 | ---------------------------------------------------- 17 | 18 | .. automodule:: twindb_backup.configuration.destinations.gcs 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | twindb\_backup.configuration.destinations.s3 module 24 | --------------------------------------------------- 25 | 26 | .. automodule:: twindb_backup.configuration.destinations.s3 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | twindb\_backup.configuration.destinations.ssh module 32 | ---------------------------------------------------- 33 | 34 | .. automodule:: twindb_backup.configuration.destinations.ssh 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | Module contents 40 | --------------- 41 | 42 | .. automodule:: twindb_backup.configuration.destinations 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | -------------------------------------------------------------------------------- /docs/twindb_backup.copy.rst: -------------------------------------------------------------------------------- 1 | twindb\_backup.copy package 2 | =========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_backup.copy.base\_copy module 8 | ------------------------------------- 9 | 10 | .. automodule:: twindb_backup.copy.base_copy 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_backup.copy.binlog\_copy module 16 | --------------------------------------- 17 | 18 | .. automodule:: twindb_backup.copy.binlog_copy 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | twindb\_backup.copy.exceptions module 24 | ------------------------------------- 25 | 26 | .. automodule:: twindb_backup.copy.exceptions 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | twindb\_backup.copy.file\_copy module 32 | ------------------------------------- 33 | 34 | .. automodule:: twindb_backup.copy.file_copy 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | twindb\_backup.copy.mysql\_copy module 40 | -------------------------------------- 41 | 42 | .. automodule:: twindb_backup.copy.mysql_copy 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | twindb\_backup.copy.periodic\_copy module 48 | ----------------------------------------- 49 | 50 | .. automodule:: twindb_backup.copy.periodic_copy 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | Module contents 56 | --------------- 57 | 58 | .. automodule:: twindb_backup.copy 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | -------------------------------------------------------------------------------- /docs/twindb_backup.exporter.rst: -------------------------------------------------------------------------------- 1 | twindb\_backup.exporter package 2 | =============================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_backup.exporter.base\_exporter module 8 | --------------------------------------------- 9 | 10 | .. automodule:: twindb_backup.exporter.base_exporter 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_backup.exporter.datadog\_exporter module 16 | ------------------------------------------------ 17 | 18 | .. automodule:: twindb_backup.exporter.datadog_exporter 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | twindb\_backup.exporter.exceptions module 24 | ----------------------------------------- 25 | 26 | .. automodule:: twindb_backup.exporter.exceptions 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | twindb\_backup.exporter.statsd\_exporter module 32 | ----------------------------------------------- 33 | 34 | .. automodule:: twindb_backup.exporter.statsd_exporter 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | Module contents 40 | --------------- 41 | 42 | .. automodule:: twindb_backup.exporter 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | -------------------------------------------------------------------------------- /docs/twindb_backup.ssh.rst: -------------------------------------------------------------------------------- 1 | twindb\_backup.ssh package 2 | ========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_backup.ssh.client module 8 | -------------------------------- 9 | 10 | .. automodule:: twindb_backup.ssh.client 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_backup.ssh.exceptions module 16 | ------------------------------------ 17 | 18 | .. automodule:: twindb_backup.ssh.exceptions 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | Module contents 24 | --------------- 25 | 26 | .. automodule:: twindb_backup.ssh 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | -------------------------------------------------------------------------------- /docs/twindb_backup.status.rst: -------------------------------------------------------------------------------- 1 | twindb\_backup.status package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | twindb\_backup.status.base\_status module 8 | ----------------------------------------- 9 | 10 | .. automodule:: twindb_backup.status.base_status 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | twindb\_backup.status.binlog\_status module 16 | ------------------------------------------- 17 | 18 | .. automodule:: twindb_backup.status.binlog_status 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | twindb\_backup.status.exceptions module 24 | --------------------------------------- 25 | 26 | .. automodule:: twindb_backup.status.exceptions 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | twindb\_backup.status.mysql\_status module 32 | ------------------------------------------ 33 | 34 | .. automodule:: twindb_backup.status.mysql_status 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | twindb\_backup.status.periodic\_status module 40 | --------------------------------------------- 41 | 42 | .. automodule:: twindb_backup.status.periodic_status 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | Module contents 48 | --------------- 49 | 50 | .. automodule:: twindb_backup.status 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | -------------------------------------------------------------------------------- /omnibus/.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | .kitchen/ 4 | .kitchen.local.yml 5 | vendor/bundle 6 | pkg/* 7 | keys/ 8 | cache/ 9 | .vagrant 10 | bin/* 11 | files/**/cache/ 12 | vendor/cookbooks 13 | *.iml 14 | .idea/ 15 | -------------------------------------------------------------------------------- /omnibus/.kitchen.yml: -------------------------------------------------------------------------------- 1 | --- 2 | driver: 3 | name: "vagrant" 4 | forward_agent: true 5 | customize: 6 | cpus: 2 7 | memory: 2048 8 | synced_folders: 9 | - ['.', '/home/vagrant/twindb-backup'] 10 | 11 | provisioner: 12 | name: "chef_zero" 13 | require_chef_omnibus: "12.4.1" 14 | 15 | platforms: 16 | - name: "centos-7.1" 17 | run_list: "yum-epel::default" 18 | - name: "centos-6.6" 19 | run_list: "yum-epel::default" 20 | - name: "debian-8.6" 21 | run_list: "apt::default" 22 | - name: "debian-7.8" 23 | run_list: "apt::default" 24 | - name: "debian-6.0.10" 25 | run_list: "apt::default" 26 | - name: "ubuntu-14.04" 27 | run_list: "apt::default" 28 | - name: "ubuntu-12.04" 29 | run_list: "apt::default" 30 | 31 | suites: 32 | - name: "default" 33 | run_list: "omnibus::default" 34 | attributes: 35 | omnibus: 36 | build_user: "vagrant" 37 | build_user_group: "vagrant" 38 | build_user_password: "vagrant" 39 | install_dir: "/opt/twindb-backup" 40 | ... 41 | -------------------------------------------------------------------------------- /omnibus/Berksfile: -------------------------------------------------------------------------------- 1 | source 'https://supermarket.chef.io' 2 | 3 | cookbook 'omnibus' 4 | 5 | # Uncomment to use the latest version of the Omnibus cookbook from GitHub 6 | cookbook 'omnibus', github: 'opscode-cookbooks/omnibus' 7 | 8 | group :integration do 9 | cookbook 'apt', '~> 2.8' 10 | cookbook 'freebsd', '~> 0.3' 11 | cookbook 'yum-epel', '~> 0.6' 12 | end 13 | -------------------------------------------------------------------------------- /omnibus/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Install omnibus 4 | gem 'omnibus', '~> 9.0' 5 | 6 | # Use Chef's software definitions. It is recommended that you write your own 7 | # software definitions, but you can clone/fork Chef's to get you started. 8 | # Using only local "softwares" 9 | # gem 'omnibus-software', github: 'twindb/omnibus-software' 10 | 11 | # gem 'ohai', '~> 14.8' 12 | # gem 'artifactory', '~> 3.0' 13 | 14 | # This development group is installed by default when you run `bundle install`, 15 | # but if you are using Omnibus in a CI-based infrastructure, you do not need 16 | # the Test Kitchen-based build lab. You can skip these unnecessary dependencies 17 | # by running `bundle install --without development` to speed up build times. 18 | group :development do 19 | # Use Berkshelf for resolving cookbook dependencies 20 | gem 'berkshelf', '~> 7.0' 21 | # Use Test Kitchen with Vagrant for converging the build environment 22 | gem 'test-kitchen', '~> 1.4' 23 | gem 'kitchen-vagrant', '~> 1.4' 24 | end 25 | -------------------------------------------------------------------------------- /omnibus/config/patches/bzip2/makefile_take_env_vars.patch: -------------------------------------------------------------------------------- 1 | --- bzip2-1.0.6/Makefile-orig 2010-09-10 17:46:02.000000000 -0500 2 | +++ bzip2-1.0.6/Makefile 2013-11-21 13:55:11.000000000 -0600 3 | @@ -15,13 +15,13 @@ 4 | 5 | SHELL=/bin/sh 6 | 7 | # To assist in cross-compiling 8 | -CC=gcc 9 | +CC?=gcc 10 | AR=ar 11 | RANLIB=ranlib 12 | -LDFLAGS= 13 | +LDFLAGS+= 14 | 15 | -BIGFILES=-D_FILE_OFFSET_BITS=64 16 | +BIGFILES?=-D_FILE_OFFSET_BITS=64 17 | -CFLAGS=-Wall -Winline -O2 -g $(BIGFILES) 18 | +CFLAGS+=-Wall -Winline -O2 -g $(BIGFILES) 19 | 20 | # Where you want it installed when you do 'make install' 21 | PREFIX=/usr/local 22 | --- bzip2-1.0.6/Makefile-libbz2_so-orig 2017-03-21 19:43:16.706849481 +0000 23 | +++ bzip2-1.0.6/Makefile-libbz2_so 2017-03-21 19:43:01.187851955 +0000 24 | @@ -22,7 +22,7 @@ 25 | 26 | 27 | SHELL=/bin/sh 28 | -CC=gcc 29 | +CC?=gcc 30 | BIGFILES=-D_FILE_OFFSET_BITS=64 31 | CFLAGS=-fpic -fPIC -Wall -Winline -O2 -g $(BIGFILES) 32 | -------------------------------------------------------------------------------- /omnibus/config/patches/openssl/openssl-1.0.1f-do-not-build-docs.patch: -------------------------------------------------------------------------------- 1 | --- openssl-1.0.1f/Makefile.org.orig 2014-01-08 10:58:40.000000000 -0800 2 | +++ openssl-1.0.1f/Makefile.org 2014-01-08 11:06:29.000000000 -0800 3 | @@ -167,7 +167,7 @@ 4 | 5 | TOP= . 6 | ONEDIRS=out tmp 7 | -EDIRS= times doc bugs util include certs ms shlib mt demos perl sf dep VMS 8 | +EDIRS= times bugs util include certs ms shlib mt demos perl sf dep VMS 9 | WDIRS= windows 10 | LIBS= libcrypto.a libssl.a 11 | SHARED_CRYPTO=libcrypto$(SHLIB_EXT) 12 | @@ -538,7 +538,7 @@ 13 | dist_pem_h: 14 | (cd crypto/pem; $(MAKE) -e $(BUILDENV) pem.h; $(MAKE) clean) 15 | 16 | -install: all install_docs install_sw 17 | +install: all install_sw 18 | 19 | install_sw: 20 | @$(PERL) $(TOP)/util/mkdir-p.pl $(INSTALL_PREFIX)$(INSTALLTOP)/bin \ 21 | @@ -603,7 +603,6 @@ 22 | echo 'OpenSSL shared libraries have been installed in:'; \ 23 | echo ' $(INSTALLTOP)'; \ 24 | echo ''; \ 25 | - sed -e '1,/^$$/d' doc/openssl-shared.txt; \ 26 | fi; \ 27 | fi 28 | cp libcrypto.pc $(INSTALL_PREFIX)$(INSTALLTOP)/$(LIBDIR)/pkgconfig 29 | -------------------------------------------------------------------------------- /omnibus/config/software/config_guess.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Chef Software, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # expeditor/ignore: no version pinning 17 | 18 | name "config_guess" 19 | default_version "master" 20 | 21 | # Use our github mirror of the savannah repository 22 | source git: "https://github.com/chef/config-mirror.git" 23 | 24 | # http://savannah.gnu.org/projects/config 25 | license "GPL-3.0 (with exception)" 26 | license_file "config.guess" 27 | license_file "config.sub" 28 | skip_transitive_dependency_licensing true 29 | 30 | relative_path "config_guess-#{version}" 31 | 32 | build do 33 | mkdir "#{install_dir}/embedded/lib/config_guess" 34 | 35 | copy "#{project_dir}/config.guess", "#{install_dir}/embedded/lib/config_guess/config.guess" 36 | copy "#{project_dir}/config.sub", "#{install_dir}/embedded/lib/config_guess/config.sub" 37 | end 38 | -------------------------------------------------------------------------------- /omnibus/config/software/libpcap.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2014-2019 Pinterest 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | name "libpcap" 18 | default_version "2.25" 19 | 20 | license "GPL-v2" 21 | license_file "LICENSE" 22 | skip_transitive_dependency_licensing true 23 | whitelist_file "#{install_dir}/embedded/sbin/getpcaps" 24 | whitelist_file "#{install_dir}/embedded/sbin/capsh" 25 | whitelist_file "#{install_dir}/embedded/sbin/getcap" 26 | whitelist_file "#{install_dir}/embedded/sbin/setcap" 27 | 28 | version("2.25") { source sha256: "693c8ac51e983ee678205571ef272439d83afe62dd8e424ea14ad9790bc35162" } 29 | 30 | source url: "http://archive.ubuntu.com/ubuntu/pool/main/libc/libcap2/libcap2_#{version}.orig.tar.xz" 31 | 32 | relative_path "libcap-#{version}" 33 | 34 | build do 35 | env = with_standard_compiler_flags(with_embedded_path) 36 | env["prefix"] = "#{install_dir}/embedded" 37 | env["LD_LIBRARY_PATH"] = "#{install_dir}/embedded/lib64" 38 | make "-j #{workers}", env: env 39 | make "-j #{workers} install", env: env 40 | end 41 | -------------------------------------------------------------------------------- /omnibus/config/software/libreadline7.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2016 TwinDB LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | name "libreadline7" 18 | default_version "7.0" 19 | 20 | version "7.0" do 21 | source sha256: "750d437185286f40a369e1e4f4764eda932b9459b5ec9a731628393dd3d32334" 22 | source url: "http://archive.ubuntu.com/ubuntu/pool/main/r/readline/readline_#{version}.orig.tar.gz" 23 | end 24 | 25 | license "GNU" 26 | license_file "README" 27 | skip_transitive_dependency_licensing true 28 | 29 | relative_path "readline-#{version}" 30 | 31 | build do 32 | env = with_standard_compiler_flags 33 | configure env: env 34 | 35 | make "-j #{workers}", env: env 36 | make "-j #{workers} install", env: env 37 | end 38 | -------------------------------------------------------------------------------- /omnibus/config/software/pip.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright:: Copyright (c) 2013-2014 Chef Software, Inc. 3 | # License:: Apache License, Version 2.0 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | name "pip" 19 | default_version "22.0.4" 20 | 21 | skip_transitive_dependency_licensing true 22 | dependency "python" 23 | dependency "setuptools" 24 | 25 | source url: "https://github.com/pypa/pip/archive/#{version}.tar.gz", 26 | extract: "seven_zip" 27 | 28 | version("19.1.1") { source sha256: "cce3a3a4cc6f7e1f1d52d0dbe843ebca153ee42660a01acd9248d110c374efa2" } 29 | version("19.3.1") { source sha256: "f12b7a6be2dbbfeefae5f14992c89175ef72ce0fe96452b4f66be855a12841ff" } 30 | version("20.0.2") { source sha256: "00bdc118df4552f654a5ccf0bd3ff1a7d101ee7d7ac899fe9a752363b3f2f070" } 31 | version("22.0.4") { source sha256: "a9828528aa21cf87093e9332f94ea65931a51c443216f5d3a8f14451ef4f2bbf" } 32 | 33 | relative_path "pip-#{version}" 34 | 35 | build do 36 | command "#{install_dir}/embedded/bin/python setup.py install --prefix=#{install_dir}/embedded" 37 | end 38 | -------------------------------------------------------------------------------- /omnibus/config/software/preparation.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2012-2014 Chef Software, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # expeditor/ignore: logic only 17 | 18 | name "preparation" 19 | description "the steps required to preprare the build" 20 | default_version "1.0.0" 21 | 22 | license :project_license 23 | skip_transitive_dependency_licensing true 24 | 25 | build do 26 | block do 27 | touch "#{install_dir}/embedded/lib/.gitkeep" 28 | touch "#{install_dir}/embedded/bin/.gitkeep" 29 | touch "#{install_dir}/bin/.gitkeep" 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /omnibus/config/software/setuptools.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2013-2014 Chef Software, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | name "setuptools" 18 | default_version "62.3.2" 19 | 20 | license "Python Software Foundation" 21 | license_file "https://raw.githubusercontent.com/pypa/setuptools/master/LICENSE" 22 | skip_transitive_dependency_licensing true 23 | 24 | dependency "python" 25 | 26 | version "51.3.3" do 27 | source md5: "83c2360e359139957c41c0798b2f4b67" 28 | source url: "https://pypi.org/packages/source/s/setuptools/setuptools-#{version}.tar.gz" 29 | end 30 | 31 | version "62.3.2" do 32 | source md5: "422e7b02215d6246aa061673ba0896f7" 33 | source url: "https://pypi.org/packages/source/s/setuptools/setuptools-#{version}.tar.gz" 34 | end 35 | 36 | relative_path "setuptools-#{version}" 37 | 38 | build do 39 | env = with_standard_compiler_flags(with_embedded_path) 40 | 41 | command "#{install_dir}/embedded/bin/python setup.py install" \ 42 | " --prefix=#{install_dir}/embedded", env: env 43 | end 44 | -------------------------------------------------------------------------------- /omnibus/config/software/version-manifest.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2012-2014 Chef Software, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | # expeditor/ignore: logic only 17 | 18 | name "version-manifest" 19 | description "generates a version manifest file" 20 | default_version "0.0.1" 21 | 22 | license :project_license 23 | skip_transitive_dependency_licensing true 24 | 25 | build do 26 | block do 27 | File.open("#{install_dir}/version-manifest.txt", "w") do |f| 28 | f.puts "#{project.name} #{project.build_version}" 29 | f.puts "" 30 | f.puts Omnibus::Reports.pretty_version_map(project) 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /omnibus/lib/ostools.rb: -------------------------------------------------------------------------------- 1 | # ------------------------------------ 2 | # OS-detection helper functions 3 | # ------------------------------------ 4 | def linux?() 5 | return %w(rhel debian fedora suse gentoo slackware arch exherbo).include? ohai['platform_family'] 6 | end 7 | 8 | def redhat?() 9 | return %w(rhel fedora).include? ohai['platform_family'] 10 | end 11 | 12 | def centos?() 13 | return ohai['platform'] == "centos" 14 | end 15 | 16 | def debian?() 17 | return ohai['platform_family'] == 'debian' 18 | end 19 | 20 | def focal?() 21 | if ohai['platform'] == 'ubuntu' 22 | return ohai['lsb']['codename'] == 'focal' 23 | end 24 | return false 25 | end 26 | 27 | def bionic?() 28 | if ohai['platform'] == 'ubuntu' 29 | return ohai['lsb']['codename'] == 'bionic' 30 | end 31 | return false 32 | end 33 | 34 | def osx?() 35 | return ohai['platform_family'] == 'mac_os_x' 36 | end 37 | 38 | def windows?() 39 | return ohai['platform_family'] == 'windows' 40 | end 41 | 42 | def suse?() 43 | print ohai["platform_family"] 44 | return ohai["platform_family"] == 'suse' 45 | end 46 | -------------------------------------------------------------------------------- /omnibus/omnibus_build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -ex 4 | 5 | #CACHE_REPO=${PLATFORM}-${OS_VERSION} 6 | 7 | cd /twindb-backup/omnibus 8 | gem install bundler:2.3.14 9 | bundle install --binstubs 10 | #bin/omnibus cache populate 11 | 12 | #git config --global user.email "omnibus@twindb.com" 13 | #git config --global user.name "Omnibus Builder" 14 | 15 | #mkdir -p cache 16 | #aws s3 cp s3://omnibus-cache-twindb-backup/${CACHE_REPO} cache/ || true 17 | 18 | #if test -f cache/${CACHE_REPO}; then 19 | # git clone --mirror \ 20 | # cache/${CACHE_REPO} \ 21 | # /var/cache/omnibus/cache/git_cache/opt/twindb-backup 22 | #fi 23 | 24 | ruby bin/omnibus build twindb-backup 25 | 26 | #if ! test -f cache/${CACHE_REPO}; then 27 | # git --git-dir=/var/cache/omnibus/cache/git_cache/opt/twindb-backup \ 28 | # bundle create cache/${CACHE_REPO} --tags 29 | # aws s3 cp cache/${CACHE_REPO} s3://omnibus-cache-twindb-backup/${CACHE_REPO} 30 | #fi 31 | -------------------------------------------------------------------------------- /omnibus/package-scripts/twindb-backup/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Perform necessary twindb-backup setup steps 4 | # after package is installed. 5 | # 6 | 7 | PROGNAME=`basename $0` 8 | INSTALL_DIR=/opt/twindb-backup 9 | CONFIG_DIR=/etc/twindb 10 | LOG_DIR=/var/log/twindb-backup 11 | PREFIX="/usr" 12 | 13 | DISTRIBUTION=$(grep -Eo "(Debian|Ubuntu|RedHat|CentOS|openSUSE|Amazon|SUSE)" /etc/issue 2>/dev/null || uname -s) 14 | 15 | error_exit() 16 | { 17 | echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2 18 | exit 1 19 | } 20 | 21 | mkdir -p "${LOG_DIR}" || error_exit "Cannot create ${LOG_DIR}!" 22 | 23 | # setup config 24 | if [ ! -e "${CONFIG_DIR}/twindb-backup.cfg" ]; then 25 | mkdir -p ${CONFIG_DIR} || error_exit "Cannot create ${CONFIG_DIR}!" 26 | cp "${INSTALL_DIR}/twindb-backup.cfg" "${CONFIG_DIR}/twindb-backup.cfg" 27 | chmod 600 "${CONFIG_DIR}/twindb-backup.cfg" 28 | else 29 | echo "Skipping setting up of ${CONFIG_DIR}/twindb-backup.cfg as it already exists." 30 | fi 31 | 32 | # setup cron 33 | if [ ! -e /etc/cron.d/twindb-backup ]; then 34 | cp "${INSTALL_DIR}/twindb-backup.cron" /etc/cron.d/twindb-backup 35 | else 36 | echo "Skipping setting up of /etc/cron.d/twindb-backup as it already exists." 37 | fi 38 | 39 | # setup links to the executables 40 | 41 | ln -sf "${INSTALL_DIR}/embedded/bin/twindb-backup" "${PREFIX}/bin/twindb-backup" 42 | 43 | chown -Rh 0:0 "${INSTALL_DIR}" 44 | 45 | echo "Thank you for installing twindb-backup!" 46 | 47 | exit 0 48 | -------------------------------------------------------------------------------- /omnibus/package-scripts/twindb-backup/postrm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Perform necessary twindb-backup removal steps 4 | # after package is uninstalled. 5 | # 6 | 7 | PREFIX="/usr" 8 | LINUX_DISTRIBUTION=$(grep -Eo "(Debian|Ubuntu|RedHat|CentOS|openSUSE|Amazon|SUSE)" /etc/issue) 9 | 10 | cleanup_symlinks() { 11 | binaries="twindb-backup" 12 | for binary in $binaries; do 13 | rm -f $PREFIX/bin/$binary 14 | done 15 | } 16 | 17 | if [ -f "/etc/debian_version" ] || [ "$LINUX_DISTRIBUTION" == "Debian" ] || [ "$LINUX_DISTRIBUTION" == "Ubuntu" ]; then 18 | set -e 19 | 20 | if [ "$1" = purge ]; then 21 | rm -rf /opt/twindb-backup 22 | rm -rf /var/log/twindb-backup 23 | rm -f /etc/twindb/twindb-backup.cfg 24 | rm -f /etc/cron.d/twindb-backup 25 | 26 | cleanup_symlinks 27 | fi 28 | elif [ -f "/etc/redhat-release" ] || [ "$LINUX_DISTRIBUTION" == "RedHat" ] || [ "$LINUX_DISTRIBUTION" == "CentOS" ] || [ "$LINUX_DISTRIBUTION" == "openSUSE" ] || [ "$LINUX_DISTRIBUTION" == "Amazon" ] || [ "$LINUX_DISTRIBUTION" == "SUSE" ]; then 29 | case "$*" in 30 | 0) 31 | # We're uninstalling. 32 | rm -rf /opt/twindb-backup 33 | rm -rf /var/log/twindb-backup 34 | rm -f /etc/twindb/twindb-backup.cfg 35 | rm -f /etc/cron.d/twindb-backup 36 | 37 | cleanup_symlinks 38 | ;; 39 | 1) 40 | # We're upgrading. 41 | ;; 42 | *) 43 | ;; 44 | esac 45 | else 46 | echo "[ ${Red}FAILED ${RCol}]\tYour system is currently not supported by this script."; 47 | exit 1; 48 | fi 49 | 50 | echo "twindb-backup has been uninstalled!" 51 | 52 | exit 0 53 | -------------------------------------------------------------------------------- /omnibus/package-scripts/twindb-backup/prerm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | find /opt/twindb-backup/ -name "*.pyc" -delete 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 120 3 | 4 | [tool.isort] 5 | profile = "black" 6 | line_length = 120 7 | -------------------------------------------------------------------------------- /requirements.in: -------------------------------------------------------------------------------- 1 | #@IgnoreInspection BashAddShebang 2 | azure-core ~= 1.24 3 | azure-storage-blob ~= 12.19 4 | Click ~= 8.1 5 | PyMySQL ~= 1.0 6 | boto3 ~= 1.7 7 | psutil ~= 5.4 8 | pyasn1 ~= 0.4 9 | datadog ~= 0.22 10 | paramiko ~= 3.4 11 | requests < 2.32.0 # See https://github.com/docker/docker-py/issues/3256 12 | google ~= 3.0 13 | google-cloud-storage ~= 2.3.0 14 | statsd ~= 3.3 15 | statsd-tags ~= 3.2 16 | -------------------------------------------------------------------------------- /requirements_dev.in: -------------------------------------------------------------------------------- 1 | #@IgnoreInspection BashAddShebang 2 | azure-storage-blob ~= 12.19 3 | black ~= 24.3 4 | Sphinx ~= 4.5 5 | bumpversion ~= 0.6 6 | docker ~= 5.0 7 | isort ~= 5.10 8 | mock ~= 4.0 9 | moto ~= 4.1 10 | pycodestyle ~= 2.3 11 | pylint ~= 2.13 12 | pytest ~= 7.1 13 | pytest-cov ~= 3.0 14 | pytest-timeout ~= 2.1 15 | python-magic == 0.4.18 16 | requests < 2.32.0 # See https://github.com/docker/docker-py/issues/3256 17 | runlike ~= 1.4 18 | yamllint ~= 1.23 19 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 3.3.0 3 | commit = True 4 | tag = False 5 | 6 | [bumpversion:file:setup.py] 7 | search = version="{current_version}" 8 | replace = version="{new_version}" 9 | 10 | [bumpversion:file:twindb_backup/__init__.py] 11 | search = __version__ = "{current_version}" 12 | replace = __version__ = "{new_version}" 13 | 14 | [bumpversion:file:omnibus/config/projects/twindb-backup.rb] 15 | search = build_version '{current_version}' 16 | replace = build_version '{new_version}' 17 | 18 | [bumpversion:file(1):README.rst] 19 | search = wget https://twindb-release.s3.amazonaws.com/twindb-backup/{current_version}/focal/twindb-backup_{current_version}-1_amd64.deb 20 | replace = wget https://twindb-release.s3.amazonaws.com/twindb-backup/{new_version}/focal/twindb-backup_{new_version}-1_amd64.deb 21 | 22 | [bumpversion:file(2):README.rst] 23 | search = apt install ./twindb-backup_{current_version}-1_amd64.deb 24 | replace = apt install ./twindb-backup_{new_version}-1_amd64.deb 25 | 26 | [bumpversion:file(3):README.rst] 27 | search = omnibus/pkg/twindb-backup_{current_version}-1_amd64.deb 28 | replace = omnibus/pkg/twindb-backup_{new_version}-1_amd64.deb 29 | 30 | [bumpversion:file:docs/installation.rst] 31 | search = omnibus/pkg/twindb-backup_{current_version}-1_amd64.deb 32 | replace = omnibus/pkg/twindb-backup_{new_version}-1_amd64.deb 33 | 34 | [bdist_wheel] 35 | universal = 1 36 | 37 | [flake8] 38 | exclude = docs 39 | 40 | [pycodestyle] 41 | max-line-length = 120 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import os 4 | 5 | from setuptools import find_packages, setup 6 | 7 | del os.link 8 | 9 | with open("README.rst") as readme_file: 10 | readme = readme_file.read() 11 | 12 | with open("HISTORY.rst") as history_file: 13 | history = history_file.read() 14 | 15 | with open("requirements.txt") as f: 16 | requirements = f.read().strip().split("\n") 17 | 18 | with open("requirements_dev.txt") as f: 19 | test_requirements = f.read().strip().split("\n") 20 | 21 | setup( 22 | name="twindb-backup", 23 | version="3.3.0", 24 | description="TwinDB Backup tool for files, MySQL et al.", 25 | long_description=readme + "\n\n" + history, 26 | author="TwinDB Development Team", 27 | author_email="dev@twindb.com", 28 | url="https://github.com/twindb/twindb_backup", 29 | packages=find_packages(exclude=("tests*",)), 30 | package_dir={"twindb_backup": "twindb_backup"}, 31 | entry_points={"console_scripts": ["twindb-backup=twindb_backup.cli:main"]}, 32 | include_package_data=True, 33 | install_requires=requirements, 34 | license="Apache Software License 2.0", 35 | zip_safe=False, 36 | keywords="twindb_backup", 37 | classifiers=[ 38 | "Development Status :: 2 - Pre-Alpha", 39 | "Intended Audience :: Developers", 40 | "License :: OSI Approved :: Apache Software License", 41 | "Natural Language :: English", 42 | "Programming Language :: Python :: 3", 43 | ], 44 | test_suite="tests", 45 | tests_require=test_requirements, 46 | ) 47 | -------------------------------------------------------------------------------- /support/bootstrap/master/centos/master1.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print __version__") 6 | 7 | wait_time=2 8 | for _ in $(seq 5) 9 | do 10 | yum clean all 11 | yum -y install /twindb-backup/omnibus/pkg/twindb-backup-"${TB_VERSION}"-1.x86_64.rpm && break 12 | echo "Waiting ${wait_time} seconds before retry" 13 | sleep ${wait_time} 14 | wait_time=$((wait_time * 2)) 15 | done 16 | 17 | set +u 18 | if ! test -z "${DEV}"; then 19 | /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages 20 | fi 21 | 22 | # MySQL sets random root password. Reset to empty 23 | systemctl stop mysqld 24 | rm -rf /var/lib/mysql 25 | mkdir /var/lib/mysql 26 | mysqld --initialize-insecure 27 | chown -R mysql:mysql /var/lib/mysql 28 | systemctl start mysqld 29 | -------------------------------------------------------------------------------- /support/bootstrap/master/centos/slave.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print __version__") 6 | 7 | wait_time=2 8 | for _ in $(seq 5) 9 | do 10 | yum clean all 11 | yum -y install /twindb-backup/omnibus/pkg/twindb-backup-"${TB_VERSION}"-1.x86_64.rpm && break 12 | echo "Waiting ${wait_time} seconds before retry" 13 | sleep ${wait_time} 14 | wait_time=$((wait_time * 2)) 15 | done 16 | 17 | set +u 18 | if ! test -z "${DEV}"; then 19 | /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages 20 | fi 21 | 22 | # MySQL sets random root password. Reset to empty 23 | systemctl stop mysqld 24 | rm -rf /var/lib/mysql 25 | mkdir /var/lib/mysql 26 | -------------------------------------------------------------------------------- /support/bootstrap/master/ubuntu/master1.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | wait_time=2 6 | for _ in $(seq 5) 7 | do 8 | apt-get -qq update && break 9 | echo "Waiting ${wait_time} seconds before retry" 10 | sleep ${wait_time} 11 | wait_time=$((wait_time * 2)) 12 | done 13 | 14 | apt-get -qqq -y install apt-utils 15 | 16 | TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print(__version__)") 17 | 18 | package="/twindb-backup/omnibus/pkg/twindb-backup_${TB_VERSION}-1_amd64.deb" 19 | 20 | apt -y install "$package" 21 | 22 | chmod 644 /etc/mysql/mysql.conf.d/mysqld.cnf 23 | 24 | set +u 25 | if ! test -z "${DEV}"; then 26 | /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages 27 | fi 28 | -------------------------------------------------------------------------------- /support/bootstrap/master/ubuntu/slave.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | 5 | wait_time=2 6 | for _ in $(seq 5) 7 | do 8 | apt-get -qq update && break 9 | echo "Waiting ${wait_time} seconds before retry" 10 | sleep ${wait_time} 11 | wait_time=$((wait_time * 2)) 12 | done 13 | 14 | apt-get -qqq -y install apt-utils 15 | 16 | TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print(__version__)") 17 | 18 | package="/twindb-backup/omnibus/pkg/twindb-backup_${TB_VERSION}-1_amd64.deb" 19 | 20 | apt -y install "$package" 21 | 22 | rm -rf /var/lib/mysql 23 | mkdir /var/lib/mysql 24 | 25 | chmod 644 /etc/mysql/mysql.conf.d/mysqld.cnf 26 | 27 | set +u 28 | if ! test -z "${DEV}"; then 29 | /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.9/site-packages 30 | fi 31 | -------------------------------------------------------------------------------- /support/bootstrap/storage_server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -exu 4 | apt-get update 5 | 6 | function install_package() { 7 | 8 | for _ in 1 2 3 9 | do 10 | # shellcheck disable=SC2068 11 | apt-get -y install $@ && break 12 | sleep 5 13 | done 14 | } 15 | 16 | function start_sshd() { 17 | if ! test -f /etc/ssh/ssh_host_rsa_key; then 18 | /usr/bin/ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -P "" 19 | fi 20 | if ! test -f /etc/ssh/ssh_host_dsa_key; then 21 | /usr/bin/ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key -P "" 22 | fi 23 | 24 | mkdir -p /run/sshd 25 | mkdir -p /root/.ssh/ 26 | /bin/chown root:root /root/.ssh 27 | /bin/chmod 700 /root/.ssh/ 28 | 29 | /bin/cp -f /twindb-backup/vagrant/environment/puppet/modules/profile/files/id_rsa.pub /root/.ssh/authorized_keys 30 | /bin/chown root:root /root/.ssh/authorized_keys 31 | /bin/chmod 600 /root/.ssh/authorized_keys 32 | 33 | /usr/sbin/sshd -D 34 | } 35 | 36 | 37 | install_package \ 38 | openssh-server \ 39 | netcat \ 40 | sudo 41 | 42 | start_sshd 43 | -------------------------------------------------------------------------------- /support/twindb-backup.cron: -------------------------------------------------------------------------------- 1 | @hourly root twindb-backup backup hourly 2 | @daily root twindb-backup backup daily 3 | @weekly root twindb-backup backup weekly 4 | @monthly root twindb-backup backup monthly 5 | @yearly root twindb-backup backup yearly 6 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /tests/development/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/__init__.py -------------------------------------------------------------------------------- /tests/development/conftest.py: -------------------------------------------------------------------------------- 1 | from twindb_backup import LOG, setup_logging 2 | 3 | setup_logging(LOG, debug=True) 4 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/destination/gcs/__init__.py -------------------------------------------------------------------------------- /tests/development/destination/gcs/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.destination.gcs import GCS 4 | 5 | 6 | @pytest.fixture 7 | def gs(creds_file): 8 | return GCS(gc_credentials_file=creds_file, bucket="twindb-backups") 9 | 10 | 11 | @pytest.fixture 12 | def creds_file(): 13 | return "/twindb_backup/env/My Project 17339-bbbc43d1bee3.json" 14 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_create_bucket.py: -------------------------------------------------------------------------------- 1 | def test_create_bucket(gs): 2 | gs.create_bucket() 3 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_delete.py: -------------------------------------------------------------------------------- 1 | def test_delete(gs): 2 | gs.delete("master1/status1") 3 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_delete_bucket.py: -------------------------------------------------------------------------------- 1 | def test_delete_bucket(gs): 2 | gs.delete_bucket() 3 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_get_stream.py: -------------------------------------------------------------------------------- 1 | from twindb_backup import LOG 2 | from twindb_backup.status.mysql_status import MySQLStatus 3 | 4 | 5 | def test_get_stream(gs): 6 | status = MySQLStatus(dst=gs) 7 | copy = status["master1/daily/mysql/mysql-2019-04-04_05_29_05.xbstream.gz"] 8 | 9 | with gs.get_stream(copy) as stream: 10 | LOG.debug("starting reading from pipe") 11 | content = stream.read() 12 | LOG.debug("finished reading from pipe") 13 | assert len(content), "Failed to read from GS" 14 | LOG.info("Read %d bytes", len(content)) 15 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_gs_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from google.cloud import storage 4 | 5 | 6 | def test_gs_client(creds_file): 7 | os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = creds_file 8 | storage_client = storage.Client() 9 | # storage_client.from_service_account_json() 10 | bucket_name = "my-new-bucket" 11 | bucket = storage_client.create_bucket(bucket_name) 12 | 13 | print("Bucket {} created.".format(bucket.name)) 14 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_list_files.py: -------------------------------------------------------------------------------- 1 | def test_list_files(gs): 2 | print(gs.list_files("master1/")) 3 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_read_status.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.mysql_status import MySQLStatus 2 | 3 | 4 | def test_read_status(gs): 5 | print(MySQLStatus(dst=gs)) 6 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_upload.py: -------------------------------------------------------------------------------- 1 | from subprocess import PIPE, Popen 2 | 3 | 4 | def test_create_bucket(gs): 5 | proc = Popen(["bash", "-c", "echo 123"], stdout=PIPE, stderr=PIPE) 6 | gs.save(proc.stdout, "test") 7 | proc.wait() 8 | # print(cout) 9 | -------------------------------------------------------------------------------- /tests/development/destination/gcs/test_write_status.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.mysql_status import MySQLStatus 2 | 3 | 4 | def test_write_status(gs): 5 | status = MySQLStatus() 6 | status.save(gs) 7 | -------------------------------------------------------------------------------- /tests/development/source/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "aleks" 2 | -------------------------------------------------------------------------------- /tests/development/source/base_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/source/base_source/__init__.py -------------------------------------------------------------------------------- /tests/development/source/base_source/test_suffix.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.base_source import BaseSource 2 | 3 | 4 | def test_suffix_update(): 5 | src = BaseSource("daily") 6 | src.suffix = "xbstream" 7 | assert src.suffix == "xbstream" 8 | src.suffix += ".gz" 9 | assert src.suffix == "xbstream.gz" 10 | -------------------------------------------------------------------------------- /tests/development/source/binlog_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/source/binlog_source/__init__.py -------------------------------------------------------------------------------- /tests/development/source/binlog_source/binlog_parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/source/binlog_source/binlog_parser/__init__.py -------------------------------------------------------------------------------- /tests/development/source/binlog_source/binlog_parser/test_created_at.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.binlog_source import BinlogParser 2 | 3 | 4 | def test_created_at(): 5 | bp = BinlogParser("/tmp/mysql-bin.000001") 6 | assert bp.name == "mysql-bin.000001" 7 | assert bp.created_at == 1533403742 8 | -------------------------------------------------------------------------------- /tests/development/source/binlog_source/binlog_parser/test_name.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.binlog_source import BinlogParser 2 | 3 | 4 | def test_name(): 5 | bp = BinlogParser("/var/lib/foo/bar") 6 | assert bp.name == "bar" 7 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/source/mysql_source/__init__.py -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_disable_wsrep_desync.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mock 4 | 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | @mock.patch.object(MySQLSource, "get_connection") 9 | def test__disable_wsrep_desync_sets_wsrep_desync_to_off(mock_connect): 10 | logging.basicConfig() 11 | 12 | mock_cursor = mock.MagicMock() 13 | mock_cursor.fetchall.return_value = [ 14 | {"Variable_name": "wsrep_local_recv_queue", "Value": "0"}, 15 | ] 16 | 17 | mock_connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value = mock_cursor 18 | 19 | source = MySQLSource(MySQLConnectInfo(None), "daily", "full") 20 | source.disable_wsrep_desync() 21 | 22 | mock_cursor.execute.assert_any_call("SHOW GLOBAL STATUS LIKE " "'wsrep_local_recv_queue'") 23 | mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=OFF") 24 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_enable_wsrep_desync.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mock 4 | 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | @mock.patch.object(MySQLSource, "get_connection") 9 | def test__enable_wsrep_desync_sets_wsrep_desync_to_on(mock_connect): 10 | logging.basicConfig() 11 | 12 | mock_cursor = mock.MagicMock() 13 | 14 | mock_connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value = mock_cursor 15 | 16 | source = MySQLSource(MySQLConnectInfo(None), "daily", "full") 17 | source.enable_wsrep_desync() 18 | 19 | mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=ON") 20 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_get_connection.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | from pymysql import OperationalError 4 | 5 | from twindb_backup.source.exceptions import MySQLSourceError 6 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 7 | 8 | 9 | @mock.patch("twindb_backup.source.mysql_source.pymysql.connect") 10 | def test__get_connection_raises_mysql_source_error(mock_connect): 11 | mock_connect.side_effect = OperationalError 12 | source = MySQLSource(MySQLConnectInfo(None, hostname=None), "daily", "full") 13 | with pytest.raises(MySQLSourceError): 14 | with source.get_connection(): 15 | pass 16 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_get_name.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 4 | 5 | 6 | @mock.patch("twindb_backup.source.base_source.socket") 7 | @mock.patch("twindb_backup.source.base_source.time") 8 | def test_get_name(mock_time, mock_socket): 9 | 10 | host = "some-host" 11 | mock_socket.gethostname.return_value = host 12 | timestamp = "2017-02-13_15_40_29" 13 | mock_time.strftime.return_value = timestamp 14 | 15 | src = MySQLSource(MySQLConnectInfo("/foo/bar"), "daily", "full", dst=mock.Mock()) 16 | 17 | assert src.get_name() == "some-host/daily/mysql/mysql-2017-02-13_15_40_29.xbstream" 18 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_init.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup.source.exceptions import MySQLSourceError 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | def test_mysql_source_has_methods(): 9 | src = MySQLSource(MySQLConnectInfo("/foo/bar"), "hourly", "full", dst=mock.Mock()) 10 | assert src._connect_info.defaults_file == "/foo/bar" 11 | assert src.run_type == "hourly" 12 | assert src.suffix == "xbstream" 13 | assert src._media_type == "mysql" 14 | 15 | 16 | def test_mysql_source_raises_on_wrong_connect_info(): 17 | with pytest.raises(MySQLSourceError): 18 | MySQLSource("/foo/bar", "hourly", "full", dst=mock.Mock()) 19 | 20 | 21 | @pytest.mark.parametrize("run_type", ["foo", None, ""]) 22 | def test_mysql_raises_on_wrong_run_type(run_type): 23 | with pytest.raises(MySQLSourceError): 24 | MySQLSource(MySQLConnectInfo("/foo/bar"), "foo", "full", dst=mock.Mock()) 25 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_mysql_client.py: -------------------------------------------------------------------------------- 1 | from os.path import expanduser 2 | 3 | from twindb_backup.source.mysql_source import MySQLClient 4 | 5 | 6 | def test_client(): 7 | client = MySQLClient(defaults_file=expanduser("~/.my.cnf")) 8 | print(client.server_vendor) 9 | print(client.variable("log_bin_basename")) 10 | -------------------------------------------------------------------------------- /tests/development/source/mysql_source/test_wsrep_provider_version.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mock 4 | 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | @mock.patch.object(MySQLSource, "get_connection") 9 | def test__wsrep_provider_version_returns_correct_version(mock_connect): 10 | logging.basicConfig() 11 | 12 | mock_cursor = mock.MagicMock() 13 | mock_cursor.fetchall.return_value = [ 14 | {"Variable_name": "wsrep_provider_version", "Value": "3.19(rb98f92f)"}, 15 | ] 16 | 17 | mock_connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value = mock_cursor 18 | 19 | source = MySQLSource(MySQLConnectInfo(None), "daily", "full") 20 | assert source.wsrep_provider_version == "3.19" 21 | -------------------------------------------------------------------------------- /tests/development/ssh/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/ssh/__init__.py -------------------------------------------------------------------------------- /tests/development/ssh/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/ssh/client/__init__.py -------------------------------------------------------------------------------- /tests/development/ssh/client/ssh_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/development/ssh/client/ssh_client/__init__.py -------------------------------------------------------------------------------- /tests/development/ssh/client/ssh_client/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.ssh.client import SshClient 4 | 5 | 6 | @pytest.fixture() 7 | def ssh_client(): 8 | return SshClient( 9 | host="127.0.0.1", 10 | key="/Users/aleks/src/twindb/backup/vagrant/.vagrant/machines/master1/virtualbox/private_key", 11 | user="vagrant", 12 | port=2222, 13 | ) 14 | -------------------------------------------------------------------------------- /tests/development/ssh/client/ssh_client/test_get_text_content.py: -------------------------------------------------------------------------------- 1 | def test_get_text_content(ssh_client): 2 | print(ssh_client.get_text_content("/etc/passwd")) 3 | -------------------------------------------------------------------------------- /tests/development/ssh/client/ssh_client/test_list_files.py: -------------------------------------------------------------------------------- 1 | from os import path as osp 2 | 3 | 4 | def test_list_files(ssh_client, tmpdir): 5 | 6 | root_dir = osp.join("/home", ssh_client.user, "tmp", "foo") 7 | print(root_dir) 8 | ssh_client.execute(f"mkdir -p '{root_dir}'") 9 | print(ssh_client.list_files(root_dir)) 10 | 11 | ssh_client.execute("touch '%s'" % osp.join(root_dir, "bar.txt")) 12 | print(ssh_client.list_files(root_dir)) 13 | print(ssh_client.list_files("blah")) 14 | 15 | ssh_client.execute("mkdir -p '%s'" % osp.join(root_dir, "subdir")) 16 | ssh_client.execute("touch '%s'" % osp.join(root_dir, "subdir", "sub_bar.txt")) 17 | 18 | print("subdir with dirs") 19 | print(ssh_client.list_files(root_dir, recursive=True)) 20 | print("subdir without dirs") 21 | print(ssh_client.list_files(root_dir, recursive=True, files_only=True)) 22 | -------------------------------------------------------------------------------- /tests/development/ssh/client/ssh_client/test_write_content.py: -------------------------------------------------------------------------------- 1 | def test_write_content(ssh_client): 2 | path = "/tmp/foo" 3 | content = "foo" 4 | ssh_client.write_content(path, content) 5 | assert ssh_client.get_text_content(path) == content 6 | -------------------------------------------------------------------------------- /tests/development/test_mysql_source.py: -------------------------------------------------------------------------------- 1 | import pymysql 2 | from pymysql import OperationalError 3 | 4 | 5 | def test_pymysql_connect_returns_error(): 6 | try: 7 | connection = pymysql.connect() 8 | except OperationalError as err: 9 | pass 10 | except BaseException as err: 11 | pass 12 | -------------------------------------------------------------------------------- /tests/development/test_ssh_exec_command.py: -------------------------------------------------------------------------------- 1 | import paramiko 2 | 3 | from twindb_backup.destination.ssh import Ssh, SshConnectInfo 4 | 5 | 6 | def test_ssh_exec_command(): 7 | 8 | connect_info = SshConnectInfo( 9 | host="192.168.36.250", 10 | key="/Users/aleks/src/backup/vagrant/.vagrant/machines/master1/virtualbox/private_key", 11 | user="vagrant", 12 | ) 13 | ssh = Ssh(ssh_connect_info=connect_info, remote_path="/tmp/aaa") 14 | _, stdout, stderr = ssh._execute_command(["/bin/ls", "/"]) 15 | print(stdout.readlines()) 16 | 17 | 18 | # def test_get_remote_stdout(): 19 | # connect_info = SshConnectInfo( 20 | # host='192.168.36.250', 21 | # key='/Users/aleks/src/backup/vagrant/.vagrant/machines/master1/virtualbox/private_key', 22 | # user='vagrant' 23 | # ) 24 | # ssh = Ssh(ssh_connect_info=connect_info, remote_path='/tmp/aaa') 25 | # 26 | # with ssh._get_remote_stdout(['ls', '/var']) as stdout: 27 | # print(stdout.readlines()) 28 | 29 | # def test_direct(): 30 | # cli = paramiko.client.SSHClient() 31 | # cli.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) 32 | # cli.connect(hostname="192.168.36.250", username="vagrant", 33 | # key_filename='/Users/aleks/src/backup/vagrant/.vagrant/machines/master1/virtualbox/private_key') 34 | # stdin_, stdout_, stderr_ = cli.exec_command("ls -l /") 35 | # stdout_.channel.recv_exit_status() 36 | # lines = stdout_.readlines() 37 | # for line in lines: 38 | # print line 39 | # 40 | # 41 | -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | 3 | __author__ = "aleks" 4 | 5 | 6 | def ensure_aws_creds(): 7 | print("Integration tests need Amazon API credentials:") 8 | print(f" AWS_ACCESS_KEY_ID .. {'found' if 'AWS_ACCESS_KEY_ID' in environ else 'not found'}") 9 | print(f" AWS_SECRET_ACCESS_KEY .. {'found' if 'AWS_SECRET_ACCESS_KEY' in environ else 'not found'}") 10 | try: 11 | try: 12 | environ["AWS_ACCESS_KEY_ID"] 13 | except KeyError: 14 | print("Environment variable AWS_ACCESS_KEY_ID is not defined.") 15 | environ["AWS_ACCESS_KEY_ID"] = input("Please enter it: ") 16 | try: 17 | environ["AWS_SECRET_ACCESS_KEY"] 18 | except KeyError: 19 | print("Environment variable AWS_SECRET_ACCESS_KEY is not defined.") 20 | environ["AWS_SECRET_ACCESS_KEY"] = input("Please enter it: ") 21 | except KeyboardInterrupt: 22 | exit(1) 23 | -------------------------------------------------------------------------------- /tests/integration/backup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/integration/backup/__init__.py -------------------------------------------------------------------------------- /tests/integration/backup/conftest.py: -------------------------------------------------------------------------------- 1 | from os import path as osp 2 | 3 | from tests.integration.conftest import assert_and_pause, docker_execute 4 | 5 | 6 | def check_either_file(docker_client, container_id, dst_dir, files_to_check): 7 | """Happy if at least one file from a files_to_check list exists.""" 8 | cmd = [ 9 | "bash", 10 | "-c", 11 | "||".join([f"test -f {osp.join(dst_dir, fl)}" for fl in files_to_check]), 12 | ] 13 | 14 | ret, cout = docker_execute(docker_client, container_id, cmd) 15 | print(cout) 16 | assert_and_pause((ret == 0,), cout) 17 | 18 | 19 | def check_files_if_xtrabackup(docker_client, container_id, dst_dir, files_to_check): 20 | # MariaDB and Percona Server work with redo logs differently 21 | # If it's xtrabackup both ib_logfile1 must exist. 22 | for datadir_file in files_to_check: 23 | # See https://en.wikipedia.org/wiki/Material_conditional 24 | cmd = [ 25 | "bash", 26 | "-c", 27 | f"! which xtrabackup || test -f {osp.join(dst_dir, datadir_file)}", 28 | ] 29 | print(cmd) 30 | ret, cout = docker_execute(docker_client, container_id, cmd) 31 | print(cout) 32 | assert_and_pause((ret == 0,), cout) 33 | -------------------------------------------------------------------------------- /tests/integration/backup/s3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/integration/backup/s3/__init__.py -------------------------------------------------------------------------------- /tests/integration/backup/ssh/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/integration/backup/ssh/__init__.py -------------------------------------------------------------------------------- /tests/integration/clone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/integration/clone/__init__.py -------------------------------------------------------------------------------- /tests/integration/clone/conftest.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture 7 | def config_content_clone(): 8 | return dedent( 9 | """ 10 | [ssh] 11 | ssh_user=root 12 | ssh_key={PRIVATE_KEY} 13 | 14 | [mysql] 15 | mysql_defaults_file={MY_CNF} 16 | """ 17 | ) 18 | -------------------------------------------------------------------------------- /tests/integration/verify/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/integration/verify/__init__.py -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "aleks" 2 | -------------------------------------------------------------------------------- /tests/unit/backup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/backup/__init__.py -------------------------------------------------------------------------------- /tests/unit/backup/test_backup_binlogs.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.backup import backup_binlogs 4 | from twindb_backup.source.mysql_source import MySQLClient 5 | from twindb_backup.status.binlog_status import BinlogStatus 6 | 7 | 8 | @mock.patch.object(BinlogStatus, "save") 9 | @mock.patch("twindb_backup.backup.osp") 10 | def test_backup_binlogs_returns_if_no_binlogs(mock_osp, mock_save): 11 | with mock.patch.object(MySQLClient, "variable", return_value=None): 12 | backup_binlogs("foo", mock.Mock()) 13 | assert mock_osp.dirname.call_count == 0 14 | assert mock_save.call_count == 0 15 | -------------------------------------------------------------------------------- /tests/unit/clone/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/clone/__init__.py -------------------------------------------------------------------------------- /tests/unit/clone/test_clone.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.clone import clone_mysql 4 | from twindb_backup.source.mysql_source import MySQLFlavor 5 | from twindb_backup.source.remote_mysql_source import RemoteMySQLSource 6 | 7 | 8 | def test_clone(): 9 | mock_cfg = mock.Mock() 10 | mock_mysql_client = mock.Mock() 11 | mock_mysql_client.server_vendor = MySQLFlavor.PERCONA 12 | with mock.patch("twindb_backup.clone.MySQLClient", return_value=mock_mysql_client), mock.patch( 13 | "twindb_backup.clone.get_src" 14 | ) as mock_get_src, mock.patch("twindb_backup.clone.detect_xbstream") as mock_detect_xbstream, mock.patch( 15 | "twindb_backup.clone.get_dst" 16 | ), mock.patch( 17 | "twindb_backup.clone.step_ensure_empty_directory" 18 | ), mock.patch( 19 | "twindb_backup.clone.step_run_remote_netcat" 20 | ), mock.patch( 21 | "twindb_backup.clone.step_clone_source" 22 | ), mock.patch( 23 | "twindb_backup.clone.step_clone_mysql_config" 24 | ), mock.patch( 25 | "twindb_backup.clone.step_start_mysql_service" 26 | ), mock.patch( 27 | "twindb_backup.clone.step_configure_replication" 28 | ), mock.patch.object( 29 | RemoteMySQLSource, "apply_backup", return_value=("bin.001", 123) 30 | ): 31 | clone_mysql(mock_cfg, "master1:3306", "slave1:3307", "foo_user", "foo_pass") 32 | mock_get_src.assert_called_once_with(mock_cfg, mock_mysql_client, "master1:3306") 33 | mock_detect_xbstream.assert_called_once_with(mock_cfg, mock_mysql_client) 34 | -------------------------------------------------------------------------------- /tests/unit/clone/test_detect_xbstream.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup import MBSTREAM_BINARY, XBSTREAM_BINARY 5 | from twindb_backup.clone import detect_xbstream 6 | from twindb_backup.configuration import MySQLConfig, TwinDBBackupConfig 7 | from twindb_backup.source.mysql_source import MySQLClient, MySQLFlavor 8 | 9 | 10 | @pytest.mark.parametrize( 11 | "given_xbstream, vendor, expected_xbstream", 12 | [ 13 | ("foo", MySQLFlavor.MARIADB, "foo"), 14 | (None, MySQLFlavor.MARIADB, MBSTREAM_BINARY), 15 | (None, MySQLFlavor.PERCONA, XBSTREAM_BINARY), 16 | (None, MySQLFlavor.ORACLE, XBSTREAM_BINARY), 17 | ], 18 | ) 19 | def test_detect_xbstream(given_xbstream, vendor, expected_xbstream): 20 | with mock.patch.object( 21 | MySQLClient, 22 | "server_vendor", 23 | new_callable=mock.PropertyMock, 24 | return_value=vendor, 25 | ), mock.patch.object( 26 | TwinDBBackupConfig, 27 | "mysql", 28 | new_callable=mock.PropertyMock, 29 | return_value=MySQLConfig(xbstream_binary=given_xbstream), 30 | ): 31 | assert detect_xbstream(TwinDBBackupConfig(), MySQLClient("foo_path")) == expected_xbstream 32 | -------------------------------------------------------------------------------- /tests/unit/clone/test_get_dst.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup.clone import get_dst 5 | from twindb_backup.configuration import SSHConfig, TwinDBBackupConfig 6 | from twindb_backup.destination.ssh import Ssh 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "destination, host, user, key", 11 | [("slave:3306", "slave", "foo", "bar"), ("slave", "slave", "foo", "bar")], 12 | ) 13 | def test_get_dst(destination, host, user, key): 14 | with mock.patch.object(Ssh, "__init__", return_value=None) as mock_dst, mock.patch.object( 15 | TwinDBBackupConfig, 16 | "ssh", 17 | new_callable=mock.PropertyMock, 18 | return_value=SSHConfig(ssh_user=user, ssh_key=key), 19 | ): 20 | get_dst(TwinDBBackupConfig(), destination) 21 | mock_dst.assert_called_once_with("/tmp", ssh_host=host, ssh_user=user, ssh_key=key) 22 | -------------------------------------------------------------------------------- /tests/unit/clone/test_get_mysql_service_name.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | from mock.mock import call 4 | 5 | from twindb_backup.clone import _get_mysql_service_name 6 | from twindb_backup.destination.ssh import Ssh 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "side_effect, expected_name", 11 | [ 12 | ([("0", ""), ("1\n", "")], "mysqld"), 13 | ([("0\n", ""), ("1\n", "")], "mysqld"), 14 | ([("\n0\n", ""), ("1\n", "")], "mysqld"), 15 | ([("1\n", ""), ("0\n", "")], "mysql"), 16 | ([("0", ""), ("0", ""), ("1", "")], "mariadb"), 17 | ], 18 | ) 19 | def test_get_mysql_service_name(side_effect, expected_name): 20 | with mock.patch.object(Ssh, "execute_command", side_effect=side_effect) as mock_execute: 21 | assert _get_mysql_service_name(Ssh("foo")) == expected_name 22 | mock_execute.assert_has_calls( 23 | [call(f"systemctl list-units --full -all | grep -F '{expected_name}.service' | wc -l")] 24 | ) 25 | -------------------------------------------------------------------------------- /tests/unit/clone/test_get_src.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup import INTERVALS 4 | from twindb_backup.clone import get_src 5 | from twindb_backup.configuration import MySQLConfig, SSHConfig, TwinDBBackupConfig 6 | from twindb_backup.source.mysql_source import MySQLClient, MySQLConnectInfo, MySQLFlavor 7 | 8 | 9 | def test_get_src(): 10 | with mock.patch("twindb_backup.clone.get_src_by_vendor") as mock_get_src_by_vendor, mock.patch.object( 11 | TwinDBBackupConfig, 12 | "ssh", 13 | new_callable=mock.PropertyMock, 14 | return_value=SSHConfig(ssh_user="foo_user", ssh_key="foo_key"), 15 | ), mock.patch.object( 16 | TwinDBBackupConfig, 17 | "mysql", 18 | new_callable=mock.PropertyMock, 19 | return_value=MySQLConfig(mysql_defaults_file="foo_path"), 20 | ), mock.patch.object( 21 | MySQLClient, 22 | "server_vendor", 23 | new_callable=mock.PropertyMock, 24 | return_value=MySQLFlavor.PERCONA, 25 | ): 26 | get_src(TwinDBBackupConfig(), MySQLClient("foo_path"), "master:3306") 27 | mock_get_src_by_vendor.assert_called_once_with( 28 | MySQLFlavor.PERCONA, 29 | "master", 30 | "foo_user", 31 | "foo_key", 32 | MySQLConnectInfo("foo_path", hostname="master"), 33 | INTERVALS[0], 34 | ) 35 | -------------------------------------------------------------------------------- /tests/unit/clone/test_get_src_by_vendor.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup import INTERVALS 5 | from twindb_backup.clone import get_src_by_vendor 6 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLFlavor 7 | from twindb_backup.source.remote_mysql_source import RemoteMySQLSource 8 | 9 | 10 | @pytest.mark.parametrize("vendor, expected_klass", [(MySQLFlavor.PERCONA, RemoteMySQLSource)]) 11 | def test_get_src_by_vendor(vendor, expected_klass): 12 | isinstance( 13 | get_src_by_vendor( 14 | vendor, 15 | "foo", 16 | "foo", 17 | "foo", 18 | mock.Mock(spec=MySQLConnectInfo), 19 | INTERVALS[0], 20 | ), 21 | expected_klass, 22 | ) 23 | -------------------------------------------------------------------------------- /tests/unit/clone/test_step_configure_replication.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.clone import step_configure_replication 4 | from twindb_backup.source.mysql_source import MySQLMasterInfo 5 | from twindb_backup.source.remote_mysql_source import RemoteMySQLSource 6 | 7 | 8 | def test_step_configure_replication(): 9 | mock_dst = mock.Mock(spec=RemoteMySQLSource) 10 | step_configure_replication( 11 | mock_dst, 12 | MySQLMasterInfo("host_foo", 1234, "foo_user", "foo_password", "foo-1.1", 8873), 13 | ) 14 | -------------------------------------------------------------------------------- /tests/unit/configuration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/configuration/__init__.py -------------------------------------------------------------------------------- /tests/unit/configuration/gpg/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/configuration/gpg/__init__.py -------------------------------------------------------------------------------- /tests/unit/configuration/gpg/test_init.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration import GPGConfig 2 | 3 | 4 | def test_init(): 5 | gpg = GPGConfig(recipient="bob", keyring="/dev/null") 6 | assert gpg.recipient == "bob" 7 | assert gpg.keyring == "/dev/null" 8 | assert gpg.secret_keyring is None 9 | -------------------------------------------------------------------------------- /tests/unit/configuration/test_compression.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.configuration import CompressionConfig 4 | from twindb_backup.configuration.exceptions import ConfigurationError 5 | 6 | 7 | def test_init_default(): 8 | cc = CompressionConfig() 9 | assert cc.program == "gzip" 10 | 11 | 12 | def test_unsupported_program_raises(): 13 | with pytest.raises(ConfigurationError): 14 | CompressionConfig(program="foo") 15 | -------------------------------------------------------------------------------- /tests/unit/configuration/test_mysql.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration.mysql import MySQLConfig 2 | 3 | 4 | def test_mysql(): 5 | mc = MySQLConfig() 6 | assert mc.defaults_file == "/root/.my.cnf" 7 | assert mc.full_backup == "daily" 8 | 9 | 10 | def test_mysql_set_xtrabackup_binary(): 11 | mc = MySQLConfig() 12 | mc.xtrabackup_binary = "foo" 13 | assert mc.xtrabackup_binary == "foo" 14 | 15 | 16 | def test_mysql_set_xbstream_binary(): 17 | mc = MySQLConfig() 18 | mc.xbstream_binary = "foo" 19 | assert mc.xbstream_binary == "foo" 20 | -------------------------------------------------------------------------------- /tests/unit/configuration/test_retention.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration.retention import RetentionPolicy 2 | 3 | 4 | def test_retention_default(): 5 | ret = RetentionPolicy() 6 | assert ret.hourly == 24 7 | assert ret.daily == 7 8 | assert ret.weekly == 4 9 | assert ret.monthly == 12 10 | assert ret.yearly == 3 11 | 12 | 13 | def test_retention(): 14 | assert RetentionPolicy(hourly=7).hourly == 7 15 | assert RetentionPolicy(daily=8).daily == 8 16 | -------------------------------------------------------------------------------- /tests/unit/configuration/test_run_intervals.py: -------------------------------------------------------------------------------- 1 | from twindb_backup import INTERVALS 2 | from twindb_backup.configuration.run_intervals import RunIntervals 3 | 4 | 5 | def test_run_intervals_default(): 6 | ri = RunIntervals() 7 | for i in INTERVALS: 8 | assert getattr(ri, i) is True 9 | 10 | 11 | def test_run_intervals(): 12 | ri = RunIntervals(hourly=False) 13 | for i in INTERVALS: 14 | if i == "hourly": 15 | assert getattr(ri, i) is False 16 | else: 17 | assert getattr(ri, i) is True 18 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/configuration/twindb_backup_config/__init__.py -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def config_file(config_content, tmpdir): 6 | cfg_file = tmpdir.join("twindb-backup.cfg") 7 | dst = tmpdir.mkdir("dst") 8 | with open(str(cfg_file), "w") as fp: 9 | fp.write(config_content.format(port=123, destination=str(dst))) 10 | return cfg_file 11 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_backup_dirs.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | import pytest 4 | 5 | from twindb_backup.configuration import ConfigurationError, TwinDBBackupConfig 6 | 7 | 8 | def test_backup_dirs(config_file): 9 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 10 | assert tbc.backup_dirs == [ 11 | "/", 12 | "/root/", 13 | "/etc", 14 | "/dir with space/", 15 | "/dir foo", 16 | ] 17 | 18 | 19 | def test_no_backup_dirs(tmpdir): 20 | cfg_file = tmpdir.join("twindb-backup.cfg") 21 | with open(str(cfg_file), "w") as fp: 22 | fp.write( 23 | dedent( 24 | """ 25 | [source] 26 | """ 27 | ) 28 | ) 29 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 30 | assert tbc.backup_dirs == [] 31 | 32 | 33 | def test_no_source(tmpdir): 34 | cfg_file = tmpdir.join("twindb-backup.cfg") 35 | with open(str(cfg_file), "w") as fp: 36 | fp.write("") 37 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 38 | with pytest.raises(ConfigurationError): 39 | assert tbc.backup_dirs == [] 40 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_gcs.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration import TwinDBBackupConfig 2 | 3 | 4 | def test_gcs(config_file): 5 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 6 | assert tbc.gcs.gc_credentials_file == "XXXXX" 7 | assert tbc.gcs.gc_encryption_key == "" 8 | assert tbc.gcs.bucket == "twindb-backups" 9 | 10 | 11 | def test_no_gcs_section(tmpdir): 12 | cfg_file = tmpdir.join("twindb-backup.cfg") 13 | with open(str(cfg_file), "w") as fp: 14 | fp.write("") 15 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 16 | assert tbc.gcs is None 17 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_keep_local_path.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | import pytest 4 | 5 | from twindb_backup.configuration import TwinDBBackupConfig 6 | 7 | 8 | def test_keep_local_path(config_file): 9 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 10 | assert tbc.keep_local_path == "/var/backup/local" 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "content", 15 | [ 16 | "", 17 | dedent( 18 | """ 19 | [destination] 20 | """ 21 | ), 22 | ], 23 | ) 24 | def test_no_keep_local_path(content, tmpdir): 25 | cfg_file = tmpdir.join("twindb-backup.cfg") 26 | with open(str(cfg_file), "w") as fp: 27 | fp.write(content) 28 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 29 | assert tbc.keep_local_path is None 30 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_retention.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration import TwinDBBackupConfig 2 | from twindb_backup.configuration.retention import RetentionPolicy 3 | 4 | 5 | def test_retention(config_file): 6 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 7 | assert tbc.retention == RetentionPolicy(hourly=24, daily=7, weekly=4, monthly=12, yearly=3) 8 | 9 | assert tbc.retention_local == RetentionPolicy(hourly=24, daily=7, weekly=4, monthly=12, yearly=3) 10 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_run_intervals.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration import TwinDBBackupConfig 2 | from twindb_backup.configuration.run_intervals import RunIntervals 3 | 4 | 5 | def test_run_intervals(config_file): 6 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 7 | assert tbc.run_intervals == RunIntervals() 8 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_s3.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.configuration import TwinDBBackupConfig 2 | 3 | 4 | def test_s3(config_file): 5 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 6 | assert tbc.s3.aws_access_key_id == "XXXXX" 7 | assert tbc.s3.aws_secret_access_key == "YYYYY" 8 | assert tbc.s3.aws_default_region == "us-east-1" 9 | assert tbc.s3.bucket == "twindb-backups" 10 | 11 | 12 | def test_no_s3_section(tmpdir): 13 | cfg_file = tmpdir.join("twindb-backup.cfg") 14 | with open(str(cfg_file), "w") as fp: 15 | fp.write("") 16 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 17 | assert tbc.s3 is None 18 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_ssh.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | from twindb_backup.configuration import TwinDBBackupConfig 4 | 5 | 6 | def test_ssh(config_file): 7 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 8 | assert tbc.ssh.user == "root" 9 | assert tbc.ssh.key == "/root/.ssh/id_rsa" 10 | assert tbc.ssh.port == 123 11 | assert tbc.ssh.host == "127.0.0.1" 12 | assert tbc.ssh.path == "/tmp/backup" 13 | 14 | 15 | def test_no_ssh_section(tmpdir): 16 | cfg_file = tmpdir.join("twindb-backup.cfg") 17 | with open(str(cfg_file), "w") as fp: 18 | fp.write("") 19 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 20 | assert tbc.ssh is None 21 | 22 | 23 | def test_default_values(tmpdir): 24 | cfg_file = tmpdir.join("twindb-backup.cfg") 25 | with open(str(cfg_file), "w") as fp: 26 | fp.write( 27 | dedent( 28 | """ 29 | [ssh] 30 | ssh_user=root 31 | ssh_key=/etc/twindb/private_key 32 | 33 | [mysql] 34 | mysql_defaults_file=/etc/twindb/my.cnf 35 | """ 36 | ) 37 | ) 38 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 39 | assert tbc.ssh.host == "127.0.0.1" 40 | assert tbc.ssh.path == "/var/backup" 41 | -------------------------------------------------------------------------------- /tests/unit/configuration/twindb_backup_config/test_tar_options.py: -------------------------------------------------------------------------------- 1 | """Tests that cover the FileSource().tar_options property.""" 2 | 3 | from textwrap import dedent 4 | 5 | from twindb_backup.configuration import TwinDBBackupConfig 6 | 7 | 8 | def test_no_tar_options(config_file): 9 | """Check config where there is no tar_options""" 10 | tbc = TwinDBBackupConfig(config_file=str(config_file)) 11 | assert tbc.tar_options is None 12 | 13 | 14 | def test_tar_options(tmpdir): 15 | """Check tar_options becomes a property.""" 16 | cfg_file = tmpdir.join("twindb-backup.cfg") 17 | with open(str(cfg_file), "w", encoding="utf-8") as config_desc: 18 | config_desc.write( 19 | dedent( 20 | """ 21 | [source] 22 | tar_options = --exclude-vcs-ignores 23 | """ 24 | ) 25 | ) 26 | tbc = TwinDBBackupConfig(config_file=str(cfg_file)) 27 | assert tbc.tar_options == "--exclude-vcs-ignores" 28 | -------------------------------------------------------------------------------- /tests/unit/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup import LOG, setup_logging 4 | 5 | setup_logging(LOG, debug=True) 6 | 7 | 8 | @pytest.fixture 9 | def cache_dir(tmpdir): 10 | cache_path = tmpdir.mkdir("cache") 11 | return cache_path 12 | 13 | 14 | @pytest.fixture 15 | def config_content(): 16 | return """ 17 | [source] 18 | backup_dirs=/ /root/ /etc "/dir with space/" '/dir foo' 19 | backup_mysql=yes 20 | 21 | [destination] 22 | backup_destination={destination} 23 | keep_local_path=/var/backup/local 24 | 25 | [s3] 26 | AWS_ACCESS_KEY_ID="XXXXX" 27 | AWS_SECRET_ACCESS_KEY="YYYYY" 28 | AWS_DEFAULT_REGION="us-east-1" 29 | BUCKET="twindb-backups" 30 | 31 | [az] 32 | connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" 33 | container_name="twindb-backups" 34 | remote_path="/backups/mysql" 35 | 36 | [gcs] 37 | GC_CREDENTIALS_FILE="XXXXX" 38 | GC_ENCRYPTION_KEY= 39 | BUCKET="twindb-backups" 40 | 41 | [mysql] 42 | mysql_defaults_file=/etc/twindb/my.cnf 43 | expire_log_days = 8 44 | 45 | [ssh] 46 | ssh_user="root" 47 | ssh_key=/root/.ssh/id_rsa 48 | port={port} 49 | backup_host='127.0.0.1' 50 | backup_dir=/tmp/backup 51 | 52 | [retention] 53 | hourly_copies=24 54 | daily_copies=7 55 | weekly_copies=4 56 | monthly_copies=12 57 | yearly_copies=3 58 | """ 59 | -------------------------------------------------------------------------------- /tests/unit/copy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/copy/__init__.py -------------------------------------------------------------------------------- /tests/unit/copy/base_copy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/copy/base_copy/__init__.py -------------------------------------------------------------------------------- /tests/unit/copy/base_copy/test_init.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.base_copy import BaseCopy 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "host, fname", 8 | [ 9 | ("test_host", "test_file"), 10 | ], 11 | ) 12 | def test_init(host, fname): 13 | instance = BaseCopy(host, fname) 14 | assert instance._host == host 15 | assert instance._name == fname 16 | assert instance._source_type is None 17 | -------------------------------------------------------------------------------- /tests/unit/copy/base_copy/test_key.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup.copy.base_copy import BaseCopy 5 | from twindb_backup.copy.exceptions import UnknownSourceType 6 | 7 | 8 | def test_key_raised_error_in_abstract_class(): 9 | instance = BaseCopy("host", "fname") 10 | with pytest.raises(UnknownSourceType): 11 | # noinspection PyStatementEffect 12 | instance.key 13 | 14 | 15 | # noinspection PyUnusedLocal 16 | @pytest.mark.parametrize( 17 | "extra_path, key", 18 | [(None, "foo/foo-type/bar"), ("foo-path", "foo/foo-path/foo-type/bar")], 19 | ) 20 | @mock.patch.object(BaseCopy, "_extra_path") 21 | def test_key(mock_extra_path, extra_path, key): 22 | 23 | copy = BaseCopy("foo", "bar") 24 | copy._source_type = "foo-type" 25 | 26 | # noinspection PyPropertyAccess 27 | copy._extra_path = extra_path 28 | 29 | assert copy.key == key 30 | -------------------------------------------------------------------------------- /tests/unit/copy/binlog_copy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/copy/binlog_copy/__init__.py -------------------------------------------------------------------------------- /tests/unit/copy/binlog_copy/test_eq.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.binlog_copy import BinlogCopy 2 | 3 | 4 | def test_two_eq(): 5 | copy1 = BinlogCopy("foo", "bar", 10) 6 | copy2 = BinlogCopy("foo", "bar", 10) 7 | assert copy1 == copy2 8 | 9 | 10 | def test_two_neq_created(): 11 | copy1 = BinlogCopy("foo", "bar", 10) 12 | copy2 = BinlogCopy("foo", "bar", 20) 13 | assert copy1 != copy2 14 | 15 | 16 | def test_two_neq_name(): 17 | copy1 = BinlogCopy("foo", "bar1", 10) 18 | copy2 = BinlogCopy("foo", "bar2", 10) 19 | assert copy1 != copy2 20 | -------------------------------------------------------------------------------- /tests/unit/copy/binlog_copy/test_init.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.binlog_copy import BinlogCopy 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "host, fname, time_created", 8 | [ 9 | ("test_host", "test_file", 100500), 10 | ("test_host2", "test_file2", 1005002), 11 | ], 12 | ) 13 | def test_init(host, fname, time_created): 14 | instance = BinlogCopy(host, fname, time_created) 15 | assert instance.created_at == time_created 16 | assert instance.name == fname 17 | assert instance.key == "{host}/binlog/{name}".format(host=host, name=fname) 18 | -------------------------------------------------------------------------------- /tests/unit/copy/binlog_copy/test_key.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.binlog_copy import BinlogCopy 2 | 3 | 4 | def test_key(): 5 | backup_copy = BinlogCopy("foo", "some_name", 100) 6 | 7 | assert backup_copy.key == "foo/binlog/some_name" 8 | -------------------------------------------------------------------------------- /tests/unit/copy/binlog_copy/test_str.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.binlog_copy import BinlogCopy 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "host, fname, time_created, copy_repr", 8 | [ 9 | ( 10 | "test_host", 11 | "test_file", 12 | 100500, 13 | "BinlogCopy(test_host/binlog/test_file)", 14 | ), 15 | ( 16 | "test_host2", 17 | "test_file2", 18 | 1005002, 19 | "BinlogCopy(test_host2/binlog/test_file2)", 20 | ), 21 | ], 22 | ) 23 | def test_repr(host, fname, time_created, copy_repr): 24 | instance = BinlogCopy(host, fname, time_created) 25 | assert repr(instance) == copy_repr 26 | 27 | 28 | @pytest.mark.parametrize( 29 | "host, fname, time_created, copy_as_str", 30 | [ 31 | ( 32 | "test_host", 33 | "test_file", 34 | 100500, 35 | "BinlogCopy: file name: test_file, created at: 100500", 36 | ), 37 | ( 38 | "test_host2", 39 | "test_file2", 40 | 1005002, 41 | "BinlogCopy: " "file name: test_file2, created at: 1005002", 42 | ), 43 | ], 44 | ) 45 | def test_str(host, fname, time_created, copy_as_str): 46 | instance = BinlogCopy(host, fname, time_created) 47 | assert str(instance) == copy_as_str 48 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/copy/mysql_copy/__init__.py -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_as_dict.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | 3 | 4 | def test_as_dict(): 5 | copy = MySQLCopy("foo", "daily", "some_file.txt", type="full") 6 | assert copy.as_dict() == { 7 | "type": "full", 8 | "backup_finished": None, 9 | "backup_started": None, 10 | "binlog": None, 11 | "config": {}, 12 | "host": "foo", 13 | "lsn": None, 14 | "name": "some_file.txt", 15 | "parent": None, 16 | "position": None, 17 | "run_type": "daily", 18 | "galera": False, 19 | "wsrep_provider_version": None, 20 | "server_vendor": "oracle", 21 | } 22 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_comparison.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | 3 | 4 | def test_lt(): 5 | assert MySQLCopy("foo", "daily", "some_file.txt", type="full", lsn=1) < MySQLCopy( 6 | "foo", "daily", "some_file.txt", type="full", lsn=2 7 | ) 8 | 9 | 10 | def test_le(): 11 | assert MySQLCopy("foo", "daily", "some_file.txt", type="full", lsn=1) <= MySQLCopy( 12 | "foo", "daily", "some_file.txt", type="full", lsn=1 13 | ) 14 | 15 | 16 | def test_gt(): 17 | assert MySQLCopy("foo", "daily", "some_file.txt", type="full", lsn=2) > MySQLCopy( 18 | "foo", "daily", "some_file.txt", type="full", lsn=1 19 | ) 20 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_eq.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | 3 | 4 | def test_eq(): 5 | copy_1 = MySQLCopy("foo", "daily", "some_file.txt", type="full") 6 | copy_2 = MySQLCopy("foo", "daily", "some_file.txt", type="full") 7 | copy_3 = MySQLCopy("bar", "daily", "some_file.txt", type="full") 8 | assert copy_1 == copy_2 9 | assert copy_1 != copy_3 10 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_key.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | 3 | 4 | def test_key(): 5 | backup_copy = MySQLCopy("foo", "daily", "some_file.txt", type="full") 6 | 7 | assert backup_copy.key == "foo/daily/mysql/some_file.txt" 8 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_str.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | 3 | 4 | def test_repr(): 5 | copy = MySQLCopy("foo", "daily", "some_file.txt", type="full") 6 | assert repr(copy) == "MySQLCopy(foo/daily/mysql/some_file.txt)" 7 | 8 | 9 | def test_str(): 10 | copy = MySQLCopy("foo", "daily", "some_file.txt", type="full") 11 | 12 | expected = """MySQLCopy(foo/daily/mysql/some_file.txt) = { 13 | "backup_finished": null, 14 | "backup_started": null, 15 | "binlog": null, 16 | "config": {}, 17 | "galera": false, 18 | "host": "foo", 19 | "lsn": null, 20 | "name": "some_file.txt", 21 | "parent": null, 22 | "position": null, 23 | "run_type": "daily", 24 | "server_vendor": "oracle", 25 | "type": "full", 26 | "wsrep_provider_version": null 27 | }""" 28 | assert str(copy) == expected 29 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_xbstream_binary.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.mysql_copy import MySQLCopy 4 | from twindb_backup.source.mysql_source import MySQLFlavor 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "vendor, binary", 9 | [ 10 | (MySQLFlavor.ORACLE, "xbstream"), 11 | (MySQLFlavor.PERCONA, "xbstream"), 12 | (MySQLFlavor.MARIADB, "mbstream"), 13 | ("mariadb", "mbstream"), 14 | ("percona", "xbstream"), 15 | ("oracle", "xbstream"), 16 | ], 17 | ) 18 | def test_xbstream_binary(vendor, binary): 19 | assert MySQLCopy("foo", "daily", "some_file.txt", type="full", server_vendor=vendor).xbstream_binary == binary 20 | -------------------------------------------------------------------------------- /tests/unit/copy/mysql_copy/test_xtrabackup_binary.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.mysql_copy import MySQLCopy 4 | from twindb_backup.source.mysql_source import MySQLFlavor 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "vendor, binary", 9 | [ 10 | (MySQLFlavor.ORACLE, "xtrabackup"), 11 | (MySQLFlavor.PERCONA, "xtrabackup"), 12 | (MySQLFlavor.MARIADB, "mariabackup"), 13 | ("mariadb", "mariabackup"), 14 | ("percona", "xtrabackup"), 15 | ("oracle", "xtrabackup"), 16 | ], 17 | ) 18 | def test_xtrabackup_binary(vendor, binary): 19 | assert MySQLCopy("foo", "daily", "some_file.txt", type="full", server_vendor=vendor).xtrabackup_binary == binary 20 | -------------------------------------------------------------------------------- /tests/unit/copy/periodic_copy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/copy/periodic_copy/__init__.py -------------------------------------------------------------------------------- /tests/unit/copy/periodic_copy/test_init.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.exceptions import WrongInputData 4 | from twindb_backup.copy.periodic_copy import PeriodicCopy 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "path, host, run_type, name", 9 | [ 10 | ("/foo/bar/hourly/mysql/mysql.tar.gz", "bar", "hourly", "mysql.tar.gz"), 11 | ( 12 | "s3://twindb-www.twindb.com/master1/hourly/mysql/mysql-2016-11-23_21_50_54.xbstream.gz", 13 | "master1", 14 | "hourly", 15 | "mysql-2016-11-23_21_50_54.xbstream.gz", 16 | ), 17 | ( 18 | "s3://twindb-www.twindb.com/master.box/hourly/mysql/mysql-2016-11-23_08_01_25.xbstream.gz", 19 | "master.box", 20 | "hourly", 21 | "mysql-2016-11-23_08_01_25.xbstream.gz", 22 | ), 23 | ( 24 | "/path/to/twindb-server-backups/master1/daily/mysql/mysql-2016-11-23_21_47_21.xbstream.gz", 25 | "master1", 26 | "daily", 27 | "mysql-2016-11-23_21_47_21.xbstream.gz", 28 | ), 29 | ], 30 | ) 31 | def test_copy_from_path(path, host, run_type, name): 32 | copy = PeriodicCopy(path=path) 33 | assert copy.host == host 34 | assert copy.run_type == run_type 35 | assert copy.name == name 36 | 37 | 38 | def test_init_raises_on_wrong_inputs(): 39 | with pytest.raises(WrongInputData): 40 | PeriodicCopy(path="foo") 41 | -------------------------------------------------------------------------------- /tests/unit/copy/periodic_copy/test_key.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.exceptions import UnknownSourceType 4 | from twindb_backup.copy.periodic_copy import PeriodicCopy 5 | 6 | 7 | def test_key_raises(): 8 | backup_copy = PeriodicCopy("foo", "daily", "some_file.txt") 9 | 10 | with pytest.raises(UnknownSourceType): 11 | assert backup_copy.key 12 | -------------------------------------------------------------------------------- /tests/unit/destination/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "aleks" 2 | -------------------------------------------------------------------------------- /tests/unit/destination/az/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/az/__init__.py -------------------------------------------------------------------------------- /tests/unit/destination/az/test_config.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.configuration.destinations.az import AZConfig 4 | 5 | from .util import AZConfigParams 6 | 7 | 8 | def test_initialization_success(): 9 | """Test initialization of AZConfig with all parameters set.""" 10 | p = AZConfigParams() 11 | c = AZConfig(**dict(p)) 12 | assert c.connection_string == p.connection_string 13 | assert c.container_name == p.container_name 14 | assert c.chunk_size == p.chunk_size 15 | assert c.remote_path == p.remote_path 16 | 17 | 18 | def test_initialization_success_defaults(): 19 | """Test initialization of AZConfig with only required parameters set and ensure default values.""" 20 | p = AZConfigParams(only_required=True) 21 | c = AZConfig(**dict(p)) 22 | assert c.connection_string == p.connection_string 23 | assert c.container_name == p.container_name 24 | assert c.chunk_size == 4 * 1024 * 1024 25 | assert c.remote_path == "/" 26 | 27 | 28 | def test_invalid_params(): 29 | """Test initialization of AZConfig with invalid parameters.""" 30 | with pytest.raises(ValueError): 31 | AZConfig( 32 | connection_string="test_connection_string", container_name="test_container", chunk_size="invalid_chunk_size" 33 | ) 34 | with pytest.raises(ValueError): 35 | AZConfig(connection_string="test_connection_string", container_name="test_container", remote_path=1) 36 | with pytest.raises(TypeError): 37 | AZConfig(connection_string="test_connection_string") 38 | -------------------------------------------------------------------------------- /tests/unit/destination/az/test_delete.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | import twindb_backup.destination.az as az 4 | 5 | from .util import mocked_az 6 | 7 | 8 | def test_delete_success(): 9 | """Tests AZ.delete method, ensuring the blob is deleted.""" 10 | c = mocked_az() 11 | 12 | c.delete("test") 13 | c._container_client.delete_blob.assert_called_once_with(c.render_path("test")) 14 | 15 | 16 | def test_delete_fail(): 17 | """Tests AZ.delete method, re-raising an exception on failure""" 18 | c = mocked_az() 19 | c._container_client.delete_blob.side_effect = Exception() 20 | 21 | with pytest.raises(Exception): 22 | c.delete("test") 23 | c._container_client.delete_blob.assert_called_once_with(c.render_path("test")) 24 | -------------------------------------------------------------------------------- /tests/unit/destination/az/test_download_to_pipe.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import MagicMock, patch 2 | 3 | import azure.core.exceptions as ae 4 | import pytest 5 | 6 | from .util import mocked_az 7 | 8 | 9 | def test_download_to_pipe_success(): 10 | """Tests AZ.download_to_pipe method, mocks calls for os and ContainerClient""" 11 | with patch("twindb_backup.destination.az.os") as mc_os: 12 | mc_fdopen = MagicMock() 13 | mc_os.fdopen.return_value = mc_fdopen 14 | 15 | c = mocked_az() 16 | 17 | mc_dbr = MagicMock() 18 | c._container_client.download_blob.return_value = mc_dbr 19 | 20 | c._download_to_pipe(c.render_path("foo-key"), 100, 200) 21 | 22 | mc_os.close.assert_called_once_with(100) 23 | mc_os.fdopen.assert_called_once_with(200, "wb") 24 | c._container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) 25 | mc_dbr.readinto.assert_called_once_with(mc_fdopen.__enter__()) 26 | 27 | 28 | def test_download_to_pipe_fail(): 29 | """Tests AZ.download_to_pipe method, re-raises exception when download fails in child process""" 30 | with patch("twindb_backup.destination.az.os") as mc_os: 31 | c = mocked_az() 32 | 33 | c._container_client.download_blob.side_effect = ae.HttpResponseError() 34 | 35 | with pytest.raises(Exception): 36 | c._download_to_pipe(c.render_path("foo-key"), 100, 200) 37 | 38 | mc_os.close.assert_called_once_with(100) 39 | mc_os.fdopen.assert_called_once_with(200, "wb") 40 | c._container_client.download_blob.assert_called_once_with(c.render_path("foo-key")) 41 | -------------------------------------------------------------------------------- /tests/unit/destination/az/test_render_path.py: -------------------------------------------------------------------------------- 1 | from .util import mocked_az 2 | 3 | 4 | def test_render_path(): 5 | """Test render_path method, ensuring the remote path is prepended to the path.""" 6 | c = mocked_az() 7 | 8 | assert c.render_path("test") == f"{c.remote_path}/test" 9 | -------------------------------------------------------------------------------- /tests/unit/destination/az/test_save.py: -------------------------------------------------------------------------------- 1 | from typing import BinaryIO 2 | from unittest.mock import MagicMock 3 | 4 | import azure.core.exceptions as ae 5 | import pytest 6 | 7 | from .util import mocked_az 8 | 9 | EXAMPLE_FILE = "test/backup.tar.gz" 10 | 11 | 12 | def test_save_success(): 13 | """Tests AZ.save method, ensuring the blob is saved to azure.""" 14 | c = mocked_az() 15 | handler = MagicMock(BinaryIO) 16 | file_obj = MagicMock() 17 | handler.__enter__.return_value = file_obj 18 | handler.__exit__.return_value = None 19 | 20 | c.save(handler, EXAMPLE_FILE) 21 | 22 | c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) 23 | 24 | 25 | def test_save_fail(): 26 | """Tests AZ.save method, re-raises an exception on failure""" 27 | c = mocked_az() 28 | handler = MagicMock(BinaryIO) 29 | file_obj = MagicMock() 30 | handler.__enter__.return_value = file_obj 31 | handler.__exit__.return_value = None 32 | c._container_client.upload_blob.side_effect = ae.HttpResponseError() 33 | 34 | with pytest.raises(Exception): 35 | c.save(handler, EXAMPLE_FILE) 36 | 37 | c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), file_obj) 38 | -------------------------------------------------------------------------------- /tests/unit/destination/az/test_write.py: -------------------------------------------------------------------------------- 1 | import azure.core.exceptions as ae 2 | import pytest 3 | 4 | from .util import mocked_az 5 | 6 | EXAMPLE_FILE = "test/backup.tar.gz" 7 | CONTENT = b"test content" 8 | 9 | 10 | def test_write_success(): 11 | """Tests AZ.write method, ensuring the blob is written to azure.""" 12 | c = mocked_az() 13 | 14 | c.write(CONTENT, EXAMPLE_FILE) 15 | 16 | c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) 17 | 18 | 19 | def test_write_fail(): 20 | """Tests AZ.write method, re-raises an exception on failure""" 21 | c = mocked_az() 22 | c._container_client.upload_blob.side_effect = ae.HttpResponseError() 23 | 24 | with pytest.raises(Exception): 25 | c.write(CONTENT, EXAMPLE_FILE) 26 | 27 | c._container_client.upload_blob.assert_called_once_with(c.render_path(EXAMPLE_FILE), CONTENT, overwrite=True) 28 | -------------------------------------------------------------------------------- /tests/unit/destination/base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/base/__init__.py -------------------------------------------------------------------------------- /tests/unit/destination/base/test_init.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.destination.base_destination import BaseDestination 4 | from twindb_backup.destination.exceptions import DestinationError 5 | 6 | 7 | def test_init_raises(): 8 | with pytest.raises(DestinationError): 9 | BaseDestination(None) 10 | -------------------------------------------------------------------------------- /tests/unit/destination/gcs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/gcs/__init__.py -------------------------------------------------------------------------------- /tests/unit/destination/gcs/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.destination.gcs import GCS 4 | 5 | 6 | @pytest.fixture 7 | def gs(): 8 | return GCS(bucket="test-bucket", gc_credentials_file="foo") 9 | -------------------------------------------------------------------------------- /tests/unit/destination/gcs/test_create_bucket.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | from google.cloud.exceptions import Conflict 4 | 5 | from twindb_backup.destination.exceptions import GCSDestinationError 6 | from twindb_backup.destination.gcs import GCS 7 | 8 | 9 | @mock.patch.object(GCS, "_gcs_client") 10 | def test_create_bucket_raises(mock_client): 11 | 12 | mock_client.create_bucket.side_effect = Conflict("error") 13 | 14 | gs = GCS(bucket="foo", gc_credentials_file="foo/bar") 15 | with pytest.raises(GCSDestinationError): 16 | gs.create_bucket() 17 | 18 | 19 | @mock.patch.object(GCS, "_gcs_client") 20 | def test_create_bucket(mock_client): 21 | 22 | gs = GCS(bucket="foo", gc_credentials_file="foo/bar") 23 | gs.create_bucket() 24 | mock_client.create_bucket.assert_called_once_with(bucket_name="foo") 25 | -------------------------------------------------------------------------------- /tests/unit/destination/gcs/test_delete_bucket.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | from google.cloud.exceptions import NotFound 4 | 5 | from twindb_backup.destination.exceptions import GCSDestinationError 6 | from twindb_backup.destination.gcs import GCS 7 | 8 | 9 | @mock.patch.object(GCS, "_gcs_client") 10 | def test_delete_bucket_raises(mock_client): 11 | 12 | mock_client.get_bucket.side_effect = NotFound("error") 13 | 14 | gs = GCS(bucket="foo", gc_credentials_file="foo/bar") 15 | with pytest.raises(GCSDestinationError): 16 | gs.delete_bucket() 17 | 18 | 19 | @pytest.mark.parametrize("force", [True, False]) 20 | @mock.patch.object(GCS, "_gcs_client") 21 | def test_delete_bucket(mock_client, force): 22 | 23 | mock_bucket = mock.Mock() 24 | mock_client.get_bucket.return_value = mock_bucket 25 | 26 | gs = GCS(bucket="foo", gc_credentials_file="foo/bar") 27 | gs.delete_bucket(force=force) 28 | mock_bucket.delete.assert_called_once_with(force=force) 29 | -------------------------------------------------------------------------------- /tests/unit/destination/gcs/test_get_stream.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.destination.gcs import GCS 4 | 5 | 6 | @mock.patch("twindb_backup.destination.gcs.Process") 7 | @mock.patch("twindb_backup.destination.gcs.os") 8 | def test_get_stream_calls_popen(mock_os, mock_process): 9 | mock_os.pipe.return_value = (100, 200) 10 | gs = GCS(gc_credentials_file="foo", bucket="bar") 11 | mock_copy = mock.Mock() 12 | mock_copy.key = "foo-key" 13 | 14 | with gs.get_stream(mock_copy): 15 | pass 16 | 17 | mock_process.assert_called_once_with(target=gs._download_to_pipe, args=("foo-key", 100, 200)) 18 | -------------------------------------------------------------------------------- /tests/unit/destination/gcs/test_init.py: -------------------------------------------------------------------------------- 1 | from os import environ 2 | 3 | import mock 4 | import pytest 5 | 6 | from twindb_backup.destination.base_destination import BaseDestination 7 | from twindb_backup.destination.exceptions import GCSDestinationError 8 | from twindb_backup.destination.gcs import GCS 9 | 10 | 11 | def test_init_set_env(): 12 | # make sure GOOGLE_APPLICATION_CREDENTIALS is not set 13 | if "GOOGLE_APPLICATION_CREDENTIALS" in environ: 14 | del environ["GOOGLE_APPLICATION_CREDENTIALS"] 15 | with pytest.raises(KeyError): 16 | assert environ["GOOGLE_APPLICATION_CREDENTIALS"] 17 | 18 | GCS(bucket="bar", gc_credentials_file="foo") 19 | assert environ["GOOGLE_APPLICATION_CREDENTIALS"] == "foo" 20 | 21 | 22 | def test_init_set_bucket(): 23 | gs = GCS(gc_credentials_file="foo", bucket="bar") 24 | assert gs.bucket == "bar" 25 | 26 | 27 | @mock.patch.object(BaseDestination, "__init__") 28 | def test_init_set_bucket(mock_base): 29 | mock_base.return_value = None 30 | gs = GCS(gc_credentials_file="foo", bucket="bar") 31 | assert gs.bucket == "bar" 32 | mock_base.assert_called_once_with("bar") 33 | 34 | 35 | def test_init_raises_if_no_file(): 36 | with pytest.raises(GCSDestinationError): 37 | GCS(bucket="foo") 38 | -------------------------------------------------------------------------------- /tests/unit/destination/local/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/local/__init__.py -------------------------------------------------------------------------------- /tests/unit/destination/s3/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/s3/__init__.py -------------------------------------------------------------------------------- /tests/unit/destination/s3/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.destination.s3 import S3 4 | 5 | 6 | @pytest.fixture 7 | def s3(): 8 | return S3( 9 | bucket="test-bucket", 10 | aws_access_key_id="access_key", 11 | aws_secret_access_key="secret_key", 12 | ) 13 | -------------------------------------------------------------------------------- /tests/unit/destination/s3/test_create_bucket.py: -------------------------------------------------------------------------------- 1 | from moto import mock_s3 2 | 3 | 4 | @mock_s3 5 | def test__create_bucket_creates_the_bucket(s3): 6 | s3.create_bucket() 7 | 8 | assert s3.s3_client.head_bucket(Bucket="test-bucket") 9 | -------------------------------------------------------------------------------- /tests/unit/destination/s3/test_delete.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from botocore.exceptions import ClientError 3 | from moto import mock_s3 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "path, key", 8 | [ 9 | ("test_server/hourly/file1.txt", "test_server/hourly/file1.txt"), 10 | ( 11 | "s3://test-bucket/test_server/hourly/file1.txt", 12 | "test_server/hourly/file1.txt", 13 | ), 14 | ], 15 | ) 16 | @mock_s3 17 | def test__delete_can_delete_an_object(path, key, s3): 18 | twindb_s3 = s3 19 | twindb_s3.create_bucket() 20 | 21 | twindb_s3.s3_client.put_object(Body="hello world", Bucket="test-bucket", Key=key) 22 | 23 | assert twindb_s3.delete(path) 24 | with pytest.raises(ClientError): 25 | twindb_s3.s3_client.head_object(Bucket="test-bucket", Key=key) 26 | -------------------------------------------------------------------------------- /tests/unit/destination/s3/test_delete_bucket.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from botocore.exceptions import ClientError 3 | from moto import mock_s3 4 | 5 | 6 | @mock_s3 7 | def test__delete_bucket_deletes_the_bucket(s3): 8 | s3.create_bucket() 9 | s3.delete_bucket() 10 | 11 | with pytest.raises(ClientError): 12 | s3.s3_client.head_bucket(Bucket="test-bucket") 13 | -------------------------------------------------------------------------------- /tests/unit/destination/s3/test_get_file_content.py: -------------------------------------------------------------------------------- 1 | import mock 2 | from botocore.exceptions import ClientError 3 | 4 | 5 | def test_get_file_content(s3): 6 | mock_body = mock.Mock() 7 | s3.s3_client = mock.Mock() 8 | s3.s3_client.get_object.return_value = {"Body": mock_body} 9 | s3.validate_client_response = mock.Mock() 10 | 11 | # noinspection PyProtectedMember 12 | s3._get_file_content("foo") 13 | s3.s3_client.get_object.assert_called_once_with(Bucket="test-bucket", Key="foo") 14 | s3.validate_client_response.assert_called_once_with({"Body": mock_body}) 15 | mock_body.read.assert_called_once_with() 16 | 17 | 18 | @mock.patch("twindb_backup.destination.s3.time.sleep") 19 | def test_get_file_content_retry(mock_sleep, s3): 20 | 21 | mock_body = mock.Mock() 22 | s3.s3_client = mock.Mock() 23 | 24 | mock_error_response = {"ResponseMetadata": {"MaxAttemptsReached": True}} 25 | s3.s3_client.get_object.side_effect = [ 26 | ClientError(mock_error_response, "GetObject"), 27 | ClientError(mock_error_response, "GetObject"), 28 | {"Body": mock_body}, 29 | ] 30 | s3.validate_client_response = mock.Mock() 31 | 32 | # noinspection PyProtectedMember 33 | s3._get_file_content("foo") 34 | assert s3.s3_client.get_object.call_count == 3 35 | -------------------------------------------------------------------------------- /tests/unit/destination/ssh/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/ssh/__init__.py -------------------------------------------------------------------------------- /tests/unit/destination/ssh/test_find_files.py: -------------------------------------------------------------------------------- 1 | # noinspection PyPackageRequirements 2 | import mock 3 | 4 | from twindb_backup.destination.ssh import Ssh 5 | 6 | # noinspection PyUnresolvedReferences 7 | # @mock.patch.object(Ssh, '__init__') 8 | # def test_find_files(mock_init): 9 | # mock_init.return_value = mock_init 10 | # 11 | # dst = Ssh(remote_path='/foo/bar') 12 | # dst.find_files('/foo/bar', 'abc') 13 | # mock_get_remote_handlers.return_value = None, None, None 14 | # mock_get_remote_handlers.assert_called_once_with( 15 | # 'find /foo/bar/*/abc -type f' 16 | # ) 17 | -------------------------------------------------------------------------------- /tests/unit/destination/ssh/test_get_remote_stdout.py: -------------------------------------------------------------------------------- 1 | def test_get_remote_stdout(): 2 | pass 3 | -------------------------------------------------------------------------------- /tests/unit/destination/ssh/test_list_files.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.destination.ssh import Ssh 4 | 5 | 6 | def test_list_files(): 7 | dst = Ssh("/var/backups") 8 | mock_client = mock.Mock() 9 | mock_client.list_files.return_value = ["foo", "bar"] 10 | dst._ssh_client = mock_client 11 | assert dst.list_files("xxx", pattern="foo") == ["foo"] 12 | -------------------------------------------------------------------------------- /tests/unit/destination/ssh/test_mkdir_r.py: -------------------------------------------------------------------------------- 1 | # noinspection PyPackageRequirements 2 | import mock 3 | 4 | from twindb_backup.destination.ssh import Ssh 5 | 6 | 7 | # noinspection PyUnresolvedReferences 8 | @mock.patch.object(Ssh, "execute_command") 9 | def test_mkdir_r(mock_execute): 10 | 11 | mock_stdout = mock.Mock() 12 | mock_stdout.channel.recv_exit_status.return_value = 0 13 | 14 | mock_execute.return_value = mock_stdout, mock.Mock() 15 | 16 | dst = Ssh(remote_path="some_dir") 17 | # noinspection PyProtectedMember 18 | dst._mkdir_r("/foo/bar") 19 | mock_execute.assert_called_once_with('mkdir -p "/foo/bar"') 20 | -------------------------------------------------------------------------------- /tests/unit/destination/ssh/test_write_status.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/ssh/test_write_status.py -------------------------------------------------------------------------------- /tests/unit/destination/test_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/destination/test_base.py -------------------------------------------------------------------------------- /tests/unit/exporter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/exporter/__init__.py -------------------------------------------------------------------------------- /tests/unit/exporter/test_statsd_exporter.py: -------------------------------------------------------------------------------- 1 | import mock 2 | from statsd import StatsClient 3 | 4 | from twindb_backup.exporter.statsd_exporter import StatsdExporter 5 | 6 | 7 | def test_stasd_exporter_constructor(): 8 | with mock.patch.object(StatsClient, "__init__", return_value=None) as mock_StatsClient_init: 9 | exporter = StatsdExporter("localhost", 8125) 10 | assert exporter._suffix == "twindb." 11 | assert isinstance(exporter._client, StatsClient) 12 | mock_StatsClient_init.assert_called_once_with("localhost", 8125) 13 | -------------------------------------------------------------------------------- /tests/unit/modifiers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/modifiers/__init__.py -------------------------------------------------------------------------------- /tests/unit/modifiers/bzip2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/modifiers/bzip2/__init__.py -------------------------------------------------------------------------------- /tests/unit/modifiers/bzip2/test_get_stream.py: -------------------------------------------------------------------------------- 1 | from subprocess import PIPE 2 | 3 | import mock 4 | 5 | from twindb_backup.modifiers import Bzip2 6 | 7 | 8 | @mock.patch("twindb_backup.modifiers.base.Popen") 9 | def test_get_stream(mock_popen): 10 | mock_stream = mock.Mock() 11 | 12 | def foo(*args, **kwargs): 13 | pass 14 | 15 | mock_stream.__enter__ = foo 16 | mock_stream.__exit__ = foo 17 | 18 | m = Bzip2(mock_stream, level=3) 19 | with m.get_stream(): 20 | mock_popen.assert_called_once_with(["bzip2", "-3", "-c", "-"], stdin=None, stdout=PIPE, stderr=PIPE) 21 | -------------------------------------------------------------------------------- /tests/unit/modifiers/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def input_file(tmpdir): 6 | filename = tmpdir.join("in.txt") 7 | with open(str(filename), "w") as f: 8 | f.write("foo bar") 9 | return filename 10 | 11 | 12 | @pytest.fixture 13 | def keyring_file(tmpdir): 14 | public_file = tmpdir.join("keyring") 15 | with open(str(public_file), "w") as f: 16 | f.write("foo bar") 17 | return public_file 18 | 19 | 20 | @pytest.fixture 21 | def secret_keyring_file(tmpdir): 22 | secret_file = tmpdir.join("secret_keyring") 23 | with open(str(secret_file), "w") as f: 24 | f.write("foo bar") 25 | return secret_file 26 | -------------------------------------------------------------------------------- /tests/unit/modifiers/gzip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/modifiers/gzip/__init__.py -------------------------------------------------------------------------------- /tests/unit/modifiers/gzip/test_get_stream.py: -------------------------------------------------------------------------------- 1 | from subprocess import PIPE 2 | 3 | import mock 4 | 5 | from twindb_backup.modifiers import Gzip 6 | 7 | 8 | @mock.patch("twindb_backup.modifiers.base.Popen") 9 | def test_get_stream(mock_popen): 10 | mock_stream = mock.Mock() 11 | 12 | def foo(*args, **kwargs): 13 | pass 14 | 15 | mock_stream.__enter__ = foo 16 | mock_stream.__exit__ = foo 17 | 18 | m = Gzip(mock_stream, level=3) 19 | with m.get_stream(): 20 | mock_popen.assert_called_once_with(["gzip", "-3", "-c", "-"], stdin=None, stdout=PIPE, stderr=PIPE) 21 | -------------------------------------------------------------------------------- /tests/unit/modifiers/lbzip2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/modifiers/lbzip2/__init__.py -------------------------------------------------------------------------------- /tests/unit/modifiers/lbzip2/test_get_stream.py: -------------------------------------------------------------------------------- 1 | from subprocess import PIPE 2 | 3 | import mock 4 | 5 | from twindb_backup.modifiers import Lbzip2 6 | 7 | 8 | @mock.patch("twindb_backup.modifiers.base.Popen") 9 | def test_get_stream(mock_popen): 10 | mock_stream = mock.Mock() 11 | 12 | def foo(*args, **kwargs): 13 | pass 14 | 15 | mock_stream.__enter__ = foo 16 | mock_stream.__exit__ = foo 17 | 18 | m = Lbzip2(mock_stream, level=3, threads=123) 19 | with m.get_stream(): 20 | mock_popen.assert_called_once_with( 21 | ["lbzip2", "-3", "-n", "123", "-c", "-"], 22 | stdin=None, 23 | stdout=PIPE, 24 | stderr=PIPE, 25 | ) 26 | -------------------------------------------------------------------------------- /tests/unit/modifiers/pigz/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/modifiers/pigz/__init__.py -------------------------------------------------------------------------------- /tests/unit/modifiers/pigz/test_get_stream.py: -------------------------------------------------------------------------------- 1 | from subprocess import PIPE 2 | 3 | import mock 4 | 5 | from twindb_backup.modifiers import Pigz 6 | 7 | 8 | @mock.patch("twindb_backup.modifiers.base.Popen") 9 | def test_get_stream(mock_popen): 10 | mock_stream = mock.Mock() 11 | 12 | def foo(*args, **kwargs): 13 | pass 14 | 15 | mock_stream.__enter__ = foo 16 | mock_stream.__exit__ = foo 17 | 18 | m = Pigz(mock_stream, level=3, threads=123) 19 | with m.get_stream(): 20 | mock_popen.assert_called_once_with( 21 | ["pigz", "-3", "-p", "123", "-c", "-"], 22 | stdin=None, 23 | stdout=PIPE, 24 | stderr=PIPE, 25 | ) 26 | -------------------------------------------------------------------------------- /tests/unit/modifiers/test_base.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.modifiers.base import Modifier 2 | 3 | 4 | def test_modifier(tmpdir): 5 | input_stream = tmpdir.join("in.txt") 6 | f_in = open(str(input_stream), "w+") 7 | m = Modifier(f_in) 8 | assert m.input == f_in 9 | 10 | 11 | def test_modifier_get_stream(input_file): 12 | with open(str(input_file), "r") as f: 13 | m = Modifier(f) 14 | with m.get_stream() as m_f: 15 | assert m_f.read().decode("utf-8") == "foo bar" 16 | -------------------------------------------------------------------------------- /tests/unit/modifiers/test_keeplocal.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.modifiers.keeplocal import KeepLocal 2 | 3 | 4 | def test_keeplocal(input_file, tmpdir): 5 | local_dir = str(tmpdir.join("/foo/bar")) 6 | with open(str(input_file), "r") as f: 7 | m = KeepLocal(f, str(local_dir)) 8 | assert m.local_path == local_dir 9 | 10 | 11 | def test_keeplocal_saves_file(input_file, tmpdir): 12 | local_copy = tmpdir.join("foo") 13 | 14 | with open(str(input_file), "r") as f: 15 | m = KeepLocal(f, str(local_copy)) 16 | with m.get_stream() as m_f: 17 | remote_copy = m_f.read().decode("utf-8") 18 | with open(str(local_copy), "r") as l: 19 | assert l.read() == remote_copy 20 | -------------------------------------------------------------------------------- /tests/unit/restore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/restore/__init__.py -------------------------------------------------------------------------------- /tests/unit/restore/test_get_free_memory.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.restore import get_free_memory 2 | 3 | 4 | def test_get_free_memory(): 5 | mem = get_free_memory() 6 | assert isinstance(mem, int) 7 | assert mem > 0 8 | -------------------------------------------------------------------------------- /tests/unit/source/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = "aleks" 2 | -------------------------------------------------------------------------------- /tests/unit/source/base_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/base_source/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/base_source/test_suffix.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.base_source import BaseSource 2 | 3 | 4 | def test_suffix_update(): 5 | src = BaseSource("daily") 6 | src.suffix = "xbstream" 7 | assert src.suffix == "xbstream" 8 | src.suffix += ".gz" 9 | assert src.suffix == "xbstream.gz" 10 | -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/binlog_source/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/binlog_parser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/binlog_source/binlog_parser/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/binlog_parser/test_created_at.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.source.binlog_source import BinlogParser 4 | from twindb_backup.source.exceptions import BinlogSourceError 5 | 6 | 7 | def test_created_at(mysql_bin_000001): 8 | bp = BinlogParser(mysql_bin_000001) 9 | assert bp.created_at == 1533403742 10 | 11 | 12 | def test_created_at_raises(): 13 | bp = BinlogParser("foo") 14 | with pytest.raises(BinlogSourceError): 15 | assert bp.created_at 16 | -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/binlog_parser/test_end_position.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.binlog_source import BinlogParser 2 | 3 | 4 | def test_end_position(mysql_bin_000001): 5 | bp = BinlogParser(mysql_bin_000001) 6 | assert bp.end_position == 46860 7 | -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/binlog_parser/test_name.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.binlog_source import BinlogParser 2 | 3 | 4 | def test_name(mysql_bin_000001): 5 | bp = BinlogParser(mysql_bin_000001) 6 | assert bp.name == "mysql-bin.000001" 7 | -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/binlog_parser/test_start_position.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.binlog_source import BinlogParser 2 | 3 | 4 | def test_start_position(mysql_bin_000001): 5 | bp = BinlogParser(mysql_bin_000001) 6 | assert bp.start_position == 4 7 | -------------------------------------------------------------------------------- /tests/unit/source/binlog_source/test_get_name.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.source.binlog_source import BinlogSource 4 | 5 | 6 | @mock.patch("twindb_backup.source.base_source.socket.gethostname") 7 | def test_get_name(mock_gethostname): 8 | 9 | mock_gethostname.return_value = "some-host" 10 | src = BinlogSource("daily", mock.Mock(), "foo-bin-log.0001") 11 | 12 | assert src.get_name() == "some-host/binlog/foo-bin-log.0001" 13 | -------------------------------------------------------------------------------- /tests/unit/source/file_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/file_source/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/file_source/test_get_stream.py: -------------------------------------------------------------------------------- 1 | """Tests that cover the FileSource().get_stream() method.""" 2 | 3 | import mock 4 | import pytest 5 | 6 | from twindb_backup.source.file_source import FileSource 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "tar_options, expected_command", 11 | [ 12 | (None, ["tar", "cf", "-", "foo"]), 13 | ("--exclude-vcs-ignores", ["tar", "cf", "-", "--exclude-vcs-ignores", "foo"]), 14 | ( 15 | "--exclude-vcs-ignores --exclude-ignore=FILE", 16 | ["tar", "cf", "-", "--exclude-vcs-ignores", "--exclude-ignore=FILE", "foo"], 17 | ), 18 | ], 19 | ) 20 | @mock.patch("twindb_backup.source.file_source.Popen") 21 | def test_get_stream(mock_popen, tar_options, expected_command): 22 | """Make sure tar command is built properly.""" 23 | mock_proc = mock.Mock() 24 | mock_proc.communicate.return_value = None, None 25 | mock_proc.returncode = 0 26 | 27 | mock_popen.return_value = mock_proc 28 | 29 | src = FileSource("foo", "daily", tar_options=tar_options) 30 | with src.get_stream(): 31 | pass 32 | 33 | mock_popen.assert_called_once_with(expected_command, stderr=-1, stdout=-1) 34 | -------------------------------------------------------------------------------- /tests/unit/source/file_source/test_suffix.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.source.file_source import FileSource 2 | 3 | 4 | def test_suffix(): 5 | fs = FileSource("/foo/bar", "daily") 6 | assert fs.suffix == "tar" 7 | fs.suffix += ".gz" 8 | assert fs.suffix == "tar.gz" 9 | -------------------------------------------------------------------------------- /tests/unit/source/mariadb_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/mariadb_source/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/mariadb_source/test_backup_tool.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup import MARIABACKUP_BINARY 4 | from twindb_backup.source.mariadb_source import MariaDBSource 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo 6 | 7 | 8 | @pytest.mark.parametrize( 9 | "kwargs, expected_tool", 10 | [ 11 | ({}, MARIABACKUP_BINARY), 12 | ({"xtrabackup_binary": None}, MARIABACKUP_BINARY), 13 | ({"xtrabackup_binary": "foo"}, "foo"), 14 | ], 15 | ) 16 | def test_backup_tool(kwargs, expected_tool): 17 | ms = MariaDBSource(MySQLConnectInfo("foo"), "daily", "full", **kwargs) 18 | assert ms.backup_tool == expected_tool 19 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/mysql_source/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/mysql_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/mysql_source/mysql_client/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/mysql_client/test_server_vendor.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup.source.mysql_source import MySQLClient, MySQLFlavor 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "version, version_comment, vendor", 9 | [ 10 | ( 11 | "10.3.34-MariaDB-0ubuntu0.20.04.1-log", 12 | "Ubuntu 20.04", 13 | MySQLFlavor.MARIADB, 14 | ), 15 | ("8.0.28", "MySQL Community Server - GPL", MySQLFlavor.ORACLE), 16 | ( 17 | "8.0.28-19", 18 | "Percona Server (GPL), Release 19, Revision 31e88966cd3", 19 | MySQLFlavor.PERCONA, 20 | ), 21 | ], 22 | ) 23 | def test_server_vendor(version, version_comment, vendor): 24 | with mock.patch.object(MySQLClient, "variable", side_effect=[version, version_comment]): 25 | assert MySQLClient("foo").server_vendor == vendor 26 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/mysql_flavor.py: -------------------------------------------------------------------------------- 1 | from json import dumps 2 | 3 | from twindb_backup.source.mariadb_source import MariaDBSource 4 | from twindb_backup.source.mysql_source import MySQLFlavor, MySQLSource 5 | 6 | 7 | def test_eq(): 8 | assert MySQLFlavor.ORACLE == MySQLFlavor.ORACLE 9 | assert MySQLFlavor.ORACLE == "oracle" 10 | assert "oracle" == MySQLFlavor.ORACLE 11 | assert MySQLFlavor.ORACLE != MySQLFlavor.PERCONA 12 | 13 | 14 | def test_json(): 15 | assert dumps(MySQLFlavor.ORACLE) == '"oracle"' 16 | 17 | 18 | def test_hash(): 19 | mysql_src_map = { 20 | MySQLFlavor.MARIADB: MariaDBSource, 21 | MySQLFlavor.ORACLE: MySQLSource, 22 | MySQLFlavor.PERCONA: MySQLSource, 23 | } 24 | assert mysql_src_map[MySQLFlavor.MARIADB] is MariaDBSource 25 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_backup_tool.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup import XTRABACKUP_BINARY 4 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "kwargs, expected_tool", 9 | [ 10 | ({}, XTRABACKUP_BINARY), 11 | ({"xtrabackup_binary": None}, XTRABACKUP_BINARY), 12 | ({"xtrabackup_binary": "foo"}, "foo"), 13 | ], 14 | ) 15 | def test_backup_tool(kwargs, expected_tool): 16 | ms = MySQLSource(MySQLConnectInfo("foo"), "daily", "full", **kwargs) 17 | assert ms.backup_tool == expected_tool 18 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_disable_wsrep_desync.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mock 4 | 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | @mock.patch.object(MySQLSource, "get_connection") 9 | def test__disable_wsrep_desync_sets_wsrep_desync_to_off(mock_connect): 10 | logging.basicConfig() 11 | 12 | mock_cursor = mock.MagicMock() 13 | mock_cursor.fetchall.return_value = [ 14 | {"Variable_name": "wsrep_local_recv_queue", "Value": "0"}, 15 | ] 16 | 17 | mock_connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value = mock_cursor 18 | 19 | source = MySQLSource(MySQLConnectInfo(None), "daily", "full") 20 | source.disable_wsrep_desync() 21 | 22 | mock_cursor.execute.assert_any_call("SHOW GLOBAL STATUS LIKE " "'wsrep_local_recv_queue'") 23 | mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=OFF") 24 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_enable_wsrep_desync.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mock 4 | 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | @mock.patch.object(MySQLSource, "get_connection") 9 | def test__enable_wsrep_desync_sets_wsrep_desync_to_on(mock_connect): 10 | logging.basicConfig() 11 | 12 | mock_cursor = mock.MagicMock() 13 | 14 | mock_connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value = mock_cursor 15 | 16 | source = MySQLSource(MySQLConnectInfo(None), "daily", "full") 17 | source.enable_wsrep_desync() 18 | 19 | mock_cursor.execute.assert_called_with("SET GLOBAL wsrep_desync=ON") 20 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_get_connection.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | from pymysql import OperationalError 4 | 5 | from twindb_backup.source.exceptions import MySQLSourceError 6 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 7 | 8 | 9 | @mock.patch("twindb_backup.source.mysql_source.pymysql.connect") 10 | def test__get_connection_raises_mysql_source_error(mock_connect): 11 | mock_connect.side_effect = OperationalError 12 | source = MySQLSource(MySQLConnectInfo(None, hostname=None), "daily", "full") 13 | with pytest.raises(MySQLSourceError): 14 | with source.get_connection(): 15 | pass 16 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_get_name.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 4 | 5 | 6 | @mock.patch("twindb_backup.source.base_source.socket") 7 | @mock.patch("twindb_backup.source.base_source.time") 8 | def test_get_name(mock_time, mock_socket): 9 | 10 | host = "some-host" 11 | mock_socket.gethostname.return_value = host 12 | timestamp = "2017-02-13_15_40_29" 13 | mock_time.strftime.return_value = timestamp 14 | 15 | src = MySQLSource(MySQLConnectInfo("/foo/bar"), "daily", "full", dst=mock.Mock()) 16 | 17 | assert src.get_name() == "some-host/daily/mysql/mysql-2017-02-13_15_40_29.xbstream" 18 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_init.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup.source.exceptions import MySQLSourceError 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | def test_mysql_source_has_methods(): 9 | src = MySQLSource(MySQLConnectInfo("/foo/bar"), "hourly", "full", dst=mock.Mock()) 10 | assert src._connect_info.defaults_file == "/foo/bar" 11 | assert src.run_type == "hourly" 12 | assert src.suffix == "xbstream" 13 | assert src._media_type == "mysql" 14 | 15 | 16 | def test_mysql_source_raises_on_wrong_connect_info(): 17 | with pytest.raises(MySQLSourceError): 18 | # noinspection PyTypeChecker 19 | MySQLSource("/foo/bar", "hourly", "full", dst=mock.Mock()) 20 | 21 | 22 | @pytest.mark.parametrize("run_type", ["foo", None, ""]) 23 | def test_mysql_raises_on_wrong_run_type(run_type): 24 | with pytest.raises(MySQLSourceError): 25 | MySQLSource(MySQLConnectInfo("/foo/bar"), "foo", "full", dst=mock.Mock()) 26 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_suffix.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup import INTERVALS 4 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 5 | 6 | 7 | def test_suffix(): 8 | fs = MySQLSource(MySQLConnectInfo("/foo/bar"), INTERVALS[0], "full") 9 | assert fs.suffix == "xbstream" 10 | fs.suffix += ".gz" 11 | assert fs.suffix == "xbstream.gz" 12 | -------------------------------------------------------------------------------- /tests/unit/source/mysql_source/test_wsrep_provider_version.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import mock 4 | 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo, MySQLSource 6 | 7 | 8 | @mock.patch.object(MySQLSource, "get_connection") 9 | def test__wsrep_provider_version_returns_correct_version(mock_connect): 10 | logging.basicConfig() 11 | 12 | mock_cursor = mock.MagicMock() 13 | mock_cursor.fetchall.return_value = [ 14 | {"Variable_name": "wsrep_provider_version", "Value": "3.19(rb98f92f)"}, 15 | ] 16 | 17 | mock_connect.return_value.__enter__.return_value.cursor.return_value.__enter__.return_value = mock_cursor 18 | 19 | source = MySQLSource(MySQLConnectInfo(None), "daily", "full") 20 | assert source.wsrep_provider_version == "3.19" 21 | -------------------------------------------------------------------------------- /tests/unit/source/remote_mariadb_source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/source/remote_mariadb_source/__init__.py -------------------------------------------------------------------------------- /tests/unit/source/remote_mariadb_source/test_get_stream.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from mock.mock import Mock 3 | 4 | from twindb_backup import INTERVALS 5 | from twindb_backup.source.mysql_source import MySQLConnectInfo 6 | from twindb_backup.source.remote_mariadb_source import RemoteMariaDBSource 7 | 8 | 9 | def test_get_stream(): 10 | with pytest.raises(NotImplementedError): 11 | RemoteMariaDBSource( 12 | { 13 | "mysql_connect_info": Mock(spec=MySQLConnectInfo), 14 | "run_type": INTERVALS[0], 15 | "backup_type": "full", 16 | } 17 | ).get_stream() 18 | -------------------------------------------------------------------------------- /tests/unit/source/remote_mariadb_source/test_init.py: -------------------------------------------------------------------------------- 1 | from mock.mock import Mock 2 | 3 | from twindb_backup import INTERVALS, MARIABACKUP_BINARY 4 | from twindb_backup.source.mysql_source import MySQLConnectInfo 5 | from twindb_backup.source.remote_mariadb_source import RemoteMariaDBSource 6 | 7 | 8 | def test_init(): 9 | assert ( 10 | RemoteMariaDBSource( 11 | { 12 | "mysql_connect_info": Mock(spec=MySQLConnectInfo), 13 | "run_type": INTERVALS[0], 14 | "backup_type": "full", 15 | } 16 | ).backup_tool 17 | == MARIABACKUP_BINARY 18 | ) 19 | -------------------------------------------------------------------------------- /tests/unit/ssh/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/ssh/__init__.py -------------------------------------------------------------------------------- /tests/unit/ssh/ssh_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/ssh/ssh_client/__init__.py -------------------------------------------------------------------------------- /tests/unit/status/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/status/__init__.py -------------------------------------------------------------------------------- /tests/unit/status/base_status/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/status/base_status/__init__.py -------------------------------------------------------------------------------- /tests/unit/status/base_status/test_get_latest_backup.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.base_status import BaseStatus 2 | 3 | 4 | def test_latest_status_none(): 5 | status = BaseStatus() 6 | assert status.latest_backup is None 7 | -------------------------------------------------------------------------------- /tests/unit/status/base_status/test_remove.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.base_copy import BaseCopy 2 | from twindb_backup.status.base_status import BaseStatus 3 | 4 | 5 | def test_remove_by_key(): 6 | status = BaseStatus() 7 | copy = BaseCopy("foo", "bar") 8 | copy._source_type = "some_type" 9 | status.add(copy) 10 | assert len(status) == 1 11 | 12 | status.remove(copy.key) 13 | assert len(status) == 0 14 | 15 | 16 | def test_remove_by_full_path(): 17 | status = BaseStatus() 18 | copy = BaseCopy("foo", "bar") 19 | copy._source_type = "some_type" 20 | status.add(copy) 21 | assert len(status) == 1 22 | 23 | status.remove("blah" + copy.key) 24 | assert len(status) == 0 25 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/status/binlog_status/__init__.py -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def raw_binlog_status(): 6 | """ 7 | Returns base64 of binlog status 8 | { 9 | "master1/binlog/mysqlbin001.bin": { 10 | "time_created": "100500" 11 | }, 12 | "master1/binlog/mysqlbin002.bin": { 13 | "time_created": "100501" 14 | }, 15 | "master1/binlog/mysqlbin003.bin": { 16 | "time_created": "100502" 17 | }, 18 | "master1/binlog/mysqlbin004.bin": { 19 | "time_created": "100503" 20 | }, 21 | "master1/binlog/mysqlbin005.bin": { 22 | "time_created": "100504" 23 | } 24 | } 25 | """ 26 | return """ 27 | { 28 | "status": "ew0KICAibWFzdGVyMS9iaW5sb2cvbXlzcWxiaW4wMDEuYmluIjogew0KICAgICJ0aW1lX2NyZWF0ZWQiOiAiMTAwNTAwIg0KICB9LA0KICAibWFzdGVyMS9iaW5sb2cvbXlzcWxiaW4wMDIuYmluIjogew0KICAgICJ0aW1lX2NyZWF0ZWQiOiAiMTAwNTAxIg0KICB9LA0KICAibWFzdGVyMS9iaW5sb2cvbXlzcWxiaW4wMDMuYmluIjogew0KICAgICJ0aW1lX2NyZWF0ZWQiOiAiMTAwNTAyIg0KICB9LA0KICAibWFzdGVyMS9iaW5sb2cvbXlzcWxiaW4wMDQuYmluIjogew0KICAgICJ0aW1lX2NyZWF0ZWQiOiAiMTAwNTAzIg0KICB9LA0KICAibWFzdGVyMS9iaW5sb2cvbXlzcWxiaW4wMDUuYmluIjogew0KICAgICJ0aW1lX2NyZWF0ZWQiOiAiMTAwNTA0Ig0KICB9DQp9", 29 | "version": 1, 30 | "md5": "2cf1662594b5d873d94d3eacf8a1bcdf" 31 | } 32 | """ 33 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_add.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.binlog_copy import BinlogCopy 2 | from twindb_backup.status.binlog_status import BinlogStatus 3 | 4 | 5 | def test_add_to_empty_status(): 6 | instance = BinlogStatus() 7 | 8 | instance.add(BinlogCopy("host", "foo/bar", 100500)) 9 | assert BinlogCopy("host", "foo/bar", 100500) in instance 10 | 11 | instance.add(BinlogCopy("host", "foo/bar-2", 100501)) 12 | assert BinlogCopy("host", "foo/bar", 100500) in instance 13 | assert BinlogCopy("host", "foo/bar-2", 100501) in instance 14 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_basename.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.binlog_status import BinlogStatus 2 | 3 | 4 | def test_basename(): 5 | assert BinlogStatus().basename == "binlog-status" 6 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_eq.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.binlog_status import BinlogStatus 2 | 3 | 4 | def test_eq(raw_binlog_status): 5 | status_1 = BinlogStatus(raw_binlog_status) 6 | status_2 = BinlogStatus(raw_binlog_status) 7 | 8 | assert status_1 == status_2 9 | 10 | 11 | def test_ne(raw_binlog_status): 12 | status_1 = BinlogStatus(raw_binlog_status) 13 | status_2 = BinlogStatus() 14 | 15 | assert status_1 != status_2 16 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_get_item.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.status.binlog_status import BinlogStatus 4 | from twindb_backup.status.exceptions import StatusKeyNotFound 5 | 6 | 7 | def test_get_item_existing_copy(raw_binlog_status): 8 | instance = BinlogStatus(raw_binlog_status) 9 | copy = instance["master1/binlog/mysqlbin001.bin"] 10 | assert copy is not None 11 | 12 | 13 | def test_get_item_not_existing_copy(raw_binlog_status): 14 | instance = BinlogStatus(raw_binlog_status) 15 | with pytest.raises(StatusKeyNotFound): 16 | # noinspection PyStatementEffect 17 | instance["foo/bar"] 18 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_get_latest_backup.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.binlog_copy import BinlogCopy 2 | from twindb_backup.status.binlog_status import BinlogStatus 3 | 4 | 5 | def test_get_latest_backup(raw_binlog_status): 6 | instance = BinlogStatus(raw_binlog_status) 7 | assert instance.latest_backup == BinlogCopy(host="master1", name="mysqlbin005.bin", created_at=100504) 8 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_init.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.binlog_status import BinlogStatus 2 | 3 | 4 | def test_init_not_empty(raw_binlog_status): 5 | instance = BinlogStatus(raw_binlog_status) 6 | assert instance.version == 1 7 | assert len(instance) == 5 8 | 9 | 10 | def test_init_empty(): 11 | instance = BinlogStatus() 12 | assert len(instance) == 0 13 | 14 | 15 | def test_init_restores_name(raw_binlog_status): 16 | status = BinlogStatus(raw_binlog_status) 17 | copy = status["master1/binlog/mysqlbin001.bin"] 18 | assert copy.name == "mysqlbin001.bin" 19 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_remove.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.binlog_copy import BinlogCopy 4 | from twindb_backup.status.binlog_status import BinlogStatus 5 | from twindb_backup.status.exceptions import StatusKeyNotFound 6 | 7 | 8 | def test_remove_existing_copy(raw_binlog_status): 9 | instance = BinlogStatus(raw_binlog_status) 10 | copy = BinlogCopy(host="master1", name="mysqlbin001.bin", created_at=100500) 11 | assert copy in instance 12 | instance.remove("master1/binlog/mysqlbin001.bin") 13 | assert copy not in instance 14 | 15 | 16 | def test_remove_non_existing_copy(raw_binlog_status): 17 | instance = BinlogStatus(raw_binlog_status) 18 | with pytest.raises(StatusKeyNotFound): 19 | instance.remove("foo") 20 | -------------------------------------------------------------------------------- /tests/unit/status/binlog_status/test_serialize.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | 3 | from twindb_backup.status.binlog_status import BinlogStatus 4 | 5 | 6 | def test_serialize_doesnt_change_orignal(raw_binlog_status): 7 | 8 | status_original = BinlogStatus(content=raw_binlog_status) 9 | status_original_before = deepcopy(status_original) 10 | 11 | assert status_original == status_original_before 12 | status_original.serialize() 13 | assert status_original == status_original_before 14 | 15 | 16 | def test_serialize(raw_binlog_status): 17 | status_original = BinlogStatus(raw_binlog_status) 18 | status_converted = status_original.serialize() 19 | 20 | assert BinlogStatus(content=status_converted) == status_original 21 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/status/mysql_status/__init__.py -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_add.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | from twindb_backup.status.mysql_status import MySQLStatus 3 | 4 | 5 | def test_add(status_raw_empty, tmpdir): 6 | status = MySQLStatus(status_raw_empty) 7 | mycnf_1 = tmpdir.join("my-1.cnf") 8 | mycnf_1.write("some_content_1") 9 | mycnf_2 = tmpdir.join("my-2.cnf") 10 | mycnf_2.write("some_content_2") 11 | 12 | backup_copy = MySQLCopy( 13 | "master1", 14 | "daily", 15 | "foo.txt", 16 | binlog="binlog1", 17 | position=101, 18 | type="full", 19 | lsn=1230, 20 | backup_started=123, 21 | backup_finished=456, 22 | config_files=[str(mycnf_1), str(mycnf_2)], 23 | ) 24 | status.add(backup_copy) 25 | assert len(status.daily) == 1 26 | assert status.daily[backup_copy.key].binlog == "binlog1" 27 | assert status.daily[backup_copy.key].position == 101 28 | assert status.daily[backup_copy.key].type == "full" 29 | assert status.daily[backup_copy.key].lsn == 1230 30 | assert status.daily[backup_copy.key].backup_started == 123 31 | assert status.daily[backup_copy.key].backup_finished == 456 32 | assert status.daily[backup_copy.key].duration == 333 33 | 34 | config_content = status.daily[backup_copy.key].config[str(mycnf_1)] 35 | assert config_content == "some_content_1" 36 | 37 | config_content = status.daily[backup_copy.key].config[str(mycnf_2)] 38 | assert config_content == "some_content_2" 39 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_backup_duration.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.mysql_status import MySQLStatus 2 | 3 | 4 | def test_backup_duration(deprecated_status_raw_content): 5 | status = MySQLStatus(deprecated_status_raw_content) 6 | copy = status["master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz"] 7 | assert copy.duration == 19 8 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_basename.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.mysql_status import MySQLStatus 2 | 3 | 4 | def test_basename(): 5 | assert MySQLStatus().basename == "status" 6 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_candidate_parent.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | from twindb_backup.status.mysql_status import MySQLStatus 3 | 4 | 5 | def test_add(status_raw_empty, tmpdir): 6 | status = MySQLStatus(status_raw_empty) 7 | 8 | status.add( 9 | MySQLCopy( 10 | "master1", 11 | "daily", 12 | "foo-1.txt", 13 | binlog="binlog1", 14 | position=101, 15 | type="full", 16 | lsn=1230, 17 | backup_started=123, 18 | backup_finished=456, 19 | ) 20 | ) 21 | status.add( 22 | MySQLCopy( 23 | "master1", 24 | "daily", 25 | "foo-2.txt", 26 | binlog="binlog1", 27 | position=101, 28 | type="full", 29 | lsn=1231, 30 | backup_started=789, 31 | backup_finished=1000, 32 | ) 33 | ) 34 | assert status.candidate_parent("hourly").lsn == 1231 35 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_eq.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.mysql_status import MySQLStatus 2 | 3 | 4 | def test_eq(deprecated_status_raw_content): 5 | status_1 = MySQLStatus(content=deprecated_status_raw_content) 6 | status_2 = MySQLStatus(content=deprecated_status_raw_content) 7 | assert status_1 == status_2 8 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_get_item.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.copy.mysql_copy import MySQLCopy 2 | from twindb_backup.status.mysql_status import MySQLStatus 3 | 4 | 5 | def test_get_item_returns_copy_by_basename(deprecated_status_raw_content): 6 | status = MySQLStatus(deprecated_status_raw_content) 7 | key = "master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz" 8 | copy = status[key] 9 | assert type(copy) == MySQLCopy 10 | assert copy.run_type == "hourly" 11 | assert copy.host == "master1" 12 | assert copy.name == "mysql-2018-03-28_04_11_16.xbstream.gz" 13 | 14 | 15 | def test_get_item_returns_copy_by_basename_unicode( 16 | deprecated_status_raw_content, 17 | ): 18 | 19 | status = MySQLStatus(deprecated_status_raw_content) 20 | key = "master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz" 21 | copy = status[key] 22 | assert type(copy) == MySQLCopy 23 | assert copy.run_type == "hourly" 24 | assert copy.host == "master1" 25 | assert copy.name == "mysql-2018-03-28_04_11_16.xbstream.gz" 26 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_get_latest_backup.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.mysql_status import MySQLStatus 2 | 3 | 4 | def test_backup_duration(deprecated_status_raw_content): 5 | status = MySQLStatus(deprecated_status_raw_content) 6 | latest_key = "master1/hourly/mysql/mysql-2018-03-28_04_11_16.xbstream.gz" 7 | assert status.latest_backup.key == latest_key 8 | -------------------------------------------------------------------------------- /tests/unit/status/mysql_status/test_remove.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.copy.mysql_copy import MySQLCopy 4 | from twindb_backup.status.exceptions import StatusError, StatusKeyNotFound 5 | from twindb_backup.status.mysql_status import MySQLStatus 6 | 7 | 8 | def test_remove(status_raw_empty): 9 | status = MySQLStatus(status_raw_empty) 10 | copy = MySQLCopy("foo", "daily", "some_file.txt", type="full") 11 | status.add(copy) 12 | assert len(status.daily) == 1 13 | status.remove(copy.key) 14 | assert len(status.daily) == 0 15 | 16 | 17 | def test_remove_raises(status_raw_empty): 18 | status = MySQLStatus(status_raw_empty) 19 | with pytest.raises(StatusKeyNotFound): 20 | status.remove("foo") 21 | -------------------------------------------------------------------------------- /tests/unit/status/periodic_status/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/status/periodic_status/__init__.py -------------------------------------------------------------------------------- /tests/unit/status/periodic_status/test_add.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup import INTERVALS 4 | from twindb_backup.copy.periodic_copy import PeriodicCopy 5 | from twindb_backup.status.periodic_status import PeriodicStatus 6 | 7 | 8 | @pytest.mark.parametrize("run_type", INTERVALS) 9 | def test_add(run_type): 10 | status = PeriodicStatus() 11 | copy = PeriodicCopy("foo", run_type, "bar") 12 | copy._source_type = "type-foo" 13 | status.add(copy) 14 | assert getattr(status, run_type) == {copy.key: copy} 15 | -------------------------------------------------------------------------------- /tests/unit/status/periodic_status/test_eq.py: -------------------------------------------------------------------------------- 1 | from twindb_backup.status.periodic_status import PeriodicStatus 2 | 3 | 4 | def test_eq_empty(): 5 | status1 = PeriodicStatus() 6 | status2 = PeriodicStatus() 7 | assert status1 == status2 8 | -------------------------------------------------------------------------------- /tests/unit/status/periodic_status/test_remove.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup import INTERVALS 4 | from twindb_backup.copy.periodic_copy import PeriodicCopy 5 | from twindb_backup.status.exceptions import StatusKeyNotFound 6 | from twindb_backup.status.periodic_status import PeriodicStatus 7 | 8 | 9 | @pytest.mark.parametrize("run_type", INTERVALS) 10 | def test_remove(run_type): 11 | status = PeriodicStatus() 12 | copy = PeriodicCopy("foo", run_type, "bar") 13 | copy._source_type = "type-foo" 14 | status.add(copy) 15 | assert getattr(status, run_type) == {copy.key: copy} 16 | status.remove(copy.key) 17 | assert getattr(status, run_type) == {} 18 | 19 | 20 | def test_remove_raises(): 21 | status = PeriodicStatus() 22 | with pytest.raises(StatusKeyNotFound): 23 | status.remove("foo") 24 | -------------------------------------------------------------------------------- /tests/unit/test_file_source.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_twindb_backup 6 | ---------------------------------- 7 | 8 | Tests for `twindb_backup` module. 9 | """ 10 | import pytest 11 | 12 | from twindb_backup.source.file_source import FileSource 13 | 14 | 15 | @pytest.mark.parametrize( 16 | "path,name", 17 | [("/etc/my.cnf", "_etc_my.cnf"), ("/var/lib/mysql/", "_var_lib_mysql")], 18 | ) 19 | def test_make_file_name_from_full(path, name): 20 | src = FileSource(path, "foo") 21 | assert src._sanitize_filename() == name 22 | -------------------------------------------------------------------------------- /tests/unit/test_share.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.share import share 4 | 5 | 6 | @mock.patch("twindb_backup.share.print") 7 | # @mock.patch('twindb_backup.share.get_destination') 8 | def test_share_backup_cli(mock_print): 9 | mock_config = mock.Mock() 10 | mock_config.get.return_value = "/foo/bar" 11 | mock_dst = mock.Mock() 12 | mock_dst.remote_path = "/foo/bar" 13 | mock_config.destination.return_value = mock_dst 14 | mock_dst.find_files.return_value = ["/foo/bar1", "/foo/bar"] 15 | 16 | share(mock_config, "/foo/bar") 17 | 18 | mock_print.assert_called_once() 19 | mock_dst.share.assert_called_once_with("/foo/bar") 20 | mock_config.destination.assert_called_once_with() 21 | -------------------------------------------------------------------------------- /tests/unit/test_twindb_backup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/test_twindb_backup/__init__.py -------------------------------------------------------------------------------- /tests/unit/test_twindb_backup/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.fixture 5 | def innobackupex_error_log(): 6 | return """ 7 | 161122 03:08:50 Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS... 8 | xtrabackup: The latest check point (for incremental): '19747438' 9 | xtrabackup: Stopping log copying thread. 10 | .161122 03:08:50 >> log scanned up to (19747446) 11 | 12 | 161122 03:08:50 Executing UNLOCK BINLOG 13 | 161122 03:08:50 Executing UNLOCK TABLES 14 | 161122 03:08:50 All tables unlocked 15 | 161122 03:08:50 Backup created in directory '/twindb_backup/.' 16 | MySQL binlog position: filename 'mysql-bin.000001', position '80960', GTID of the last change '2e8afc7a-af69-11e6-8aaf-080027f6b007:1-178' 17 | 161122 03:08:50 [00] Streaming backup-my.cnf 18 | 161122 03:08:50 [00] ...done 19 | 161122 03:08:50 [00] Streaming xtrabackup_info 20 | 161122 03:08:50 [00] ...done 21 | xtrabackup: Transaction log of lsn (19747438) to (19747446) was copied. 22 | 161122 03:08:50 completed OK! """ 23 | -------------------------------------------------------------------------------- /tests/unit/test_twindb_backup/test_delete_local_files.py: -------------------------------------------------------------------------------- 1 | import mock 2 | import pytest 3 | 4 | from twindb_backup import delete_local_files 5 | 6 | 7 | @pytest.mark.parametrize( 8 | "keep, calls", 9 | [ 10 | (1, [mock.call("aaa"), mock.call("bbb")]), 11 | (2, [mock.call("aaa")]), 12 | (3, []), 13 | (0, [mock.call("aaa"), mock.call("bbb"), mock.call("ccc")]), 14 | ], 15 | ) 16 | @mock.patch("twindb_backup.os") 17 | @mock.patch("twindb_backup.glob") 18 | def test_delete_local_files(mock_glob, mock_os, keep, calls): 19 | mock_glob.glob.return_value = ["aaa", "bbb", "ccc"] 20 | 21 | delete_local_files("/foo", keep) 22 | mock_os.unlink.assert_has_calls(calls) 23 | -------------------------------------------------------------------------------- /tests/unit/test_twindb_backup/test_get_timeout.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup import get_timeout 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "run_type, timeout", 8 | [ 9 | ("hourly", 3600 / 2), 10 | ("daily", 24 * 3600 / 2), 11 | ("weekly", 7 * 24 * 3600 / 2), 12 | ("monthly", 30 * 24 * 3600 / 2), 13 | ("yearly", 365 * 24 * 3600 / 2), 14 | ], 15 | ) 16 | def test_get_timeout(run_type, timeout): 17 | assert get_timeout(run_type) == timeout 18 | -------------------------------------------------------------------------------- /tests/unit/test_twindb_backup/test_run_backup_job.py: -------------------------------------------------------------------------------- 1 | import mock 2 | 3 | from twindb_backup.backup import run_backup_job 4 | from twindb_backup.configuration import TwinDBBackupConfig 5 | 6 | 7 | @mock.patch("twindb_backup.backup.backup_everything") 8 | @mock.patch("twindb_backup.backup.get_timeout") 9 | def test_run_backup_job_gets_lock(mock_get_timeout, mock_backup_everything, tmpdir): 10 | config_content = """ 11 | [source] 12 | backup_dirs=/etc /root /home 13 | 14 | [intervals] 15 | run_hourly=yes 16 | run_daily=yes 17 | run_weekly=yes 18 | run_monthly=yes 19 | run_yearly=yes 20 | """ 21 | lock_file = str(tmpdir.join("foo.lock")) 22 | config_file = tmpdir.join("foo.cfg") 23 | config_file.write(config_content) 24 | 25 | cfg = TwinDBBackupConfig(config_file=str(config_file)) 26 | 27 | mock_get_timeout.return_value = 1 28 | 29 | run_backup_job(cfg, "hourly", lock_file=lock_file) 30 | mock_backup_everything.assert_called_once_with("hourly", cfg, binlogs_only=False) 31 | -------------------------------------------------------------------------------- /tests/unit/test_twindb_backup/test_save_measures.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | import pytest 5 | 6 | from twindb_backup import save_measures 7 | 8 | 9 | @pytest.mark.parametrize( 10 | "data", 11 | [ 12 | ( 13 | """ 14 | {"measures": [{"duration": 100, "start": 0, "finish": 100}]} 15 | """ 16 | ), 17 | ( 18 | """ 19 | {"measures": [{"duration": 150, "start": 0, "finish": 150}]} 20 | """ 21 | ), 22 | ( 23 | """ 24 | {"measures": [{"duration": 444, "start": 444, "finish": 888}]} 25 | """ 26 | ), 27 | ], 28 | ) 29 | def test_save_measures_if_log_is_exist(data, tmpdir): 30 | log_file = tmpdir.join("log") 31 | log_file.write(data) 32 | save_measures(50, 100, str(log_file)) 33 | with open(str(log_file)) as data_file: 34 | data = json.load(data_file) 35 | last_data = data["measures"][-1] 36 | assert last_data["duration"] == (last_data["finish"] - last_data["start"]) 37 | 38 | 39 | def test_save_measures_if_log_is_does_not_exist(tmpdir): 40 | log_file = tmpdir.join("log") 41 | save_measures(50, 100, str(log_file)) 42 | assert os.path.isfile(str(log_file)) 43 | with open(str(log_file)) as data_file: 44 | data = json.load(data_file) 45 | assert len(data["measures"]) == 1 46 | last_data = data["measures"][-1] 47 | assert last_data["duration"] == (last_data["finish"] - last_data["start"]) 48 | -------------------------------------------------------------------------------- /tests/unit/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/util/__init__.py -------------------------------------------------------------------------------- /tests/unit/util/test_split_host_port.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from twindb_backup.util import split_host_port 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "pair, host, port", 8 | [ 9 | ("10.20.31.1:3306", "10.20.31.1", 3306), 10 | ("10.20.31.1", "10.20.31.1", None), 11 | ("10.20.31.1:", "10.20.31.1", None), 12 | (None, None, None), 13 | ("", None, None), 14 | ], 15 | ) 16 | def test_split_host_port(pair, host, port): 17 | assert split_host_port(pair) == (host, port) 18 | -------------------------------------------------------------------------------- /tests/unit/verify/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/tests/unit/verify/__init__.py -------------------------------------------------------------------------------- /tests/unit/verify/test_edit_backup_my_cnf.py: -------------------------------------------------------------------------------- 1 | from configparser import ConfigParser 2 | from os import path as osp 3 | from textwrap import dedent 4 | 5 | from twindb_backup.verify import edit_backup_my_cnf 6 | 7 | 8 | def test_edit_backup_my_cnf(tmpdir): 9 | datadir = tmpdir.mkdir("mysql") 10 | with open(osp.join(str(datadir), "backup-my.cnf"), "w") as fp: 11 | fp.write( 12 | dedent( 13 | """ 14 | # This MySQL options file was generated by innobackupex. 15 | 16 | # The MySQL server 17 | [mysqld] 18 | innodb_checksum_algorithm=crc32 19 | innodb_log_checksums=1 20 | innodb_data_file_path=ibdata1:12M:autoextend 21 | innodb_log_files_in_group=2 22 | innodb_log_file_size=50331648 23 | innodb_page_size=16384 24 | innodb_undo_directory=./ 25 | innodb_undo_tablespaces=2 26 | server_id=0 27 | innodb_log_checksums=ON 28 | innodb_redo_log_encrypt=OFF 29 | innodb_undo_log_encrypt=OFF 30 | server_uuid=00eb2d82-dc60-11ec-86a2-0242ac100301 31 | master_key_id=0 32 | """ 33 | ) 34 | ) 35 | edit_backup_my_cnf(str(datadir)) 36 | cfg = ConfigParser() 37 | cfg.read(osp.join(str(datadir), "backup-my.cnf")) 38 | assert cfg.getboolean("mysqld", "innodb_log_checksums") is True 39 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, lint, cov 3 | 4 | [testenv:lint] 5 | basepython=python 6 | deps=-rrequirements_dev.txt 7 | commands=pylint twindb_backup 8 | 9 | [testenv] 10 | passenv = CI TRAVIS TRAVIS_* 11 | deps=-rrequirements_dev.txt 12 | commands = 13 | pip install -U pip 14 | pytest --cov=./twindb_backup tests/unit 15 | codecov 16 | 17 | [testenv:cov] 18 | deps=-rrequirements_dev.txt 19 | commands= 20 | coverage run -m py.test tests/unit 21 | coverage report -------------------------------------------------------------------------------- /twindb_backup/cache/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/cache/__init__.py -------------------------------------------------------------------------------- /twindb_backup/cache/exceptions.py: -------------------------------------------------------------------------------- 1 | """TwinDB Backup cache exceptions.""" 2 | 3 | from twindb_backup.exceptions import TwinDBBackupError 4 | 5 | 6 | class CacheException(TwinDBBackupError): 7 | """Cache errors""" 8 | -------------------------------------------------------------------------------- /twindb_backup/configuration/destinations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/configuration/destinations/__init__.py -------------------------------------------------------------------------------- /twindb_backup/configuration/destinations/az.py: -------------------------------------------------------------------------------- 1 | """Azure Blob Storage destination configuration""" 2 | 3 | 4 | class AZConfig: 5 | """Azure Blob Storage Configuration.""" 6 | 7 | def __init__( 8 | self, connection_string: str, container_name: str, chunk_size: int = 1024 * 1024 * 4, remote_path: str = "/" 9 | ): 10 | self._connection_string = connection_string 11 | self._container_name = container_name 12 | self._chunk_size = chunk_size 13 | self._remote_path = remote_path 14 | self.validate_config() 15 | 16 | def validate_config(self): 17 | """Validate configuration.""" 18 | if not isinstance(self._connection_string, str): 19 | raise ValueError("CONNECTION_STRING must be a string") 20 | if not isinstance(self._container_name, str): 21 | raise ValueError("CONTAINER_NAME must be a string") 22 | if not isinstance(self._chunk_size, int): 23 | raise ValueError("CHUNK_SIZE must be an integer") 24 | if not isinstance(self._remote_path, str): 25 | raise ValueError("REMOTE_PATH must be a string") 26 | 27 | @property 28 | def connection_string(self) -> str: 29 | """CONNECTION_STRING""" 30 | return self._connection_string 31 | 32 | @property 33 | def container_name(self) -> str: 34 | """CONTAINER_NAME""" 35 | return self._container_name 36 | 37 | @property 38 | def chunk_size(self) -> int: 39 | """CHUNK_SIZE""" 40 | return self._chunk_size 41 | 42 | @property 43 | def remote_path(self) -> str: 44 | """REMOTE_PATH""" 45 | return self._remote_path 46 | -------------------------------------------------------------------------------- /twindb_backup/configuration/destinations/gcs.py: -------------------------------------------------------------------------------- 1 | """Google Cloud Storage destination configuration""" 2 | 3 | 4 | class GCSConfig: 5 | """Google Cloud Storage configuration.""" 6 | 7 | __attr__ = ["gc_credentials_file", "gc_encryption_key", "bucket"] 8 | 9 | def __init__(self, **kwargs): 10 | 11 | for opt in self.__attr__: 12 | setattr(self, f"_{opt}", kwargs.get(opt)) 13 | 14 | @property 15 | def gc_credentials_file(self): 16 | """GC_CREDENTIALS_FILE""" 17 | return getattr(self, "_gc_credentials_file") 18 | 19 | @property 20 | def gc_encryption_key(self): 21 | """GC_ENCRYPTION_KEY""" 22 | return getattr(self, "_gc_encryption_key") 23 | 24 | @property 25 | def bucket(self): 26 | """GCS bucket""" 27 | return getattr(self, "_bucket") 28 | -------------------------------------------------------------------------------- /twindb_backup/configuration/destinations/s3.py: -------------------------------------------------------------------------------- 1 | """Amazon S3 destrination configuration""" 2 | 3 | 4 | class S3Config: 5 | """Amazon S3 configuration.""" 6 | 7 | def __init__( 8 | self, 9 | aws_access_key_id, 10 | aws_secret_access_key, 11 | bucket, 12 | aws_default_region="us-east-1", 13 | ): 14 | 15 | self._aws_access_key_id = aws_access_key_id 16 | self._aws_secret_access_key = aws_secret_access_key 17 | self._bucket = bucket 18 | self._aws_default_region = aws_default_region 19 | 20 | @property 21 | def aws_access_key_id(self): 22 | """AWS_ACCESS_KEY_ID""" 23 | return self._aws_access_key_id 24 | 25 | @property 26 | def aws_secret_access_key(self): 27 | """AWS_SECRET_ACCESS_KEY""" 28 | return self._aws_secret_access_key 29 | 30 | @property 31 | def bucket(self): 32 | """S3 bucket""" 33 | return self._bucket 34 | 35 | @property 36 | def aws_default_region(self): 37 | """AWS_DEFAULT_REGION""" 38 | return self._aws_default_region 39 | -------------------------------------------------------------------------------- /twindb_backup/configuration/destinations/ssh.py: -------------------------------------------------------------------------------- 1 | """SSH destination configuration""" 2 | 3 | 4 | class SSHConfig: 5 | """SSH destination configuration.""" 6 | 7 | def __init__( 8 | self, 9 | backup_host="127.0.0.1", 10 | backup_dir="/var/backup", 11 | ssh_user="root", 12 | port=22, 13 | ssh_key="/root/.ssh/id_rsa", 14 | ): 15 | 16 | self._host = backup_host 17 | self._path = backup_dir 18 | self._user = ssh_user 19 | self._port = int(port) 20 | self._key = ssh_key 21 | 22 | @property 23 | def user(self): 24 | """SSH user""" 25 | return self._user 26 | 27 | @property 28 | def key(self): 29 | """Path to private SSH key""" 30 | return self._key 31 | 32 | @property 33 | def host(self): 34 | """Hostname or IP address of the SSH destination""" 35 | return self._host 36 | 37 | @property 38 | def port(self): 39 | """SSH port""" 40 | return self._port 41 | 42 | @property 43 | def path(self): 44 | """Remote path to root directory with backups""" 45 | return self._path 46 | -------------------------------------------------------------------------------- /twindb_backup/configuration/exceptions.py: -------------------------------------------------------------------------------- 1 | """TwinDB Backup configuration exceptions.""" 2 | 3 | from twindb_backup.exceptions import TwinDBBackupError 4 | 5 | 6 | class ConfigurationError(TwinDBBackupError): 7 | """Base configuration error""" 8 | -------------------------------------------------------------------------------- /twindb_backup/configuration/gpg.py: -------------------------------------------------------------------------------- 1 | """GPG configuration""" 2 | 3 | 4 | class GPGConfig: 5 | """ 6 | GPG configuration 7 | """ 8 | 9 | def __init__(self, **kwargs): 10 | 11 | for arg in ["recipient", "keyring", "secret_keyring"]: 12 | setattr(self, f"_{arg}", kwargs.get(arg, None)) 13 | 14 | @property 15 | def recipient(self): 16 | """E-mail address of the message recipient.""" 17 | 18 | return getattr(self, "_recipient") 19 | 20 | @property 21 | def keyring(self): 22 | """Path to keyring.""" 23 | 24 | return getattr(self, "_keyring") 25 | 26 | @property 27 | def secret_keyring(self): 28 | """Path to secret keyring.""" 29 | 30 | return getattr(self, "_secret_keyring") 31 | -------------------------------------------------------------------------------- /twindb_backup/configuration/retention.py: -------------------------------------------------------------------------------- 1 | """Retention policy configuration""" 2 | 3 | from collections import namedtuple 4 | 5 | RetentionPolicy = namedtuple("RetentionPolicy", ["hourly", "daily", "weekly", "monthly", "yearly"]) 6 | 7 | RetentionPolicy.__new__.__defaults__ = (24, 7, 4, 12, 3) 8 | -------------------------------------------------------------------------------- /twindb_backup/configuration/run_intervals.py: -------------------------------------------------------------------------------- 1 | """Run policy configuration""" 2 | 3 | from collections import namedtuple 4 | 5 | from twindb_backup import INTERVALS 6 | 7 | RunIntervals = namedtuple("RunIntervals", INTERVALS) 8 | 9 | RunIntervals.__new__.__defaults__ = (True,) * len(INTERVALS) 10 | -------------------------------------------------------------------------------- /twindb_backup/copy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/copy/__init__.py -------------------------------------------------------------------------------- /twindb_backup/copy/exceptions.py: -------------------------------------------------------------------------------- 1 | """Module for Backup copy exception classes.""" 2 | 3 | from twindb_backup.exceptions import TwinDBBackupError 4 | 5 | 6 | class BackupCopyError(TwinDBBackupError): 7 | """General backup copy error""" 8 | 9 | 10 | class UnknownSourceType(BackupCopyError): 11 | """Raises when source type is not set""" 12 | 13 | 14 | class WrongInputData(BackupCopyError): 15 | """Raises when incorrent inputs are used""" 16 | -------------------------------------------------------------------------------- /twindb_backup/copy/file_copy.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module describes class to work with backup copies of the file type. 3 | """ 4 | 5 | from twindb_backup.copy.periodic_copy import PeriodicCopy 6 | 7 | 8 | class FileCopy(PeriodicCopy): 9 | """ 10 | Backup copy of a file or directory. 11 | 12 | :param host: hostname where the backup was taken from. 13 | :type host: str 14 | :param name: backup copy basename 15 | :type name: str 16 | :param run_type: run type. daily, hourly, etc. 17 | :type run_type: str 18 | """ 19 | 20 | def __init__(self, *args, **kwargs): 21 | super(FileCopy, self).__init__(*args, **kwargs) 22 | self._source_type = "files" 23 | -------------------------------------------------------------------------------- /twindb_backup/destination/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/destination/__init__.py -------------------------------------------------------------------------------- /twindb_backup/destination/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for destination exceptions. 3 | """ 4 | 5 | from twindb_backup.exceptions import TwinDBBackupError 6 | 7 | 8 | class DestinationError(TwinDBBackupError): 9 | """General destination error""" 10 | 11 | pass 12 | 13 | 14 | class FileNotFound(DestinationError): 15 | """File doesn't exist on destination""" 16 | 17 | pass 18 | 19 | 20 | class S3DestinationError(DestinationError): 21 | """S3 destination errors""" 22 | 23 | pass 24 | 25 | 26 | class GCSDestinationError(DestinationError): 27 | """GCS destination errors""" 28 | 29 | pass 30 | 31 | 32 | class SshDestinationError(DestinationError): 33 | """SSH destination errors""" 34 | 35 | pass 36 | 37 | 38 | class AzureBlobDestinationError(DestinationError): 39 | """Azure-blob destination errors""" 40 | 41 | pass 42 | -------------------------------------------------------------------------------- /twindb_backup/exceptions.py: -------------------------------------------------------------------------------- 1 | """Module that describes exceptions of twindb_backup module.""" 2 | 3 | 4 | class TwinDBBackupError(Exception): 5 | """Catch-all exceptions""" 6 | 7 | 8 | class OperationError(TwinDBBackupError): 9 | """Exceptions that prevent normal TwinDB Backup operation""" 10 | 11 | 12 | class LockWaitTimeoutError(TwinDBBackupError): 13 | """Timeout expired while waiting for a lock""" 14 | 15 | 16 | class TwinDBBackupInternalError(TwinDBBackupError): 17 | """Internal errors in the tool itself""" 18 | -------------------------------------------------------------------------------- /twindb_backup/export.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module to process export 4 | """ 5 | 6 | 7 | def export_info(cfg, data, category, measure_type): 8 | """ 9 | Export data to service 10 | 11 | :param cfg: Config file 12 | :type cfg: TwinDBBackupConfig 13 | :param data: Data 14 | :param category: Category of data 15 | :param measure_type: Type of measure 16 | :param category: Category 17 | """ 18 | 19 | transport = cfg.exporter 20 | 21 | if transport: 22 | transport.export(category=category, measure_type=measure_type, data=data) 23 | -------------------------------------------------------------------------------- /twindb_backup/exporter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/exporter/__init__.py -------------------------------------------------------------------------------- /twindb_backup/exporter/base_exporter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines base exporter class. 4 | """ 5 | from abc import abstractmethod 6 | 7 | 8 | class ExportCategory(object): # pylint: disable=too-few-public-methods 9 | """Category of export data: files or mysql""" 10 | 11 | files = 0 12 | mysql = 1 13 | 14 | 15 | class ExportMeasureType(object): # pylint: disable=too-few-public-methods 16 | """Type of measure time: backup or restore""" 17 | 18 | backup = 0 19 | restore = 1 20 | 21 | 22 | class BaseExporter(object): # pylint: disable=too-few-public-methods 23 | """ 24 | Base exporter class 25 | """ 26 | 27 | def __init__(self): 28 | pass 29 | 30 | @abstractmethod 31 | def export(self, category, measure_type, data): 32 | """ 33 | Send data to server 34 | """ 35 | -------------------------------------------------------------------------------- /twindb_backup/exporter/datadog_exporter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Module defines DataDog exporter class. 5 | """ 6 | from datadog import initialize, statsd 7 | 8 | from twindb_backup.exporter.base_exporter import BaseExporter, ExportCategory, ExportMeasureType 9 | from twindb_backup.exporter.exceptions import DataDogExporterError 10 | 11 | 12 | class DataDogExporter(BaseExporter): # pylint: disable=too-few-public-methods 13 | """ 14 | DataDog exporter class 15 | """ 16 | 17 | def __init__(self, app_key, api_key): 18 | super(DataDogExporter, self).__init__() 19 | options = {"api_key": api_key, "app_key": app_key} 20 | initialize(**options) 21 | self._suffix = "twindb." 22 | 23 | def export(self, category, measure_type, data): 24 | """ 25 | Export data to DataDog 26 | :param category: Data meant 27 | :param measure_type: Type of measure 28 | :param data: Data to posting 29 | :raise: DataDogExporterError if data is invalid 30 | """ 31 | if isinstance(data, (int, float)): 32 | metric_name = self._suffix 33 | if category == ExportCategory.files: 34 | metric_name += "files." 35 | else: 36 | metric_name += "mysql." 37 | if measure_type == ExportMeasureType.backup: 38 | metric_name += "backup_time" 39 | else: 40 | metric_name += "restore_time" 41 | statsd.gauge(metric_name, data) 42 | else: 43 | raise DataDogExporterError("Invalid input data") 44 | -------------------------------------------------------------------------------- /twindb_backup/exporter/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for exporters exceptions. 3 | """ 4 | 5 | from twindb_backup.exceptions import TwinDBBackupError 6 | 7 | 8 | class BaseExporterError(TwinDBBackupError): 9 | """General exporters error""" 10 | 11 | pass 12 | 13 | 14 | class DataDogExporterError(BaseExporterError): 15 | """DataDog exporters error""" 16 | 17 | pass 18 | 19 | 20 | class StatsdExporterError(BaseExporterError): 21 | """Statsd exporters error""" 22 | 23 | pass 24 | -------------------------------------------------------------------------------- /twindb_backup/exporter/statsd_exporter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Module defines Statsd exporter class. 5 | """ 6 | import statsd 7 | 8 | from twindb_backup.exporter.base_exporter import BaseExporter, ExportCategory, ExportMeasureType 9 | from twindb_backup.exporter.exceptions import StatsdExporterError 10 | 11 | 12 | class StatsdExporter(BaseExporter): # pylint: disable=too-few-public-methods 13 | """ 14 | Statsd exporter class 15 | """ 16 | 17 | def __init__(self, statsd_host, statsd_port): 18 | super(StatsdExporter, self).__init__() 19 | self._client = statsd.StatsClient(statsd_host, statsd_port) 20 | self._suffix = "twindb." 21 | 22 | def export(self, category, measure_type, data): 23 | """ 24 | Export data to StatsD server 25 | :param category: Data meant 26 | :param measure_type: Type of measure 27 | :param data: Data to posting 28 | :raise: StatsdExporterError if data is invalid 29 | """ 30 | if isinstance(data, (int, float)): 31 | metric_name = self._suffix 32 | if category == ExportCategory.files: 33 | metric_name += "files." 34 | else: 35 | metric_name += "mysql." 36 | if measure_type == ExportMeasureType.backup: 37 | metric_name += "backup_time" 38 | else: 39 | metric_name += "restore_time" 40 | self._client.timing(metric_name, data) 41 | else: 42 | raise StatsdExporterError("Invalid input data") 43 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Modifiers module. 3 | 4 | Modifier take a stream as input, do something with it (compress, encrypt, etc) 5 | and return the modified stream for a next modifier or backup destination. 6 | 7 | Modifiers also do reverse operation - i.e. decompress, decrypt. 8 | """ 9 | 10 | from twindb_backup.modifiers.bzip2 import Bzip2 11 | from twindb_backup.modifiers.gzip import Gzip 12 | from twindb_backup.modifiers.lbzip2 import Lbzip2 13 | from twindb_backup.modifiers.pigz import Pigz 14 | 15 | COMPRESSION_MODIFIERS = { 16 | "gzip": {"class": Gzip, "kwargs": ["level"]}, 17 | "bzip2": {"class": Bzip2, "kwargs": ["level"]}, 18 | "lbzip2": {"class": Lbzip2, "kwargs": ["threads", "level"]}, 19 | "pigz": {"class": Pigz, "kwargs": ["threads", "level"]}, 20 | } 21 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/bzip2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines modifier that compresses a stream with bzip2 4 | """ 5 | from twindb_backup.modifiers.base import Modifier 6 | 7 | 8 | class Bzip2(Modifier): 9 | """ 10 | Modifier that compresses the input_stream with bzip2. 11 | """ 12 | 13 | suffix = ".bz" 14 | 15 | def __init__(self, input_stream, level=9): 16 | """ 17 | Modifier that uses bzip2 compression 18 | 19 | :param input_stream: Input stream. Must be file object 20 | :param level: compression level from 1 to 9 (fastest to best) 21 | :type level: int 22 | """ 23 | super(Bzip2, self).__init__(input_stream) 24 | 25 | self._level = level 26 | 27 | @property 28 | def _modifier_cmd(self): 29 | """get compression program cmd""" 30 | return ["bzip2", "-{0}".format(self._level), "-c", "-"] 31 | 32 | @property 33 | def _unmodifier_cmd(self): 34 | """get decompression program cmd""" 35 | return ["bunzip2", "-d", "-c"] 36 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/exceptions.py: -------------------------------------------------------------------------------- 1 | """Modifier exceptions.""" 2 | 3 | from twindb_backup.exceptions import TwinDBBackupError 4 | 5 | 6 | class ModifierException(TwinDBBackupError): 7 | """Base Exception for Modifier error""" 8 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/gzip.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines modifier that compresses a stream with gzip 4 | """ 5 | from twindb_backup.modifiers.base import Modifier 6 | 7 | 8 | class Gzip(Modifier): 9 | """ 10 | Modifier that compresses the input_stream with gzip. 11 | """ 12 | 13 | suffix = ".gz" 14 | 15 | def __init__(self, input_stream, level=9): 16 | """ 17 | Modifier that uses gzip compression 18 | 19 | :param input_stream: Input stream. Must be file object 20 | :param level: compression level from 1 to 9 (fastest to best) 21 | :type level: int 22 | """ 23 | super(Gzip, self).__init__(input_stream) 24 | 25 | self._level = level 26 | 27 | @property 28 | def _modifier_cmd(self): 29 | """get compression program cmd""" 30 | return ["gzip", "-{0}".format(self._level), "-c", "-"] 31 | 32 | @property 33 | def _unmodifier_cmd(self): 34 | """get decompression program cmd""" 35 | return ["gunzip", "-c"] 36 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/keeplocal.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines modifier that save a stream on the local file system 4 | """ 5 | import os 6 | 7 | from twindb_backup.destination.local import Local 8 | from twindb_backup.modifiers.base import Modifier, ModifierException 9 | from twindb_backup.status.mysql_status import MySQLStatus 10 | from twindb_backup.util import mkdir_p 11 | 12 | 13 | class KeepLocal(Modifier): 14 | """KeepLocal() class saves a copy of the stream on the local file system. 15 | It doesn't alter the stream.""" 16 | 17 | def __init__(self, input_stream, local_path): 18 | """ 19 | Modifier that saves a local copy of the stream in local_path file. 20 | 21 | :param input_stream: Input stream. Must be file object 22 | :param local_path: path to local file 23 | """ 24 | super(KeepLocal, self).__init__(input_stream) 25 | self.local_path = local_path 26 | local_dir = os.path.dirname(self.local_path) 27 | try: 28 | mkdir_p(local_dir) 29 | except OSError as err: 30 | raise ModifierException("Failed to create directory %s: %s" % (local_dir, err)) 31 | 32 | def callback(self, **kwargs): 33 | local_dst = Local(kwargs["keep_local_path"]) 34 | status = MySQLStatus(dst=kwargs["dst"]) 35 | status.save(local_dst) 36 | 37 | @property 38 | def _modifier_cmd(self): 39 | """get compression program cmd""" 40 | return ["tee", self.local_path] 41 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/lbzip2.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines modifier that compresses a stream with lbzip2 4 | """ 5 | from psutil import cpu_count 6 | 7 | from twindb_backup.modifiers.parallel_compressor import ParallelCompressor 8 | 9 | DEFAULT_THREADS = cpu_count() - 1 10 | 11 | 12 | class Lbzip2(ParallelCompressor): 13 | """ 14 | Modifier that compresses the input_stream with lbzip2. 15 | """ 16 | 17 | def __init__(self, input_stream, threads=DEFAULT_THREADS, level=9): 18 | """ 19 | Modifier that uses lbzip2 compression 20 | 21 | :param input_stream: Input stream. Must be file object 22 | :param threads: number of threads to use (defaults to total-1) 23 | :type threads: int 24 | :param level: compression level from 1 to 9 (fastest to best) 25 | :type level: int 26 | """ 27 | super(Lbzip2, self).__init__( 28 | input_stream, 29 | program="lbzip2", 30 | threads=threads, 31 | level=level, 32 | suffix=".bz", 33 | ) 34 | 35 | @property 36 | def _modifier_cmd(self): 37 | """get compression program cmd""" 38 | return [ 39 | self._program, 40 | "-{0}".format(self._level), 41 | "-n", 42 | str(self._threads), 43 | "-c", 44 | "-", 45 | ] 46 | 47 | @property 48 | def _unmodifier_cmd(self): 49 | """get decompression program cmd""" 50 | return [self._program, "-n", str(self._threads), "-d", "-c"] 51 | -------------------------------------------------------------------------------- /twindb_backup/modifiers/pigz.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module defines modifier that compresses a stream with pigz 4 | """ 5 | from psutil import cpu_count 6 | 7 | from twindb_backup.modifiers.parallel_compressor import ParallelCompressor 8 | 9 | DEFAULT_THREADS = cpu_count() - 1 10 | 11 | 12 | class Pigz(ParallelCompressor): 13 | """ 14 | Modifier that compresses the input_stream with pigz. 15 | """ 16 | 17 | def __init__(self, input_stream, threads=DEFAULT_THREADS, level=9): 18 | """ 19 | Modifier that uses pigz compression 20 | 21 | :param input_stream: Input stream. Must be file object 22 | :param threads: number of threads to use (defaults to total-1) 23 | :type threads: int 24 | :param level: compression level from 1 to 9 (fastest to best) 25 | :type level: int 26 | """ 27 | super(Pigz, self).__init__( 28 | input_stream, 29 | program="pigz", 30 | threads=threads, 31 | level=level, 32 | suffix=".gz", 33 | ) 34 | -------------------------------------------------------------------------------- /twindb_backup/share.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Module that works with sharing backups 4 | """ 5 | from __future__ import print_function 6 | 7 | from twindb_backup.exceptions import TwinDBBackupInternalError 8 | 9 | 10 | def share(twindb_config, s3_url): 11 | """ 12 | Function for generate make public file and get public url 13 | 14 | :param twindb_config: tool configuration 15 | :type twindb_config: TwinDBBackupConfig 16 | :param s3_url: S3 url to file 17 | :type s3_url: str 18 | :raise: TwinDBBackupError 19 | """ 20 | try: 21 | print(twindb_config.destination().share(s3_url)) 22 | except NotImplementedError as err: 23 | raise TwinDBBackupInternalError(err) from err 24 | -------------------------------------------------------------------------------- /twindb_backup/source/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/source/__init__.py -------------------------------------------------------------------------------- /twindb_backup/source/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for backup source exceptions. 3 | """ 4 | 5 | from twindb_backup.exceptions import TwinDBBackupError 6 | 7 | 8 | class SourceError(TwinDBBackupError): 9 | """General source error""" 10 | 11 | def __str__(self): 12 | return "%s: %s" % (self.__class__, self.message) 13 | 14 | 15 | class MySQLSourceError(SourceError): 16 | """Exceptions in MySQL source""" 17 | 18 | 19 | class RemoteMySQLSourceError(MySQLSourceError): 20 | """Exceptions in remote MySQL source""" 21 | 22 | 23 | class BinlogSourceError(SourceError): 24 | """Exceptions in Binlog source""" 25 | -------------------------------------------------------------------------------- /twindb_backup/source/mariadb_source.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module defines MySQL source class for backing up local MariaDB server. 3 | """ 4 | 5 | from twindb_backup import MARIABACKUP_BINARY 6 | from twindb_backup.source.mysql_source import MySQLSource 7 | 8 | 9 | class MariaDBSource(MySQLSource): 10 | def __init__(self, mysql_connect_info, run_type, backup_type, **kwargs): 11 | super().__init__(mysql_connect_info, run_type, backup_type, **kwargs) 12 | self._xtrabackup = kwargs.get("xtrabackup_binary") or MARIABACKUP_BINARY 13 | -------------------------------------------------------------------------------- /twindb_backup/source/remote_mariadb_source.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module defines MySQL source class for backing up local MariaDB server. 3 | """ 4 | 5 | from twindb_backup import MARIABACKUP_BINARY 6 | from twindb_backup.source.remote_mysql_source import RemoteMySQLSource 7 | 8 | 9 | class RemoteMariaDBSource(RemoteMySQLSource): 10 | def get_stream(self): 11 | raise NotImplementedError("Method get_stream not implemented") 12 | 13 | def __init__(self, kwargs): 14 | super().__init__(kwargs) 15 | self._xtrabackup = kwargs.get("xtrabackup_binary") or MARIABACKUP_BINARY 16 | -------------------------------------------------------------------------------- /twindb_backup/ssh/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/ssh/__init__.py -------------------------------------------------------------------------------- /twindb_backup/ssh/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | SSH Client Exceptions. 3 | """ 4 | 5 | from twindb_backup.exceptions import TwinDBBackupError 6 | 7 | 8 | class SshClientException(TwinDBBackupError): 9 | """Exception in SshClient""" 10 | 11 | pass 12 | -------------------------------------------------------------------------------- /twindb_backup/status/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twindb/backup/0761d050955dfc594131491d5e5f3e4245c406ab/twindb_backup/status/__init__.py -------------------------------------------------------------------------------- /twindb_backup/status/exceptions.py: -------------------------------------------------------------------------------- 1 | """Status exceptions.""" 2 | 3 | from twindb_backup.exceptions import TwinDBBackupError 4 | 5 | 6 | class StatusError(TwinDBBackupError): 7 | """General status error""" 8 | 9 | 10 | class CorruptedStatus(StatusError): 11 | """Status file is corrupt""" 12 | 13 | 14 | class StatusKeyNotFound(StatusError): 15 | """Accessing a key that doesn't exist""" 16 | -------------------------------------------------------------------------------- /vagrant/README.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | Using Vagrant for TwinDB backup development 3 | =========================================== 4 | 5 | Versions 6 | -------- 7 | 8 | We use vagrant with VirtualBox_. Some version combinations do not work as expected. If you experience try exactly 9 | same versions we used for development. 10 | 11 | Vagrant 12 | ~~~~~~~ 13 | :: 14 | 15 | $ vagrant --version 16 | Vagrant 1.9.1 17 | 18 | Virtualbox 19 | ~~~~~~~~~~ 20 | 21 | :: 22 | 23 | $ VBoxManage --version 24 | 5.1.14r112924 25 | 26 | 27 | 28 | Vagrant boxes 29 | ~~~~~~~~~~~~~ 30 | 31 | ====== ================ 32 | OS Version 33 | ====== ================ 34 | CentOS bento/centos-7.3 35 | ====== ================ 36 | 37 | 38 | Check Vagrantfile if this README is outdated 39 | 40 | 41 | Starting vagrant machines 42 | ------------------------- 43 | 44 | :: 45 | 46 | $ cd vagrant 47 | $ vagrant up 48 | 49 | 50 | A directory with the source code will be mounted on ``/twindb_backup`` 51 | 52 | A sample config file is installed in ``/etc/twindb/twindb-backup.cfg`` 53 | 54 | Editable python module can be installed with following command: 55 | 56 | :: 57 | 58 | # pip install -e /twindb_backup 59 | 60 | 61 | .. _VirtualBox: https://www.virtualbox.org/wiki/Downloads 62 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/manifests/site.pp: -------------------------------------------------------------------------------- 1 | node /^master1/ { 2 | include role::master 3 | } 4 | 5 | node /^master2/ { 6 | include role::base 7 | } 8 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/.my.cnf: -------------------------------------------------------------------------------- 1 | [client] 2 | socket=/var/lib/mysql/mysql.sock 3 | user=root 4 | password= 5 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/bashrc: -------------------------------------------------------------------------------- 1 | # .bashrc 2 | 3 | # Source global definitions 4 | if [ -f /etc/bashrc ]; then 5 | . /etc/bashrc 6 | fi 7 | 8 | # User specific aliases and functions 9 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/id_rsa.pub: -------------------------------------------------------------------------------- 1 | ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAyXxAjPShNGAedbaEtltFI6A7RlsyI+4evxTq6uQrgbJ6Hm+pHBXshXQYXDyVjvytaM+6GKF+r+6+C+6Wc5Xz4lLO/ZiSCdPbyEgqw1JoHrgPNpc6wmCtjJExxjzvpwSVgbZg3xOdqW1y+TyqeUkXEg/Lm4VZhN1Q/KyGCgBlWuAXoOYRGhaNWqcnr/Wn5YzVHAx2yJNrurtKLVYVMIkGcN/6OUaPpWqKZLaXiK/28PSZ5GdTDmxRg4W0pdyGEYQndpPlpLF4w5gNUEhVZM8hWVE29+DIW3XXVYGYchxmkhU7wrGxxZR+k5AT+7g8VspVS8zNMXM9Z27w55EQuluNMQ== root@testdriver.box -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/my-master-legacy.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | datadir=/var/lib/mysql 3 | socket=/var/lib/mysql/mysql.sock 4 | user=mysql 5 | # Disabling symbolic-links is recommended to prevent assorted security risks 6 | symbolic-links=0 7 | 8 | server_id=100 9 | log-bin=mysql-bin 10 | log-slave-updates 11 | 12 | [mysqld_safe] 13 | log-error=/var/log/mysqld.log 14 | pid-file=/var/run/mysqld/mysqld.pid 15 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/my-master.cnf: -------------------------------------------------------------------------------- 1 | [client] 2 | socket=/var/lib/mysql/mysql.sock 3 | 4 | [mysqld] 5 | datadir=/var/lib/mysql 6 | socket=/var/lib/mysql/mysql.sock 7 | user=mysql 8 | # Disabling symbolic-links is recommended to prevent assorted security risks 9 | symbolic-links=0 10 | 11 | server_id=100 12 | gtid_mode=ON 13 | log-bin=mysql-bin 14 | log-slave-updates 15 | enforce-gtid-consistency 16 | 17 | [mysqld_safe] 18 | log-error=/var/log/mysqld.log 19 | pid-file=/var/run/mysqld/mysqld.pid 20 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/mysql-init: -------------------------------------------------------------------------------- 1 | ALTER USER 'root'@'localhost' IDENTIFIED BY 'MyNewPass'; 2 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/mysql_grants.sh: -------------------------------------------------------------------------------- 1 | mysql < /home/vagrant/mysql_grants.sql 2 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/mysql_grants.sql: -------------------------------------------------------------------------------- 1 | CREATE USER 'dba'@'%' IDENTIFIED WITH mysql_native_password BY 'qwerty'; 2 | CREATE USER 'dba'@'localhost' IDENTIFIED WITH mysql_native_password BY 'qwerty'; 3 | CREATE USER 'repl'@'%' IDENTIFIED WITH mysql_native_password BY 'qwerty'; 4 | 5 | GRANT ALL ON *.* TO 'dba'@'%' WITH GRANT OPTION; 6 | GRANT ALL ON *.* TO 'dba'@'localhost' WITH GRANT OPTION; 7 | GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'; 8 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/sysconfig_docker: -------------------------------------------------------------------------------- 1 | # /etc/sysconfig/docker 2 | 3 | # Modify these options if you want to change the way the docker daemon runs 4 | OPTIONS='--selinux-enabled=false --log-driver=journald --signature-verification=false' 5 | if [ -z "${DOCKER_CERT_PATH}" ]; then 6 | DOCKER_CERT_PATH=/etc/docker 7 | fi 8 | 9 | # Do not add registries in this file anymore. Use /etc/containers/registries.conf 10 | # from the atomic-registries package. 11 | # 12 | 13 | # On an SELinux system, if you remove the --selinux-enabled option, you 14 | # also need to turn on the docker_transition_unconfined boolean. 15 | # setsebool -P docker_transition_unconfined 1 16 | 17 | # Location used for temporary files, such as those created by 18 | # docker load and build operations. Default is /var/lib/docker/tmp 19 | # Can be overriden by setting the following environment variable. 20 | # DOCKER_TMPDIR=/var/tmp 21 | 22 | # Controls the /etc/cron.daily/docker-logrotate cron job status. 23 | # To disable, uncomment the line below. 24 | # LOGROTATE=false 25 | 26 | # docker-latest daemon can be used by starting the docker-latest unitfile. 27 | # To use docker-latest client, uncomment below lines 28 | #DOCKERBINARY=/usr/bin/docker-latest 29 | #DOCKERDBINARY=/usr/bin/dockerd-latest 30 | #DOCKER_CONTAINERD_BINARY=/usr/bin/docker-containerd-latest 31 | #DOCKER_CONTAINERD_SHIM_BINARY=/usr/bin/docker-containerd-shim-latest 32 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg: -------------------------------------------------------------------------------- 1 | # What to backup 2 | [source] 3 | backup_dirs=/etc /root /home 4 | backup_mysql=yes 5 | 6 | # Destination 7 | [destination] 8 | # destination "s3", "gcs" or "ssh" 9 | backup_destination=gcs 10 | keep_local_path=/var/backup 11 | 12 | # Compression 13 | #[compression] 14 | # gzip, pigz, bzip2 or lbzip2 15 | #program=gzip 16 | 17 | 18 | # S3 destination settings 19 | [s3] 20 | AWS_ACCESS_KEY_ID="XXXXX" 21 | AWS_SECRET_ACCESS_KEY="YYYYY" 22 | AWS_DEFAULT_REGION="us-east-1" 23 | BUCKET="twindb-backups" 24 | 25 | # Azure destination settings 26 | [az] 27 | connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net" 28 | container_name="twindb-backups" 29 | #remote_path = /backups/mysql # optional 30 | 31 | # GCS destination settings 32 | [gcs] 33 | GC_CREDENTIALS_FILE=/twindb_backup/env/My Project 17339-bbbc43d1bee3.json 34 | # GC_ENCRYPTION_KEY="ZZZZZZ" 35 | BUCKET="twindb-backups" 36 | 37 | # SSH destination settings 38 | [ssh] 39 | backup_host=192.168.36.250 40 | ssh_user=root 41 | backup_dir=/path/to/twindb-server-backups 42 | 43 | # MySQL 44 | [mysql] 45 | mysql_defaults_file=/root/.my.cnf 46 | full_backup=daily 47 | 48 | # Retention 49 | [retention] 50 | hourly_copies=24 51 | daily_copies=7 52 | weekly_copies=4 53 | monthly_copies=12 54 | yearly_copies=3 55 | 56 | [retention_local] 57 | # Local retention policy 58 | hourly_copies=1 59 | daily_copies=1 60 | weekly_copies=0 61 | monthly_copies=0 62 | yearly_copies=0 63 | 64 | # Run intervals 65 | [intervals] 66 | run_hourly=yes 67 | run_daily=yes 68 | run_weekly=yes 69 | run_monthly=no 70 | run_yearly=yes 71 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/role/manifests/base.pp: -------------------------------------------------------------------------------- 1 | class role::base { 2 | include profile::base 3 | } 4 | -------------------------------------------------------------------------------- /vagrant/environment/puppet/modules/role/manifests/master.pp: -------------------------------------------------------------------------------- 1 | class role::master { 2 | include profile::master 3 | } 4 | --------------------------------------------------------------------------------