├── getPassword.h ├── getPassword.c ├── .bzrignore ├── cmake └── modules │ ├── Findcppcheck.cpp │ ├── FindGLIB2.cmake │ ├── FindPCRE.cmake │ ├── FindSphinx.cmake │ ├── FindMySQL.cmake │ ├── Findcppcheck.cmake │ └── CppcheckTargets.cmake ├── g_unix_signal.h ├── docs ├── authors.rst ├── index.rst ├── _build │ ├── sources.cmake.in │ └── conf.py.in ├── examples.rst ├── compiling.rst ├── files.rst ├── myloader_usage.rst ├── CMakeLists.txt └── mydumper_usage.rst ├── config.h.in ├── connection.h ├── server_detect.h ├── binlog.h ├── myloader.h ├── connection.c ├── server_detect.c ├── mydumper.h ├── CMakeLists.txt ├── README.md ├── common.h ├── g_unix_signal.c ├── binlog.c ├── myloader.c └── LICENSE /getPassword.h: -------------------------------------------------------------------------------- 1 | #ifndef GET_PASSWORD_PROMPT 2 | #define GET_PASSWORD_PROMPT 3 | char* passwordPrompt(void); 4 | #endif 5 | -------------------------------------------------------------------------------- /getPassword.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "getPassword.h" 4 | 5 | char* passwordPrompt(void) { 6 | char *password; 7 | password = getpass("Enter MySQL Password: "); 8 | 9 | return password; 10 | } 11 | -------------------------------------------------------------------------------- /.bzrignore: -------------------------------------------------------------------------------- 1 | mydumper 2 | myloader 3 | *~ 4 | *.dSYM 5 | export 6 | CMakeCache.txt 7 | CMakeFiles/ 8 | Makefile 9 | cmake_install.cmake 10 | config.h 11 | docs/CMakeFiles/ 12 | docs/Makefile 13 | docs/_doctrees/ 14 | docs/_sources/ 15 | docs/cmake_install.cmake 16 | docs/html/ 17 | docs/man/ 18 | docs/_build/conf.py 19 | docs/_build/sources.cmake 20 | .project 21 | -------------------------------------------------------------------------------- /cmake/modules/Findcppcheck.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * \file Findcppcheck.cpp 3 | * \brief Dummy C++ source file used by CMake module Findcppcheck.cmake 4 | * 5 | * \author 6 | * Ryan Pavlik, 2009-2010 7 | * 8 | * http://academic.cleardefinition.com/ 9 | * 10 | */ 11 | 12 | 13 | 14 | int main(int argc, char* argv[]) { 15 | return 0; 16 | } 17 | -------------------------------------------------------------------------------- /g_unix_signal.h: -------------------------------------------------------------------------------- 1 | #ifndef G_UNIX_SIGNAL_H 2 | #define G_UNIX_SIGNAL_H 3 | 4 | #include 5 | 6 | GSource *g_unix_signal_source_new(gint signum); 7 | guint g_unix_signal_add(gint signum, GSourceFunc function, gpointer data); 8 | guint g_unix_signal_add_full(gint priority, gint signum, GSourceFunc function, gpointer data, GDestroyNotify notify); 9 | 10 | #endif /* G_UNIX_SIGNAL_H */ 11 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | The code for mydumper has been written by the following people: 5 | 6 | * `Domas Mituzas `_, Facebook ( domas at fb dot com ) 7 | * `Andrew Hutchings `_, SkySQL ( andrew at skysql dot com ) 8 | * `Mark Leith `_, Oracle Corporation ( mark dot leith at oracle dot com ) 9 | * `Max Bubenick `_, Percona RDBA ( max dot bubenick at percona dot com ) 10 | -------------------------------------------------------------------------------- /config.h.in: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #cmakedefine VERSION "@VERSION@" 5 | #cmakedefine WITH_BINLOG 6 | #cmakedefine WITH_SSL 7 | 8 | #if defined(LIBMYSQL_VERSION) 9 | #define MYSQL_VERSION_STR LIBMYSQL_VERSION 10 | #elif defined(MARIADB_CLIENT_VERSION_STR) 11 | #define MYSQL_VERSION_STR MARIADB_CLIENT_VERSION_STR 12 | #elif defined(MYSQL_VERSION_NUMBER) 13 | #define MYSQL_VERSION_STR MYSQL_VERSION_NUMBER 14 | #else 15 | #define MYSQL_VERSION_STR MYSQL_SERVER_VERSION 16 | #endif 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. MySQL Data Dumper documentation master file 2 | You can adapt this file completely to your liking, but it should at least 3 | contain the root `toctree` directive. 4 | 5 | Welcome to MySQL Data Dumper's documentation! 6 | ============================================= 7 | 8 | Contents: 9 | 10 | .. toctree:: 11 | :maxdepth: 2 12 | 13 | authors 14 | compiling 15 | mydumper_usage 16 | myloader_usage 17 | files 18 | examples 19 | 20 | Indices and tables 21 | ================== 22 | 23 | * :ref:`genindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /docs/_build/sources.cmake.in: -------------------------------------------------------------------------------- 1 | # This script recursively copies all ReST documents from the source directory to 2 | # the binary directory. CMAKE_CURRENT_SOURCE_DIR and SOURCES_DIR are substituted 3 | # upon the cmake stage. The script is executed upon the make stage to ensure 4 | # that the binary sources directory is always up to date. 5 | 6 | file(GLOB SOURCES 7 | RELATIVE "@CMAKE_CURRENT_SOURCE_DIR@" 8 | "@CMAKE_CURRENT_SOURCE_DIR@/*.rst" 9 | ) 10 | foreach(source ${SOURCES}) 11 | configure_file( 12 | "@CMAKE_CURRENT_SOURCE_DIR@/${source}" 13 | "@SOURCES_DIR@/${source}" 14 | COPYONLY 15 | ) 16 | endforeach(source) 17 | -------------------------------------------------------------------------------- /cmake/modules/FindGLIB2.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find the GLIB2 libraries 2 | 3 | if(GLIB2_INCLUDE_DIR AND GLIB2_LIBRARIES AND GTHREAD2_LIBRARIES) 4 | # Already in cache, be silent 5 | set(GLIB2_FIND_QUIETLY TRUE) 6 | endif(GLIB2_INCLUDE_DIR AND GLIB2_LIBRARIES AND GTHREAD2_LIBRARIES) 7 | 8 | if (NOT WIN32) 9 | include(FindPkgConfig) 10 | pkg_search_module(PC_GLIB2 REQUIRED glib-2.0) 11 | pkg_search_module(PC_GTHREAD2 REQUIRED gthread-2.0) 12 | endif(NOT WIN32) 13 | 14 | set(GLIB2_INCLUDE_DIR ${PC_GLIB2_INCLUDE_DIRS}) 15 | 16 | find_library(GLIB2_LIBRARIES NAMES glib-2.0 HINTS ${PC_GLIB2_LIBDIR} ${PC_GLIB2_LIBRARY_DIRS}) 17 | 18 | find_library(GTHREAD2_LIBRARIES NAMES gthread-2.0 HINTS ${PC_GTHREAD2_LIBDIR} ${PC_GTHREAD2_LIBRARY_DIRS}) 19 | 20 | 21 | mark_as_advanced(GLIB2_INCLUDE_DIR GLIB2_LIBRARIES GTHREAD2_LIBRARIES) 22 | 23 | -------------------------------------------------------------------------------- /connection.h: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Aaron Brady, Shopify (insom) 16 | */ 17 | #ifndef _connection_h 18 | #define _connection_h 19 | #include 20 | 21 | void configure_connection(MYSQL *conn, const char* name); 22 | #endif 23 | -------------------------------------------------------------------------------- /server_detect.h: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Andrew Hutchings, SkySQL (andrew at skysql dot com) 16 | */ 17 | #ifndef _server_detect_h 18 | #define _server_detect_h 19 | 20 | #include 21 | 22 | #define DETECT_MYSQL_REGEX "^([3-9]\\.[0-9]+\\.[0-9]+)" 23 | #define DETECT_DRIZZLE_REGEX "^(20[0-9]{2}\\.(0[1-9]|1[012])\\.[0-9]+)" 24 | #define DETECT_MARIADB_REGEX "^([0-9]{1,2}\\.[0-9]+\\.[0-9]+)" 25 | 26 | enum server_type { SERVER_TYPE_UNKNOWN, SERVER_TYPE_MYSQL, SERVER_TYPE_DRIZZLE }; 27 | int detect_server(MYSQL *conn); 28 | #endif 29 | -------------------------------------------------------------------------------- /binlog.h: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Domas Mituzas, Facebook ( domas at fb dot com ) 16 | Mark Leith, Oracle Corporation (mark dot leith at oracle dot com) 17 | Andrew Hutchings, SkySQL (andrew at skysql dot com) 18 | 19 | */ 20 | 21 | #ifndef _binlog_h 22 | #define _binlog_h 23 | #include "mydumper.h" 24 | 25 | void get_binlogs(MYSQL *conn, struct configuration *conf); 26 | void get_binlog_file(MYSQL *conn, char *binlog_file, const char *binlog_directory, guint64 start_position, guint64 stop_position, gboolean continuous); 27 | unsigned int get_event(const char *buf, unsigned int len); 28 | void write_binlog(FILE* file, const char* data, guint64 len); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /myloader.h: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Domas Mituzas, Facebook ( domas at fb dot com ) 16 | Mark Leith, Oracle Corporation (mark dot leith at oracle dot com) 17 | Andrew Hutchings, SkySQL (andrew at skysql dot com) 18 | 19 | */ 20 | 21 | #ifndef _myloader_h 22 | #define _myloader_h 23 | 24 | enum job_type { JOB_SHUTDOWN, JOB_RESTORE }; 25 | 26 | struct configuration { 27 | GAsyncQueue* queue; 28 | GAsyncQueue* ready; 29 | GMutex* mutex; 30 | int done; 31 | }; 32 | 33 | struct thread_data { 34 | struct configuration *conf; 35 | guint thread_id; 36 | }; 37 | 38 | struct job { 39 | enum job_type type; 40 | void *job_data; 41 | struct configuration *conf; 42 | }; 43 | 44 | struct restore_job { 45 | char *database; 46 | char *table; 47 | char *filename; 48 | guint part; 49 | }; 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | Examples 2 | ======== 3 | 4 | Simple Usage 5 | ------------ 6 | Just running :program:`mydumper` without any options will try to connect to a 7 | server using the default socket path. It will then dump the tables from all 8 | databases using 4 worker threads. 9 | 10 | Regex 11 | ----- 12 | To use :program:`mydumper`'s regex feature simply use the 13 | :option:`--regex ` option. In the following example mydumper 14 | will ignore the ``test`` and ``mysql`` databases:: 15 | 16 | mydumper --regex '^(?!(mysql\.|test\.))' 17 | 18 | Restoring a dump 19 | ---------------- 20 | Mydumper now include myloader which is a multi-threaded restoration tool. To 21 | use myloader with a mydumper dump you simply need to pass it the directory of 22 | the dump along with a user capable of restoring the schemas and data. As an 23 | example the following will restore a dump overwriting any existing tables:: 24 | 25 | myloader --directory=export-20110614-094953 --overwrite-tables --user=root 26 | 27 | Daemon mode 28 | ----------- 29 | Mydumper has a daemon mode which will snapshot the dump data every so often 30 | whilst continuously retreiving the binary log files. This gives a continuous 31 | consistent backup right up to the point where the database server fails. To use 32 | this you simply need to use the :option:`--daemon ` option. 33 | 34 | In the following example mydumper will use daemon mode, creating a snapshot 35 | every half an hour and log to an output file:: 36 | 37 | mydumper --daemon --snapshot-interval=30 --logfile=dump.log 38 | -------------------------------------------------------------------------------- /connection.c: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Aaron Brady, Shopify (insom) 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include "connection.h" 22 | 23 | extern char *defaults_file; 24 | #ifdef WITH_SSL 25 | extern char *key; 26 | extern char *cert; 27 | extern char *ca; 28 | extern char *capath; 29 | extern char *cipher; 30 | extern gboolean ssl; 31 | #endif 32 | extern guint compress_protocol; 33 | 34 | void configure_connection(MYSQL *conn, const char *name) { 35 | if (defaults_file != NULL) { 36 | mysql_options(conn,MYSQL_READ_DEFAULT_FILE,defaults_file); 37 | } 38 | mysql_options(conn, MYSQL_READ_DEFAULT_GROUP, name); 39 | 40 | if (compress_protocol) 41 | mysql_options(conn, MYSQL_OPT_COMPRESS, NULL); 42 | 43 | #ifdef WITH_SSL 44 | unsigned int i; 45 | if (ssl) { 46 | i = SSL_MODE_REQUIRED; 47 | } else { 48 | i = SSL_MODE_DISABLED; 49 | } 50 | 51 | mysql_ssl_set(conn,key,cert,ca,capath,cipher); 52 | mysql_options(conn,MYSQL_OPT_SSL_MODE,&i); 53 | #endif 54 | } 55 | -------------------------------------------------------------------------------- /cmake/modules/FindPCRE.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find the PCRE regular expression library 2 | # Once done this will define 3 | # 4 | # PCRE_FOUND - system has the PCRE library 5 | # PCRE_INCLUDE_DIR - the PCRE include directory 6 | # PCRE_LIBRARIES - The libraries needed to use PCRE 7 | 8 | # Copyright (c) 2006, Alexander Neundorf, 9 | # 10 | # Redistribution and use is allowed according to the terms of the BSD license. 11 | # For details see the accompanying COPYING-CMAKE-SCRIPTS file. 12 | 13 | 14 | if (PCRE_INCLUDE_DIR AND PCRE_PCREPOSIX_LIBRARY AND PCRE_PCRE_LIBRARY) 15 | # Already in cache, be silent 16 | set(PCRE_FIND_QUIETLY TRUE) 17 | endif (PCRE_INCLUDE_DIR AND PCRE_PCREPOSIX_LIBRARY AND PCRE_PCRE_LIBRARY) 18 | 19 | 20 | if (NOT WIN32) 21 | # use pkg-config to get the directories and then use these values 22 | # in the FIND_PATH() and FIND_LIBRARY() calls 23 | find_package(PkgConfig) 24 | 25 | pkg_check_modules(PC_PCRE REQUIRED libpcre) 26 | 27 | set(PCRE_DEFINITIONS ${PC_PCRE_CFLAGS_OTHER}) 28 | 29 | endif (NOT WIN32) 30 | 31 | find_path(PCRE_INCLUDE_DIR pcre.h 32 | HINTS ${PC_PCRE_INCLUDEDIR} ${PC_PCRE_INCLUDE_DIRS} 33 | PATH_SUFFIXES pcre) 34 | 35 | find_library(PCRE_PCRE_LIBRARY NAMES pcre HINTS ${PC_PCRE_LIBDIR} ${PC_PCRE_LIBRARY_DIRS}) 36 | 37 | find_library(PCRE_PCREPOSIX_LIBRARY NAMES pcreposix HINTS ${PC_PCRE_LIBDIR} ${PC_PCRE_LIBRARY_DIRS}) 38 | 39 | include(FindPackageHandleStandardArgs) 40 | find_package_handle_standard_args(PCRE DEFAULT_MSG PCRE_INCLUDE_DIR PCRE_PCRE_LIBRARY PCRE_PCREPOSIX_LIBRARY ) 41 | 42 | set(PCRE_LIBRARIES ${PCRE_PCRE_LIBRARY} ${PCRE_PCREPOSIX_LIBRARY}) 43 | 44 | mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARIES PCRE_PCREPOSIX_LIBRARY PCRE_PCRE_LIBRARY) 45 | 46 | -------------------------------------------------------------------------------- /docs/compiling.rst: -------------------------------------------------------------------------------- 1 | Compiling 2 | ========= 3 | 4 | Requirements 5 | ------------ 6 | 7 | mydumper requires the following before it can be compiled: 8 | 9 | * `CMake `_ 10 | * `Glib2 `_ (with development packages) 11 | * `PCRE `_ (with development packages) 12 | * `MySQL `_ client libraries (with development packages) 13 | 14 | Additionally the following packages are optional: 15 | 16 | * `python-sphinx `_ (for documentation) 17 | 18 | Ubuntu/Debian 19 | ^^^^^^^^^^^^^ 20 | 21 | .. code-block:: bash 22 | 23 | apt-get install libglib2.0-dev libmysqlclient15-dev zlib1g-dev libpcre3-dev 24 | 25 | Fedora/Redhat/CentOS 26 | ^^^^^^^^^^^^^^^^^^^^ 27 | 28 | .. code-block:: bash 29 | 30 | yum install glib2-devel mysql-devel zlib-devel pcre-devel 31 | 32 | OpenSUSE 33 | ^^^^^^^^ 34 | 35 | .. code-block:: bash 36 | 37 | zypper install glib2-devel libmysqlclient-devel pcre-devel zlib-devel 38 | 39 | Mac OSX 40 | ^^^^^^^ 41 | 42 | .. code-block:: bash 43 | 44 | port install glib2 mysql5 pcre 45 | 46 | CMake 47 | ----- 48 | 49 | CMake is used for mydumper's build system and is executed as follows:: 50 | 51 | cmake . 52 | make 53 | 54 | You can optionally provide parameters for CMake, the possible options are: 55 | 56 | * ``-DMYSQL_CONFIG=/path/to/mysql_config`` - The path and filename for the mysql_config executable 57 | * ``-DCMAKE_INSTALL_PREFIX=/install/path`` - The path where mydumper should be installed 58 | 59 | Documentation 60 | ------------- 61 | 62 | If you wish to just compile the documentation you can do so with:: 63 | 64 | cmake . 65 | make doc_html 66 | 67 | or for a man page output:: 68 | 69 | cmake . 70 | make doc_man 71 | -------------------------------------------------------------------------------- /cmake/modules/FindSphinx.cmake: -------------------------------------------------------------------------------- 1 | # - This module looks for Sphinx 2 | # Find the Sphinx documentation generator 3 | # 4 | # This modules defines 5 | # SPHINX_EXECUTABLE 6 | # SPHINX_FOUND 7 | # SPHINX_MAJOR_VERSION 8 | # SPHINX_MINOR_VERSION 9 | # SPHINX_VERSION 10 | 11 | #============================================================================= 12 | # Copyright 2002-2009 Kitware, Inc. 13 | # Copyright 2009-2011 Peter Colberg 14 | # 15 | # Distributed under the OSI-approved BSD License (the "License"); 16 | # see accompanying file COPYING-CMAKE-SCRIPTS for details. 17 | # 18 | # This software is distributed WITHOUT ANY WARRANTY; without even the 19 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 20 | # See the License for more information. 21 | #============================================================================= 22 | # (To distribute this file outside of CMake, substitute the full 23 | # License text for the above reference.) 24 | 25 | find_program(SPHINX_EXECUTABLE NAMES sphinx-build 26 | HINTS 27 | $ENV{SPHINX_DIR} 28 | PATH_SUFFIXES bin 29 | DOC "Sphinx documentation generator" 30 | ) 31 | 32 | include(FindPackageHandleStandardArgs) 33 | 34 | find_package_handle_standard_args(Sphinx DEFAULT_MSG 35 | SPHINX_EXECUTABLE 36 | ) 37 | 38 | if (SPHINX_EXECUTABLE) 39 | execute_process ( 40 | COMMAND "${SPHINX_EXECUTABLE}" -h 41 | OUTPUT_VARIABLE _SPHINX_VERSION_OUTPUT 42 | ERROR_VARIABLE _SPHINX_VERSION_OUTPUT 43 | ) 44 | if (_SPHINX_VERSION_OUTPUT MATCHES "Sphinx v([0-9]+\\.[0-9]+\\.[0-9]+)") 45 | set (SPHINX_VERSION "${CMAKE_MATCH_1}") 46 | string (REPLACE "." ";" _SPHINX_VERSION_LIST "${SPHINX_VERSION}") 47 | list (GET _SPHINX_VERSION_LIST 0 SPHINX_MAJOR_VERSION) 48 | list (GET _SPHINX_VERSION_LIST 1 SPHINX_MINOR_VERSION) 49 | # patch version meh :) 50 | endif() 51 | endif() 52 | 53 | message("${SPHINX_MAJOR_VERSION}") 54 | 55 | mark_as_advanced( 56 | SPHINX_EXECUTABLE 57 | ) 58 | -------------------------------------------------------------------------------- /server_detect.c: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Andrew Hutchings, SkySQL (andrew at skysql dot com) 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | #include "server_detect.h" 22 | 23 | int detect_server(MYSQL *conn) { 24 | pcre *re= NULL; 25 | const char *error; 26 | int erroroffset; 27 | int ovector[9]= {0}; 28 | int rc; 29 | const char* db_version= mysql_get_server_info(conn); 30 | 31 | re= pcre_compile(DETECT_MYSQL_REGEX, 0, &error, &erroroffset, NULL); 32 | if (!re) { 33 | g_critical("Regular expression fail: %s", error); 34 | exit(EXIT_FAILURE); 35 | } 36 | 37 | rc = pcre_exec(re, NULL, db_version, strlen(db_version), 0, 0, ovector, 9); 38 | pcre_free(re); 39 | 40 | if (rc > 0) { 41 | return SERVER_TYPE_MYSQL; 42 | } 43 | 44 | re= pcre_compile(DETECT_DRIZZLE_REGEX, 0, &error, &erroroffset, NULL); 45 | if (!re) { 46 | g_critical("Regular expression fail: %s", error); 47 | exit(EXIT_FAILURE); 48 | } 49 | 50 | rc = pcre_exec(re, NULL, db_version, strlen(db_version), 0, 0, ovector, 9); 51 | pcre_free(re); 52 | 53 | if (rc > 0) { 54 | return SERVER_TYPE_DRIZZLE; 55 | } 56 | 57 | re= pcre_compile(DETECT_MARIADB_REGEX, 0, &error, &erroroffset, NULL); 58 | if (!re) { 59 | g_critical("Regular expression fail: %s", error); 60 | exit(EXIT_FAILURE); 61 | } 62 | 63 | rc = pcre_exec(re, NULL, db_version, strlen(db_version), 0, 0, ovector, 9); 64 | pcre_free(re); 65 | 66 | if (rc > 0) { 67 | return SERVER_TYPE_MYSQL; 68 | } 69 | 70 | return SERVER_TYPE_UNKNOWN; 71 | } 72 | -------------------------------------------------------------------------------- /docs/files.rst: -------------------------------------------------------------------------------- 1 | Output Files 2 | ============ 3 | 4 | mydumper generates several files during the generation of the dump. Many of 5 | these are for the table data itself since every table has at least one file. 6 | 7 | Metadata 8 | -------- 9 | When a dump is executed a file called ``metadata.partial`` is created in the output 10 | directory and is renamed to ``metadata`` when mydumper finish without error. 11 | This contains the start and end time of the dump as well as the 12 | master binary log positions if applicable. 13 | 14 | This is an example of the content of this file:: 15 | 16 | Started dump at: 2011-05-05 13:57:17 17 | SHOW MASTER STATUS: 18 | Log: linuxjedi-laptop-bin.000001 19 | Pos: 106 20 | 21 | Finished dump at: 2011-05-05 13:57:17 22 | 23 | Table Data 24 | ---------- 25 | The data from every table is written into a separate file, also if the 26 | :option:`--rows ` option is used then each chunk of table will 27 | be in a separate file. The file names for this are in the format:: 28 | 29 | database.table.sql(.gz) 30 | 31 | or if chunked:: 32 | 33 | database.table.chunk.sql(.gz) 34 | 35 | Where 'chunk' is a number padded with up to 5 zeros. 36 | 37 | Table Schemas 38 | ------------- 39 | When the :option:`--schemas ` option is used mydumper will 40 | create a file for the schema of every table it is writing data for. The files 41 | for this are in the following format:: 42 | 43 | database.table-schema.sql(.gz) 44 | 45 | Binary Logs 46 | ----------- 47 | Binary logs are retrieved when :option:`--binlogs ` option 48 | has been set. This will store them in the ``binlog_snapshot/`` sub-directory 49 | inside the dump directory. 50 | 51 | The binary log files have the same filename as the MySQL server that supplies them and will also have a .gz on the end if they are compressed. 52 | 53 | Daemon mode 54 | ----------- 55 | Daemon mode does things a little differently. There are the directories ``0`` 56 | and ``1`` inside the dump directory. These alternate when dumping so that if 57 | mydumper fails for any reason there is still a good snapshot. When a snapshot 58 | dump is complete the ``last_dump`` symlink is updated to point to that dump. 59 | 60 | If binary logging is enabled mydumper will connect as if it is a slave server 61 | and constantly retreives the binary logs into the ``binlogs`` subdirectory. 62 | -------------------------------------------------------------------------------- /mydumper.h: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Domas Mituzas, Facebook ( domas at fb dot com ) 16 | Mark Leith, Oracle Corporation (mark dot leith at oracle dot com) 17 | Andrew Hutchings, SkySQL (andrew at skysql dot com) 18 | Max Bubenick, Percona RDBA (max dot bubenick at percona dot com) 19 | 20 | */ 21 | 22 | #ifndef _mydumper_h 23 | #define _mydumper_h 24 | 25 | enum job_type { JOB_SHUTDOWN, JOB_RESTORE, JOB_DUMP, JOB_DUMP_NON_INNODB, JOB_SCHEMA, JOB_VIEW, JOB_TRIGGERS, JOB_SCHEMA_POST, JOB_BINLOG, JOB_LOCK_DUMP_NON_INNODB }; 26 | 27 | struct configuration { 28 | char use_any_index; 29 | GAsyncQueue* queue; 30 | GAsyncQueue* queue_less_locking; 31 | GAsyncQueue* ready; 32 | GAsyncQueue* ready_less_locking; 33 | GAsyncQueue* unlock_tables; 34 | GMutex* mutex; 35 | int done; 36 | }; 37 | 38 | struct thread_data { 39 | struct configuration *conf; 40 | guint thread_id; 41 | }; 42 | 43 | struct job { 44 | enum job_type type; 45 | void *job_data; 46 | struct configuration *conf; 47 | }; 48 | 49 | struct table_job { 50 | char *database; 51 | char *table; 52 | char *filename; 53 | char *where; 54 | }; 55 | 56 | struct tables_job { 57 | GList* table_job_list; 58 | }; 59 | 60 | struct schema_job { 61 | char *database; 62 | char *table; 63 | char *filename; 64 | }; 65 | 66 | struct view_job { 67 | char *database; 68 | char *table; 69 | char *filename; 70 | char *filename2; 71 | }; 72 | 73 | struct schema_post_job { 74 | char *database; 75 | char *filename; 76 | }; 77 | 78 | struct restore_job { 79 | char *database; 80 | char *table; 81 | char *filename; 82 | }; 83 | 84 | struct binlog_job { 85 | char *filename; 86 | guint64 start_position; 87 | guint64 stop_position; 88 | }; 89 | 90 | struct db_table { 91 | char* database; 92 | char* table; 93 | guint64 datalength; 94 | }; 95 | 96 | struct schema_post { 97 | char* database; 98 | }; 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | project(mydumper) 3 | set(VERSION 0.9.5) 4 | set(ARCHIVE_NAME "${CMAKE_PROJECT_NAME}-${VERSION}") 5 | 6 | #Required packages 7 | set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules) 8 | find_package(MySQL) 9 | find_package(ZLIB) 10 | find_package(GLIB2) 11 | find_package(PCRE) 12 | 13 | option(BUILD_DOCS "Build the documentation" ON) 14 | 15 | if (BUILD_DOCS) 16 | add_subdirectory(docs) 17 | endif (BUILD_DOCS) 18 | 19 | option(WITH_BINLOG "Build binlog dump options" OFF) 20 | option(WITH_SSL "Build SSL support" ON) 21 | 22 | set(CMAKE_C_FLAGS "-Wall -Wno-deprecated-declarations -Wunused -Wwrite-strings -Wno-strict-aliasing -Wextra -Wshadow -Werror -O3 -g ${MYSQL_CFLAGS}") 23 | 24 | include_directories(${MYDUMPER_SOURCE_DIR} ${MYSQL_INCLUDE_DIR} ${GLIB2_INCLUDE_DIR} ${PCRE_INCLUDE_DIR} ${ZLIB_INCLUDE_DIRS}) 25 | 26 | if (NOT CMAKE_INSTALL_PREFIX) 27 | SET(CMAKE_INSTALL_PREFIX "/usr/local" CACHE STRING "Install path" FORCE) 28 | endif (NOT CMAKE_INSTALL_PREFIX) 29 | MARK_AS_ADVANCED(CMAKE) 30 | 31 | CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_SOURCE_DIR}/config.h) 32 | 33 | 34 | if (WITH_BINLOG) 35 | 36 | add_executable(mydumper mydumper.c binlog.c server_detect.c g_unix_signal.c connection.c getPassword.c) 37 | else (WITH_BINLOG) 38 | add_executable(mydumper mydumper.c server_detect.c g_unix_signal.c connection.c getPassword.c) 39 | endif (WITH_BINLOG) 40 | target_link_libraries(mydumper ${MYSQL_LIBRARIES} ${GLIB2_LIBRARIES} ${GTHREAD2_LIBRARIES} ${PCRE_PCRE_LIBRARY} ${ZLIB_LIBRARIES} stdc++) 41 | 42 | 43 | add_executable(myloader myloader.c connection.c getPassword.c) 44 | target_link_libraries(myloader ${MYSQL_LIBRARIES} ${GLIB2_LIBRARIES} ${GTHREAD2_LIBRARIES} ${PCRE_PCRE_LIBRARY} ${ZLIB_LIBRARIES} stdc++) 45 | 46 | INSTALL(TARGETS mydumper myloader 47 | RUNTIME DESTINATION bin 48 | ) 49 | 50 | add_custom_target(dist 51 | COMMAND bzr export --root=${ARCHIVE_NAME} 52 | ${CMAKE_BINARY_DIR}/${ARCHIVE_NAME}.tar.gz 53 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) 54 | 55 | OPTION(RUN_CPPCHECK "Run cppcheck" OFF) 56 | 57 | IF(RUN_CPPCHECK) 58 | include(CppcheckTargets) 59 | add_cppcheck(mydumper) 60 | add_cppcheck(myloader) 61 | ENDIF(RUN_CPPCHECK) 62 | 63 | 64 | MESSAGE(STATUS "------------------------------------------------") 65 | MESSAGE(STATUS "MYSQL_CONFIG = ${MYSQL_CONFIG}") 66 | MESSAGE(STATUS "CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}") 67 | MESSAGE(STATUS "BUILD_DOCS = ${BUILD_DOCS}") 68 | MESSAGE(STATUS "WITH_BINLOG = ${WITH_BINLOG}") 69 | MESSAGE(STATUS "RUN_CPPCHECK = ${RUN_CPPCHECK}") 70 | MESSAGE(STATUS "Change a values with: cmake -D=") 71 | MESSAGE(STATUS "------------------------------------------------") 72 | MESSAGE(STATUS) 73 | 74 | -------------------------------------------------------------------------------- /docs/myloader_usage.rst: -------------------------------------------------------------------------------- 1 | Myloader Usage 2 | ============== 3 | 4 | Synopsis 5 | -------- 6 | 7 | :program:`myloader` :option:`--directory ` = /path/to/mydumper/backup [:ref:`OPTIONS `] 8 | 9 | Description 10 | ----------- 11 | 12 | :program:`myloader` is a tool used for multi-threaded restoration of mydumper 13 | backups. 14 | 15 | .. _myloader-options-label: 16 | 17 | Options 18 | ------- 19 | 20 | The :program:`myloader` tool has several available options: 21 | 22 | .. program:: myloader 23 | 24 | .. option:: --help, -? 25 | 26 | Show help text 27 | 28 | .. option:: --defaults-file 29 | 30 | Use the given option file. If the file does not exist or is otherwise inaccessible, no failure occurs 31 | 32 | .. option:: --host, -h 33 | 34 | Hostname of MySQL server to connect to (default localhost) 35 | 36 | .. option:: --user, -u 37 | 38 | MySQL username with the correct privileges to execute the restoration 39 | 40 | .. option:: --password, -p 41 | 42 | The corresponding password for the MySQL user 43 | 44 | .. option:: --port, -P 45 | 46 | The port for the MySQL connection. 47 | 48 | .. note:: 49 | 50 | For localhost TCP connections use 127.0.0.1 for :option:`--host`. 51 | 52 | .. option:: --socket, -S 53 | 54 | The UNIX domain socket file to use for the connection 55 | 56 | .. option:: --threads, -t 57 | 58 | The number of threads to use for restoring data, default is 4 59 | 60 | .. option:: --version, -V 61 | 62 | Show the program version and exit 63 | 64 | .. option:: --compress-protocol, -C 65 | 66 | Use client protocol compression for connections to the MySQL server 67 | 68 | .. option:: --directory, -d 69 | 70 | The directory of the mydumper backup to restore 71 | 72 | .. option:: --database, -B 73 | 74 | An alternative database to load the dump into 75 | 76 | .. note:: 77 | 78 | For use with single database dumps. When using with multi-database dumps 79 | that have duplicate table names in more than one database it may cause 80 | errors. Alternatively this scenario may give unpredictable results with 81 | :option:`--overwrite-tables`. 82 | 83 | .. option:: --source-db, -s 84 | 85 | Database to restore, useful in combination with --database 86 | 87 | .. option:: --queries-per-transaction, -q 88 | 89 | Number of INSERT queries to execute per transaction during restore, default 90 | is 1000. 91 | 92 | .. option:: --overwrite-tables, -o 93 | 94 | Drop any existing tables when restoring schemas 95 | 96 | .. option:: --enable-binlog, -e 97 | 98 | Log the data loading in the MySQL binary log if enabled (off by default) 99 | 100 | .. option:: --verbose, -v 101 | 102 | The verbosity of messages. 0 = silent, 1 = errors, 2 = warnings, 3 = info. 103 | Default is 2. 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What is mydumper? Why? 2 | 3 | * Parallelism (hence, speed) and performance (avoids expensive character set conversion routines, efficient code overall) 4 | * Easier to manage output (separate files for tables, dump metadata, etc, easy to view/parse data) 5 | * Consistency - maintains snapshot across all threads, provides accurate master and slave log positions, etc 6 | * Manageability - supports PCRE for specifying database and tables inclusions and exclusions 7 | 8 | ## How to install mydumper/myloader? 9 | 10 | First get the correct url from the [releases section](https://github.com/maxbube/mydumper/releases) then: 11 | 12 | ### RedHat / Centos 13 | 14 | ```bash 15 | yum install https://github.com/maxbube/mydumper/releases/download/v0.9.5/mydumper-0.9.5-1.el7.x86_64.rpm 16 | ``` 17 | 18 | ### Ubuntu / Debian 19 | 20 | ```bash 21 | wget https://github.com/maxbube/mydumper/releases/download/v0.9.5/mydumper_0.9.5-1.xenial_amd64.deb 22 | dpkg -i mydumper_0.9.5-1.xenial_amd64.deb 23 | ``` 24 | 25 | ## How to build it? 26 | 27 | Run: 28 | 29 | ```bash 30 | cmake . 31 | make 32 | ``` 33 | 34 | One needs to install development versions of required libaries (MySQL, GLib, ZLib, PCRE): 35 | NOTE: you must use the correspondent mysql devel package. 36 | 37 | * Ubuntu or Debian: apt-get install libglib2.0-dev libmysqlclient15-dev zlib1g-dev libpcre3-dev libssl-dev 38 | * Fedora, RedHat and CentOS: yum install glib2-devel mysql-devel zlib-devel pcre-devel openssl-devel 39 | * openSUSE: zypper install glib2-devel libmysqlclient-devel pcre-devel zlib-devel 40 | * MacOSX: port install glib2 mysql5 pcre pkgconfig cmake 41 | (You may want to run 'port select mysql mysql5' afterwards) 42 | 43 | One has to make sure, that pkg-config, mysql_config, pcre-config are all in $PATH 44 | 45 | Binlog dump is disabled by default to compile with it you need to add -DWITH_BINLOG=ON to cmake options 46 | 47 | To build against mysql libs < 5.7 you need to disable SSL adding -DWITH_SSL=OFF 48 | 49 | ## How does consistent snapshot work? 50 | 51 | This is all done following best MySQL practices and traditions: 52 | 53 | * As a precaution, slow running queries on the server either abort the dump, or get killed 54 | * Global write lock is acquired ("FLUSH TABLES WITH READ LOCK") 55 | * Various metadata is read ("SHOW SLAVE STATUS","SHOW MASTER STATUS") 56 | * Other threads connect and establish snapshots ("START TRANSACTION WITH CONSISTENT SNAPSHOT") 57 | ** On pre-4.1.8 it creates dummy InnoDB table, and reads from it. 58 | * Once all worker threads announce the snapshot establishment, master executes "UNLOCK TABLES" and starts queueing jobs. 59 | 60 | This for now does not provide consistent snapshots for non-transactional engines - support for that is expected in 0.2 :) 61 | 62 | ## How to exclude (or include) databases? 63 | 64 | Once can use --regex functionality, for example not to dump mysql and test databases: 65 | 66 | ```bash 67 | mydumper --regex '^(?!(mysql\.|test\.))' 68 | ``` 69 | 70 | To dump only mysql and test databases: 71 | 72 | ```bash 73 | mydumper --regex '^(mysql\.|test\.)' 74 | ``` 75 | 76 | To not dump all databases starting with test: 77 | 78 | ```bash 79 | mydumper --regex '^(?!(test))' 80 | ``` 81 | 82 | Of course, regex functionality can be used to describe pretty much any list of tables. 83 | 84 | 85 | -------------------------------------------------------------------------------- /common.h: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Andrew Hutchings, SkySQL (andrew at skysql dot com) 16 | */ 17 | #ifndef _common_h 18 | #define _common_h 19 | 20 | char *hostname=NULL; 21 | char *username=NULL; 22 | char *password=NULL; 23 | char *socket_path=NULL; 24 | char *db=NULL; 25 | char *defaults_file=NULL; 26 | #ifdef WITH_SSL 27 | char *key=NULL; 28 | char *cert=NULL; 29 | char *ca=NULL; 30 | char *capath=NULL; 31 | char *cipher=NULL; 32 | #endif 33 | 34 | gboolean askPassword=FALSE; 35 | guint port=0; 36 | guint num_threads= 4; 37 | guint verbose=2; 38 | gboolean ssl= FALSE; 39 | gboolean compress_protocol= FALSE; 40 | gboolean program_version= FALSE; 41 | 42 | GOptionEntry common_entries[] = 43 | { 44 | { "host", 'h', 0, G_OPTION_ARG_STRING, &hostname, "The host to connect to", NULL }, 45 | { "user", 'u', 0, G_OPTION_ARG_STRING, &username, "Username with the necessary privileges", NULL }, 46 | { "password", 'p', 0, G_OPTION_ARG_STRING, &password, "User password", NULL }, 47 | { "ask-password", 'a', 0, G_OPTION_ARG_NONE, &askPassword, "Prompt For User password", NULL }, 48 | { "port", 'P', 0, G_OPTION_ARG_INT, &port, "TCP/IP port to connect to", NULL }, 49 | { "socket", 'S', 0, G_OPTION_ARG_STRING, &socket_path, "UNIX domain socket file to use for connection", NULL }, 50 | { "threads", 't', 0, G_OPTION_ARG_INT, &num_threads, "Number of threads to use, default 4", NULL }, 51 | { "compress-protocol", 'C', 0, G_OPTION_ARG_NONE, &compress_protocol, "Use compression on the MySQL connection", NULL }, 52 | { "version", 'V', 0, G_OPTION_ARG_NONE, &program_version, "Show the program version and exit", NULL }, 53 | { "verbose", 'v', 0, G_OPTION_ARG_INT, &verbose, "Verbosity of output, 0 = silent, 1 = errors, 2 = warnings, 3 = info, default 2", NULL }, 54 | { "defaults-file", 0, 0, G_OPTION_ARG_FILENAME, &defaults_file, "Use a specific defaults file", NULL }, 55 | #ifdef WITH_SSL 56 | { "ssl", 0, 0, G_OPTION_ARG_NONE, &ssl, "Connect using SSL", NULL}, 57 | { "key", 0, 0, G_OPTION_ARG_STRING, &key, "The path name to the key file", NULL }, 58 | { "cert", 0, 0, G_OPTION_ARG_STRING, &cert, "The path name to the certificate file", NULL }, 59 | { "ca", 0, 0, G_OPTION_ARG_STRING, &ca, "The path name to the certificate authority file", NULL }, 60 | { "capath", 0, 0, G_OPTION_ARG_STRING, &capath, "The path name to a directory that contains trusted SSL CA certificates in PEM format", NULL }, 61 | { "cipher", 0, 0, G_OPTION_ARG_STRING, &cipher, "A list of permissible ciphers to use for SSL encryption", NULL }, 62 | #endif 63 | { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL } 64 | }; 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /g_unix_signal.c: -------------------------------------------------------------------------------- 1 | #define _POSIX_SOURCE 2 | #include 3 | #include 4 | 5 | static GPtrArray *signal_data = NULL; 6 | 7 | typedef struct _GUnixSignalData { 8 | guint source_id; 9 | GMainContext *context; 10 | gboolean triggered; 11 | gint signum; 12 | } GUnixSignalData; 13 | 14 | typedef struct _GUnixSignalSource { 15 | GSource source; 16 | GUnixSignalData *data; 17 | } GUnixSignalSource; 18 | 19 | static inline GUnixSignalData* get_signal_data(guint index) 20 | { 21 | return (GUnixSignalData*)g_ptr_array_index(signal_data, index); 22 | } 23 | 24 | static void handler(gint signum) { 25 | g_assert(signal_data != NULL); 26 | guint i; 27 | for (i = 0; i < signal_data->len; ++i) 28 | if (get_signal_data(i)->signum == signum) 29 | get_signal_data(i)->triggered = TRUE; 30 | 31 | struct sigaction action; 32 | action.sa_handler= handler; 33 | sigemptyset (&action.sa_mask); 34 | action.sa_flags = 0; 35 | sigaction(signum, &action, NULL); 36 | } 37 | 38 | static gboolean check(GSource *source) 39 | { 40 | GUnixSignalSource *signal_source = (GUnixSignalSource*) source; 41 | return signal_source->data->triggered; 42 | } 43 | 44 | static gboolean prepare(GSource *source, gint *timeout_) 45 | { 46 | GUnixSignalSource *signal_source = (GUnixSignalSource*) source; 47 | if (signal_source->data->context == NULL) { 48 | g_main_context_ref(signal_source->data->context = g_source_get_context(source)); 49 | signal_source->data->source_id = g_source_get_id(source); 50 | } 51 | 52 | *timeout_ = -1; 53 | return signal_source->data->triggered; 54 | } 55 | 56 | static gboolean dispatch(GSource *source, GSourceFunc callback, gpointer user_data) 57 | { 58 | GUnixSignalSource *signal_source = (GUnixSignalSource*) source; 59 | signal_source->data->triggered = FALSE; 60 | return callback(user_data) ? TRUE : FALSE; 61 | } 62 | static void finalize(GSource *source) 63 | { 64 | GUnixSignalSource *signal_source = (GUnixSignalSource*) source; 65 | 66 | struct sigaction action; 67 | action.sa_handler= NULL; 68 | sigemptyset (&action.sa_mask); 69 | action.sa_flags = 0; 70 | 71 | sigaction(signal_source->data->signum, &action, NULL); 72 | g_main_context_unref(signal_source->data->context); 73 | g_ptr_array_remove_fast(signal_data, signal_source->data); 74 | if (signal_data->len == 0) 75 | signal_data = (GPtrArray*) g_ptr_array_free(signal_data, TRUE); 76 | g_free(signal_source->data); 77 | 78 | } 79 | static GSourceFuncs SourceFuncs = 80 | { 81 | .prepare = prepare, 82 | .check = check, 83 | .dispatch = dispatch, 84 | .finalize = finalize, 85 | .closure_callback = NULL, .closure_marshal = NULL 86 | }; 87 | 88 | static void g_unix_signal_source_init(GSource *source, gint signum) 89 | { 90 | GUnixSignalSource *signal_source = (GUnixSignalSource *) source; 91 | signal_source->data = g_new(GUnixSignalData, 1); 92 | signal_source->data->triggered = FALSE; 93 | signal_source->data->signum = signum; 94 | signal_source->data->context = NULL; 95 | 96 | if (signal_data == NULL) 97 | signal_data = g_ptr_array_new(); 98 | g_ptr_array_add(signal_data, signal_source->data); 99 | } 100 | 101 | GSource *g_unix_signal_source_new(gint signum) 102 | { 103 | GSource *source = g_source_new(&SourceFuncs, sizeof(GUnixSignalSource)); 104 | g_unix_signal_source_init(source, signum); 105 | struct sigaction action; 106 | action.sa_handler= handler; 107 | sigemptyset (&action.sa_mask); 108 | action.sa_flags = 0; 109 | sigaction(signum, &action, NULL); 110 | return source; 111 | } 112 | 113 | guint g_unix_signal_add_full(gint priority, gint signum, GSourceFunc function, gpointer data, GDestroyNotify notify) 114 | { 115 | g_return_val_if_fail(function != NULL, 0); 116 | GSource *source = g_unix_signal_source_new(signum); 117 | if (priority != G_PRIORITY_DEFAULT) 118 | g_source_set_priority (source, priority); 119 | g_source_set_callback(source, function, data, notify); 120 | guint id = g_source_attach(source, NULL); 121 | g_source_unref(source); 122 | return id; 123 | } 124 | 125 | guint g_unix_signal_add(gint signum, GSourceFunc function, gpointer data) 126 | { 127 | return g_unix_signal_add_full(G_PRIORITY_DEFAULT, signum, function, data, NULL); 128 | } 129 | -------------------------------------------------------------------------------- /cmake/modules/FindMySQL.cmake: -------------------------------------------------------------------------------- 1 | # - Find MySQL 2 | # Find the MySQL includes and client library 3 | # This module defines 4 | # MYSQL_INCLUDE_DIR, where to find mysql.h 5 | # MYSQL_LIBRARIES, the libraries needed to use MySQL. 6 | # MYSQL_FOUND, If false, do not try to use MySQL. 7 | # 8 | # Copyright (c) 2006, Jaroslaw Staniek, 9 | # Lot of adustmens by Michal Cihar 10 | # 11 | # vim: expandtab sw=4 ts=4 sts=4: 12 | # 13 | # Redistribution and use is allowed according to the terms of the BSD license. 14 | 15 | if(UNIX) 16 | set(MYSQL_CONFIG_PREFER_PATH "$ENV{MYSQL_HOME}/bin" CACHE FILEPATH 17 | "preferred path to MySQL (mysql_config)") 18 | find_program(MYSQL_CONFIG mysql_config 19 | ${MYSQL_CONFIG_PREFER_PATH} 20 | /usr/local/mysql/bin/ 21 | /usr/local/bin/ 22 | /usr/bin/ 23 | ) 24 | 25 | if(MYSQL_CONFIG) 26 | message(STATUS "Using mysql-config: ${MYSQL_CONFIG}") 27 | # set CFLAGS 28 | exec_program(${MYSQL_CONFIG} 29 | ARGS --cflags 30 | OUTPUT_VARIABLE MY_TMP) 31 | 32 | set(MYSQL_CFLAGS ${MY_TMP} CACHE STRING INTERNAL) 33 | 34 | # set INCLUDE_DIR 35 | exec_program(${MYSQL_CONFIG} 36 | ARGS --include 37 | OUTPUT_VARIABLE MY_TMP) 38 | 39 | string(REGEX REPLACE "-I([^ ]*)( .*)?" "\\1" MY_TMP "${MY_TMP}") 40 | 41 | set(MYSQL_ADD_INCLUDE_DIR ${MY_TMP} CACHE FILEPATH INTERNAL) 42 | 43 | # set LIBRARY_DIR 44 | exec_program(${MYSQL_CONFIG} 45 | ARGS --libs_r 46 | OUTPUT_VARIABLE MY_TMP) 47 | 48 | set(MYSQL_ADD_LIBRARIES "") 49 | 50 | # prepend space in order to match separate words only (e.g. rather 51 | # than "-linux" from within "-L/usr/lib/i386-linux-gnu") 52 | string(REGEX MATCHALL " +-l[^ ]*" MYSQL_LIB_LIST " ${MY_TMP}") 53 | foreach(MY_LIB ${MYSQL_LIB_LIST}) 54 | string(REGEX REPLACE "[ ]*-l([^ ]*)" "\\1" MY_LIB "${MY_LIB}") 55 | list(APPEND MYSQL_ADD_LIBRARIES "${MY_LIB}") 56 | endforeach(MY_LIB ${MYSQL_LIBS}) 57 | 58 | set(MYSQL_ADD_LIBRARY_PATH "") 59 | 60 | string(REGEX MATCHALL " +-L[^ ]*" MYSQL_LIBDIR_LIST " ${MY_TMP}") 61 | foreach(MY_LIB ${MYSQL_LIBDIR_LIST}) 62 | string(REGEX REPLACE "[ ]*-L([^ ]*)" "\\1" MY_LIB "${MY_LIB}") 63 | list(APPEND MYSQL_ADD_LIBRARY_PATH "${MY_LIB}") 64 | endforeach(MY_LIB ${MYSQL_LIBS}) 65 | 66 | else(MYSQL_CONFIG) 67 | set(MYSQL_ADD_LIBRARIES "") 68 | list(APPEND MYSQL_ADD_LIBRARIES "mysqlclient") 69 | endif(MYSQL_CONFIG) 70 | else(UNIX) 71 | set(MYSQL_ADD_INCLUDE_DIR "c:/msys/local/include" CACHE FILEPATH INTERNAL) 72 | set(MYSQL_ADD_LIBRARY_PATH "c:/msys/local/lib" CACHE FILEPATH INTERNAL) 73 | ENDIF(UNIX) 74 | 75 | find_path(MYSQL_INCLUDE_DIR mysql.h 76 | ${MYSQL_ADD_INCLUDE_DIR} 77 | /usr/local/include 78 | /usr/local/include/mysql 79 | /usr/local/mysql/include 80 | /usr/local/mysql/include/mysql 81 | /usr/include 82 | /usr/include/mysql 83 | /usr/include/mysql/private 84 | ) 85 | 86 | set(TMP_MYSQL_LIBRARIES "") 87 | set(CMAKE_FIND_LIBRARY_SUFFIXES .so .a .lib .so.1) 88 | foreach(MY_LIB ${MYSQL_ADD_LIBRARIES}) 89 | find_library("MYSQL_LIBRARIES_${MY_LIB}" NAMES ${MY_LIB} 90 | HINTS 91 | ${MYSQL_ADD_LIBRARY_PATH} 92 | /usr/lib/mysql 93 | /usr/lib 94 | /usr/local/lib 95 | /usr/local/lib/mysql 96 | /usr/local/mysql/lib 97 | ) 98 | list(APPEND TMP_MYSQL_LIBRARIES "${MYSQL_LIBRARIES_${MY_LIB}}") 99 | endforeach(MY_LIB ${MYSQL_ADD_LIBRARIES}) 100 | 101 | set(MYSQL_LIBRARIES ${TMP_MYSQL_LIBRARIES} CACHE FILEPATH INTERNAL) 102 | 103 | if(MYSQL_INCLUDE_DIR AND MYSQL_LIBRARIES) 104 | set(MYSQL_FOUND TRUE CACHE INTERNAL "MySQL found") 105 | message(STATUS "Found MySQL: ${MYSQL_INCLUDE_DIR}, ${MYSQL_LIBRARIES}") 106 | else(MYSQL_INCLUDE_DIR AND MYSQL_LIBRARIES) 107 | set(MYSQL_FOUND FALSE CACHE INTERNAL "MySQL found") 108 | message(STATUS "MySQL not found.") 109 | endif(MYSQL_INCLUDE_DIR AND MYSQL_LIBRARIES) 110 | 111 | mark_as_advanced(MYSQL_INCLUDE_DIR MYSQL_LIBRARIES MYSQL_CFLAGS) 112 | -------------------------------------------------------------------------------- /cmake/modules/Findcppcheck.cmake: -------------------------------------------------------------------------------- 1 | # - try to find cppcheck tool 2 | # 3 | # Cache Variables: 4 | # CPPCHECK_EXECUTABLE 5 | # 6 | # Non-cache variables you might use in your CMakeLists.txt: 7 | # CPPCHECK_FOUND 8 | # CPPCHECK_POSSIBLEERROR_ARG 9 | # CPPCHECK_UNUSEDFUNC_ARG 10 | # CPPCHECK_STYLE_ARG 11 | # CPPCHECK_QUIET_ARG 12 | # CPPCHECK_INCLUDEPATH_ARG 13 | # CPPCHECK_FAIL_REGULAR_EXPRESSION 14 | # CPPCHECK_WARN_REGULAR_EXPRESSION 15 | # CPPCHECK_MARK_AS_ADVANCED - whether to mark our vars as advanced even 16 | # if we don't find this program. 17 | # 18 | # Requires these CMake modules: 19 | # FindPackageHandleStandardArgs (known included with CMake >=2.6.2) 20 | # 21 | # Original Author: 22 | # 2009-2010 Ryan Pavlik 23 | # http://academic.cleardefinition.com 24 | # Iowa State University HCI Graduate Program/VRAC 25 | # 26 | # Copyright Iowa State University 2009-2010. 27 | # Distributed under the Boost Software License, Version 1.0. 28 | # (See accompanying file LICENSE_1_0.txt or copy at 29 | # http://www.boost.org/LICENSE_1_0.txt) 30 | 31 | file(TO_CMAKE_PATH "${CPPCHECK_ROOT_DIR}" CPPCHECK_ROOT_DIR) 32 | set(CPPCHECK_ROOT_DIR 33 | "${CPPCHECK_ROOT_DIR}" 34 | CACHE 35 | PATH 36 | "Path to search for cppcheck") 37 | 38 | # cppcheck app bundles on Mac OS X are GUI, we want command line only 39 | set(_oldappbundlesetting ${CMAKE_FIND_APPBUNDLE}) 40 | set(CMAKE_FIND_APPBUNDLE NEVER) 41 | 42 | # If we have a custom path, look there first. 43 | if(CPPCHECK_ROOT_DIR) 44 | find_program(CPPCHECK_EXECUTABLE 45 | NAMES 46 | cppcheck 47 | cli 48 | PATHS 49 | "${CPPCHECK_ROOT_DIR}" 50 | PATH_SUFFIXES 51 | cli 52 | NO_DEFAULT_PATH) 53 | endif() 54 | 55 | find_program(CPPCHECK_EXECUTABLE NAMES cppcheck) 56 | 57 | # Restore original setting for appbundle finding 58 | set(CMAKE_FIND_APPBUNDLE ${_oldappbundlesetting}) 59 | 60 | if(CPPCHECK_EXECUTABLE) 61 | # Find out where our test file is 62 | get_filename_component(_cppcheckmoddir ${CMAKE_CURRENT_LIST_FILE} PATH) 63 | set(_cppcheckdummyfile "${_cppcheckmoddir}/Findcppcheck.cpp") 64 | 65 | # Check for the two types of command line arguments by just trying them 66 | execute_process(COMMAND 67 | "${CPPCHECK_EXECUTABLE}" 68 | "--enable=style" 69 | "--quiet" 70 | "${_cppcheckdummyfile}" 71 | RESULT_VARIABLE 72 | _cppcheck_new_result 73 | OUTPUT_QUIET 74 | ERROR_QUIET) 75 | execute_process(COMMAND 76 | "${CPPCHECK_EXECUTABLE}" 77 | "--style" 78 | "--quiet" 79 | "${_cppcheckdummyfile}" 80 | RESULT_VARIABLE 81 | _cppcheck_old_result 82 | OUTPUT_QUIET 83 | ERROR_QUIET) 84 | if("${_cppcheck_new_result}" EQUAL 0) 85 | # New arguments 86 | set(CPPCHECK_UNUSEDFUNC_ARG "--enable=unusedFunctions") 87 | set(CPPCHECK_POSSIBLEERROR_ARG "--enable=possibleError") 88 | set(CPPCHECK_STYLE_ARG "--enable=style") 89 | set(CPPCHECK_QUIET_ARG "--quiet") 90 | set(CPPCHECK_INCLUDEPATH_ARG "-I") 91 | if(MSVC) 92 | set(CPPCHECK_TEMPLATE_ARG --template vs) 93 | set(CPPCHECK_FAIL_REGULAR_EXPRESSION "[(]error[)]") 94 | set(CPPCHECK_WARN_REGULAR_EXPRESSION "[(]style[)]") 95 | elseif(CMAKE_COMPILER_IS_GNUCXX) 96 | set(CPPCHECK_TEMPLATE_ARG --template gcc) 97 | set(CPPCHECK_FAIL_REGULAR_EXPRESSION " error: ") 98 | set(CPPCHECK_WARN_REGULAR_EXPRESSION " style: ") 99 | else() 100 | message(STATUS 101 | "Warning: FindCppcheck doesn't know how to format error messages for your compiler!") 102 | set(CPPCHECK_TEMPLATE_ARG --template gcc) 103 | set(CPPCHECK_FAIL_REGULAR_EXPRESSION " error: ") 104 | set(CPPCHECK_WARN_REGULAR_EXPRESSION " style: ") 105 | endif() 106 | elseif("${_cppcheck_old_result}" EQUAL 0) 107 | # Old arguments 108 | set(CPPCHECK_UNUSEDFUNC_ARG "--unused-functions") 109 | set(CPPCHECK_POSSIBLEERROR_ARG "--all") 110 | set(CPPCHECK_STYLE_ARG "--style") 111 | set(CPPCHECK_QUIET_ARG "--quiet") 112 | set(CPPCHECK_INCLUDEPATH_ARG "-I") 113 | set(CPPCHECK_FAIL_REGULAR_EXPRESSION "error:") 114 | set(CPPCHECK_WARN_REGULAR_EXPRESSION "[(]style[)]") 115 | else() 116 | # No idea - some other issue must be getting in the way 117 | message(STATUS 118 | "WARNING: Can't detect whether CPPCHECK wants new or old-style arguments!") 119 | endif() 120 | 121 | 122 | endif() 123 | 124 | set(CPPCHECK_ALL 125 | "${CPPCHECK_EXECUTABLE} ${CPPCHECK_POSSIBLEERROR_ARG} ${CPPCHECK_UNUSEDFUNC_ARG} ${CPPCHECK_STYLE_ARG} ${CPPCHECK_QUIET_ARG} ${CPPCHECK_INCLUDEPATH_ARG} some/include/path") 126 | 127 | include(FindPackageHandleStandardArgs) 128 | find_package_handle_standard_args(cppcheck 129 | DEFAULT_MSG 130 | CPPCHECK_ALL 131 | CPPCHECK_EXECUTABLE 132 | CPPCHECK_POSSIBLEERROR_ARG 133 | CPPCHECK_UNUSEDFUNC_ARG 134 | CPPCHECK_STYLE_ARG 135 | CPPCHECK_INCLUDEPATH_ARG 136 | CPPCHECK_QUIET_ARG) 137 | 138 | if(CPPCHECK_FOUND OR CPPCHECK_MARK_AS_ADVANCED) 139 | mark_as_advanced(CPPCHECK_ROOT_DIR) 140 | endif() 141 | 142 | mark_as_advanced(CPPCHECK_EXECUTABLE) 143 | -------------------------------------------------------------------------------- /docs/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Generate documentation in HTML and PDF format using Sphinx. 2 | 3 | set(GENERATE_DOC TRUE) 4 | 5 | # We use the Sphinx documentation generator to render HTML and manual 6 | # pages from the user and reference documentation in ReST format. 7 | find_package(Sphinx QUIET) 8 | if(NOT SPHINX_FOUND) 9 | message(WARNING "Unable to find Sphinx documentation generator") 10 | set(GENERATE_DOC FALSE) 11 | endif(NOT SPHINX_FOUND) 12 | 13 | if(SPHINX_MAJOR_VERSION LESS 1) 14 | message(WARNING "Sphinx is older than v1.0, not building docs") 15 | set(GENERATE_DOC FALSE) 16 | endif(SPHINX_MAJOR_VERSION LESS 1) 17 | 18 | if(GENERATE_DOC) 19 | # documentation tools 20 | set(SOURCE_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/_build") 21 | # configured documentation tools and intermediate build results 22 | set(BINARY_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/_build") 23 | # static ReST documentation sources 24 | set(SOURCES_DIR "${CMAKE_CURRENT_BINARY_DIR}/_sources") 25 | # generated ReST documentation sources 26 | set(REF_SOURCES_DIR "${SOURCES_DIR}/reference") 27 | # master document with modules index 28 | set(REF_MASTER_DOC "modules") 29 | 30 | # substitute variables in configuration and scripts 31 | foreach(file 32 | conf.py 33 | sources.cmake 34 | ) 35 | configure_file( 36 | "${SOURCE_BUILD_DIR}/${file}.in" 37 | "${BINARY_BUILD_DIR}/${file}" 38 | @ONLY 39 | ) 40 | endforeach(file) 41 | 42 | set(CLEAN_FILES 43 | "${BINARY_BUILD_DIR}/html" 44 | ) 45 | 46 | add_custom_target(ALL 47 | DEPENDS "${REF_SOURCES_DIR}/${REF_MASTER_DOC}.rst" 48 | ) 49 | 50 | # Sphinx requires all sources in the same directory tree. As we wish 51 | # to include generated reference documention from the build tree, we 52 | # copy static ReST documents to the build tree before calling Sphinx. 53 | add_custom_target(doc_sources ALL 54 | "${CMAKE_COMMAND}" -P "${BINARY_BUILD_DIR}/sources.cmake" 55 | ) 56 | list(APPEND CLEAN_FILES 57 | "${SOURCES_DIR}" 58 | ) 59 | 60 | # note the trailing slash to exclude directory name 61 | install(DIRECTORY "${SOURCES_DIR}/" 62 | DESTINATION "share/doc/mydumper" 63 | ) 64 | 65 | # Sphinx cache with pickled ReST documents 66 | set(SPHINX_CACHE_DIR "${CMAKE_CURRENT_BINARY_DIR}/_doctrees") 67 | # HTML output directory 68 | set(SPHINX_HTML_DIR "${CMAKE_CURRENT_BINARY_DIR}/html") 69 | 70 | # This target builds HTML documentation using Sphinx. 71 | add_custom_target(doc_html ALL 72 | ${SPHINX_EXECUTABLE} 73 | -q -b html 74 | -c "${BINARY_BUILD_DIR}" 75 | -d "${SPHINX_CACHE_DIR}" 76 | "${SOURCES_DIR}" 77 | "${SPHINX_HTML_DIR}" 78 | COMMENT "Building HTML documentation with Sphinx" 79 | ) 80 | list(APPEND CLEAN_FILES 81 | "${SPHINX_CACHE_DIR}" 82 | "${SPHINX_HTML_DIR}" 83 | ) 84 | add_dependencies(doc_html 85 | doc_sources 86 | ) 87 | install(DIRECTORY "${SPHINX_HTML_DIR}" 88 | DESTINATION "share/doc/mydumper" 89 | ) 90 | 91 | # HTML output directory 92 | set(SPHINX_MAN_DIR "${CMAKE_CURRENT_BINARY_DIR}/man") 93 | # This target builds a manual page using Sphinx. 94 | 95 | add_custom_target(doc_man ALL 96 | ${SPHINX_EXECUTABLE} 97 | -q -b man 98 | -c "${BINARY_BUILD_DIR}" 99 | -d "${SPHINX_CACHE_DIR}" 100 | "${SOURCES_DIR}" 101 | "${SPHINX_MAN_DIR}" 102 | COMMENT "Building manual page with Sphinx" 103 | ) 104 | list(APPEND CLEAN_FILES 105 | "${SPHINX_MAN_DIR}" 106 | ) 107 | add_dependencies(doc_man 108 | doc_sources 109 | ) 110 | # serialize Sphinx targets to avoid cache conflicts in parallel builds 111 | add_dependencies(doc_man 112 | doc_html 113 | ) 114 | install(FILES "${SPHINX_MAN_DIR}/mydumper.1" "${SPHINX_MAN_DIR}/myloader.1" 115 | DESTINATION "share/man/man1" 116 | ) 117 | 118 | # This target builds PDF documentation using Sphinx and LaTeX. 119 | if(PDFLATEX_COMPILER) 120 | # PDF output directory 121 | set(SPHINX_PDF_DIR "${CMAKE_CURRENT_BINARY_DIR}/pdf") 122 | 123 | add_custom_target(doc_pdf ALL 124 | ${SPHINX_EXECUTABLE} 125 | -q -b latex 126 | -c "${BINARY_BUILD_DIR}" 127 | -d "${SPHINX_CACHE_DIR}" 128 | "${SOURCES_DIR}" 129 | "${SPHINX_PDF_DIR}" 130 | COMMENT "Building PDF documentation with Sphinx" 131 | ) 132 | add_custom_command(TARGET doc_pdf POST_BUILD 133 | COMMAND ${CMAKE_MAKE_PROGRAM} LATEXOPTS=-interaction=batchmode 134 | WORKING_DIRECTORY "${SPHINX_PDF_DIR}" 135 | ) 136 | list(APPEND CLEAN_FILES 137 | "${SPHINX_PDF_DIR}" 138 | ) 139 | add_dependencies(doc_pdf 140 | doc_sources 141 | ) 142 | # serialize Sphinx targets to avoid cache conflicts in parallel builds 143 | add_dependencies(doc_pdf 144 | doc_man 145 | ) 146 | install(FILES "${SPHINX_PDF_DIR}/mydumper.pdf" 147 | DESTINATION "share/doc/mydumper" 148 | ) 149 | endif(PDFLATEX_COMPILER) 150 | 151 | # Add output directories to clean target. 152 | set_directory_properties(PROPERTIES 153 | ADDITIONAL_MAKE_CLEAN_FILES "${CLEAN_FILES}" 154 | ) 155 | 156 | endif(GENERATE_DOC) 157 | -------------------------------------------------------------------------------- /docs/mydumper_usage.rst: -------------------------------------------------------------------------------- 1 | Mydumper Usage 2 | ============== 3 | 4 | Synopsis 5 | -------- 6 | 7 | :program:`mydumper` [:ref:`OPTIONS `] 8 | 9 | Description 10 | ----------- 11 | 12 | :program:`mydumper` is a tool used for backing up MySQL database servers much 13 | faster than the mysqldump tool distributed with MySQL. It also has the 14 | capability to retrieve the binary logs from the remote server at the same time 15 | as the dump itself. The advantages of mydumper are: 16 | 17 | * Parallelism (hence, speed) and performance (avoids expensive character set conversion routines, efficient code overall) 18 | * Easier to manage output (separate files for tables, dump metadata, etc, easy to view/parse data) 19 | * Consistency - maintains snapshot across all threads, provides accurate master and slave log positions, etc 20 | * Manageability - supports PCRE for specifying database and tables inclusions and exclusions 21 | 22 | .. _mydumper-options-label: 23 | 24 | Options 25 | ------- 26 | 27 | The :program:`mydumper` tool has several available options: 28 | 29 | .. program:: mydumper 30 | 31 | .. option:: --help, -? 32 | 33 | Show help text 34 | 35 | .. option:: --defaults-file 36 | 37 | Use the given option file. If the file does not exist or is otherwise inaccessible, no failure occurs 38 | 39 | .. option:: --host, -h 40 | 41 | Hostname of MySQL server to connect to (default localhost) 42 | 43 | .. option:: --user, -u 44 | 45 | MySQL username with the correct privileges to execute the dump 46 | 47 | .. option:: --password, -p 48 | 49 | The corresponding password for the MySQL user 50 | 51 | .. option:: --port, -P 52 | 53 | The port for the MySQL connection. 54 | 55 | .. note:: 56 | 57 | For localhost TCP connections use 127.0.0.1 for :option:`--host`. 58 | 59 | .. option:: --socket, -S 60 | 61 | The UNIX domain socket file to use for the connection 62 | 63 | .. option:: --database, -B 64 | 65 | Database to dump 66 | 67 | .. option:: --tables-list, -T 68 | 69 | A comma separated list of tables to dump 70 | 71 | .. option:: --threads, -t 72 | 73 | The number of threads to use for dumping data, default is 4 74 | 75 | .. note:: 76 | 77 | Other threads are used in mydumper, this option does not control these 78 | 79 | .. option:: --outputdir, -o 80 | 81 | Output directory name, default is export-YYYYMMDD-HHMMSS 82 | 83 | .. option:: --statement-size, -s 84 | 85 | The maximum size for an insert statement before breaking into a new 86 | statement, default 1,000,000 bytes 87 | 88 | .. option:: --rows, -r 89 | 90 | Split table into chunks of this many rows, default unlimited 91 | 92 | .. option:: --compress, -c 93 | 94 | Compress the output files 95 | 96 | .. option:: --compress-input, -C 97 | 98 | Use client protocol compression for connections to the MySQL server 99 | 100 | .. option:: --build-empty-files, -e 101 | 102 | Create empty dump files if there is no data to dump 103 | 104 | .. option:: --regex, -x 105 | 106 | A regular expression to match against database and table 107 | 108 | .. option:: --omit-from-file, -O 109 | 110 | File containing a list of database.table entries to skip, one per line; the 111 | skipped entries have precedence over patterns specified by the regex option 112 | 113 | .. option:: --ignore-engines, -i 114 | 115 | Comma separated list of storage engines to ignore 116 | 117 | .. option:: --no-schemas, -m 118 | 119 | Do not dump schemas with the data 120 | 121 | .. option:: --no-data, -d 122 | 123 | Do not dump table data 124 | 125 | .. option:: --triggers, -G 126 | 127 | Dump triggers 128 | 129 | .. option:: --events, -E 130 | 131 | Dump events 132 | 133 | .. option:: --routines, -R 134 | 135 | Dump stored procedures and functions 136 | 137 | .. option:: --no-views, -W 138 | 139 | Do not dump views 140 | 141 | .. option:: --long-query-guard, -l 142 | 143 | Timeout for long query execution in seconds, default 60 144 | 145 | .. option:: --kill-long-queries, -K 146 | 147 | Kill long running queries instead of aborting the dump 148 | 149 | .. option:: --version, -V 150 | 151 | Show the program version and exit 152 | 153 | .. option:: --verbose, -v 154 | 155 | The verbosity of messages. 0 = silent, 1 = errors, 2 = warnings, 3 = info. 156 | Default is 2. 157 | 158 | .. option:: --binlogs, -b 159 | 160 | Get the binlogs from the server as well as the dump files (You need to compile with -DWITH_BINLOG=ON) 161 | 162 | .. option:: --daemon, -D 163 | 164 | Enable daemon mode 165 | 166 | .. option:: --snapshot-interval, -I 167 | 168 | Interval between each dump snapshot (in minutes), requires 169 | :option:`--daemon`, default 60 (minutes) 170 | 171 | .. option:: --logfile, -L 172 | 173 | A file to log mydumper output to instead of console output. Useful for 174 | daemon mode. 175 | 176 | .. option:: --no-locks, -k 177 | 178 | Do not execute the temporary shared read lock. 179 | 180 | .. warning:: 181 | 182 | This will cause inconsistent backups. 183 | 184 | .. option:: --no-backup-locks 185 | 186 | Do not use Percona Backup Locks 187 | 188 | .. option:: --[skip-]tz-utc 189 | 190 | SET TIME_ZONE='+00:00' at top of dump to allow dumping of TIMESTAMP data 191 | when a server has data in different time zones or data is being moved 192 | between servers with different time zones, defaults to on use --skip-tz-utc 193 | to disable. 194 | 195 | .. option:: --less-locking 196 | 197 | Minimize locking time on InnoDB tables grabbing a LOCK TABLE ... READ 198 | on all non-innodb tables. 199 | 200 | .. option:: --chunk-filesize -F 201 | 202 | Split tables into chunks of this output file size. This value is in MB 203 | 204 | .. option:: --success-on-1146 205 | 206 | Not increment error count and Warning instead of Critical in case of table doesn't exist 207 | 208 | .. option:: --use-savepoints 209 | 210 | Use savepoints to reduce metadata locking issues, needs SUPER privilege 211 | 212 | .. option:: --complete-insert 213 | 214 | Use complete INSERT statements that include column names. 215 | -------------------------------------------------------------------------------- /cmake/modules/CppcheckTargets.cmake: -------------------------------------------------------------------------------- 1 | # - Run cppcheck on c++ source files as a custom target and a test 2 | # 3 | # include(CppcheckTargets) 4 | # add_cppcheck( [UNUSED_FUNCTIONS] [STYLE] [POSSIBLE_ERROR] [FAIL_ON_WARNINGS]) - 5 | # Create a target to check a target's sources with cppcheck and the indicated options 6 | # add_cppcheck_sources( [UNUSED_FUNCTIONS] [STYLE] [POSSIBLE_ERROR] [FAIL_ON_WARNINGS]) - 7 | # Create a target to check standalone sources with cppcheck and the indicated options 8 | # 9 | # Requires these CMake modules: 10 | # Findcppcheck 11 | # 12 | # Requires CMake 2.6 or newer (uses the 'function' command) 13 | # 14 | # Original Author: 15 | # 2009-2010 Ryan Pavlik 16 | # http://academic.cleardefinition.com 17 | # Iowa State University HCI Graduate Program/VRAC 18 | # 19 | # Copyright Iowa State University 2009-2010. 20 | # Distributed under the Boost Software License, Version 1.0. 21 | # (See accompanying file LICENSE_1_0.txt or copy at 22 | # http://www.boost.org/LICENSE_1_0.txt) 23 | 24 | if(__add_cppcheck) 25 | return() 26 | endif() 27 | set(__add_cppcheck YES) 28 | 29 | if(NOT CPPCHECK_FOUND) 30 | find_package(cppcheck QUIET) 31 | endif() 32 | 33 | if(CPPCHECK_FOUND) 34 | if(NOT TARGET all_cppcheck) 35 | add_custom_target(all_cppcheck) 36 | set_target_properties(all_cppcheck PROPERTIES EXCLUDE_FROM_ALL TRUE) 37 | endif() 38 | endif() 39 | 40 | function(add_cppcheck_sources _targetname) 41 | if(CPPCHECK_FOUND) 42 | set(_cppcheck_args) 43 | set(_input ${ARGN}) 44 | list(FIND _input UNUSED_FUNCTIONS _unused_func) 45 | if("${_unused_func}" GREATER "-1") 46 | list(APPEND _cppcheck_args ${CPPCHECK_UNUSEDFUNC_ARG}) 47 | list(REMOVE_AT _input ${_unused_func}) 48 | endif() 49 | 50 | list(FIND _input STYLE _style) 51 | if("${_style}" GREATER "-1") 52 | list(APPEND _cppcheck_args ${CPPCHECK_STYLE_ARG}) 53 | list(REMOVE_AT _input ${_style}) 54 | endif() 55 | 56 | list(FIND _input POSSIBLE_ERROR _poss_err) 57 | if("${_poss_err}" GREATER "-1") 58 | list(APPEND _cppcheck_args ${CPPCHECK_POSSIBLEERROR_ARG}) 59 | list(REMOVE_AT _input ${_poss_err}) 60 | endif() 61 | 62 | list(FIND _input FAIL_ON_WARNINGS _fail_on_warn) 63 | if("${_fail_on_warn}" GREATER "-1") 64 | list(APPEND 65 | CPPCHECK_FAIL_REGULAR_EXPRESSION 66 | ${CPPCHECK_WARN_REGULAR_EXPRESSION}) 67 | list(REMOVE_AT _input ${_fail_on_warn}) 68 | endif() 69 | 70 | set(_files) 71 | foreach(_source ${_input}) 72 | get_source_file_property(_cppcheck_loc "${_source}" LOCATION) 73 | if(_cppcheck_loc) 74 | # This file has a source file property, carry on. 75 | get_source_file_property(_cppcheck_lang "${_source}" LANGUAGE) 76 | if("${_cppcheck_lang}" MATCHES "C") 77 | list(APPEND _files "${_cppcheck_loc}") 78 | endif() 79 | else() 80 | # This file doesn't have source file properties - figure it out. 81 | get_filename_component(_cppcheck_loc "${_source}" ABSOLUTE) 82 | if(EXISTS "${_cppcheck_loc}") 83 | list(APPEND _files "${_cppcheck_loc}") 84 | else() 85 | message(FATAL_ERROR 86 | "Adding CPPCHECK for file target ${_targetname}: " 87 | "File ${_source} does not exist or needs a corrected path location " 88 | "since we think its absolute path is ${_cppcheck_loc}") 89 | endif() 90 | endif() 91 | endforeach() 92 | 93 | if("1.${CMAKE_VERSION}" VERSION_LESS "1.2.8.0") 94 | # Older than CMake 2.8.0 95 | add_test(${_targetname}_cppcheck_test 96 | "${CPPCHECK_EXECUTABLE}" 97 | ${CPPCHECK_TEMPLATE_ARG} 98 | ${_cppcheck_args} 99 | ${_files}) 100 | else() 101 | # CMake 2.8.0 and newer 102 | add_test(NAME 103 | ${_targetname}_cppcheck_test 104 | COMMAND 105 | "${CPPCHECK_EXECUTABLE}" 106 | ${CPPCHECK_TEMPLATE_ARG} 107 | ${_cppcheck_args} 108 | ${_files}) 109 | endif() 110 | 111 | set_tests_properties(${_targetname}_cppcheck_test 112 | PROPERTIES 113 | FAIL_REGULAR_EXPRESSION 114 | "${CPPCHECK_FAIL_REGULAR_EXPRESSION}") 115 | 116 | add_custom_command(TARGET 117 | all_cppcheck 118 | PRE_BUILD 119 | COMMAND 120 | ${CPPCHECK_EXECUTABLE} 121 | ${CPPCHECK_QUIET_ARG} 122 | ${CPPCHECK_TEMPLATE_ARG} 123 | ${_cppcheck_args} 124 | ${_files} 125 | WORKING_DIRECTORY 126 | "${CMAKE_CURRENT_SOURCE_DIR}" 127 | COMMENT 128 | "${_targetname}_cppcheck: Running cppcheck on target ${_targetname}..." 129 | VERBATIM) 130 | endif() 131 | endfunction() 132 | 133 | function(add_cppcheck _name) 134 | if(NOT TARGET ${_name}) 135 | message(FATAL_ERROR 136 | "add_cppcheck given a target name that does not exist: '${_name}' !") 137 | endif() 138 | if(CPPCHECK_FOUND) 139 | set(_cppcheck_args) 140 | 141 | list(FIND ARGN UNUSED_FUNCTIONS _unused_func) 142 | if("${_unused_func}" GREATER "-1") 143 | list(APPEND _cppcheck_args ${CPPCHECK_UNUSEDFUNC_ARG}) 144 | endif() 145 | 146 | list(FIND ARGN STYLE _style) 147 | if("${_style}" GREATER "-1") 148 | list(APPEND _cppcheck_args ${CPPCHECK_STYLE_ARG}) 149 | endif() 150 | 151 | list(FIND ARGN POSSIBLE_ERROR _poss_err) 152 | if("${_poss_err}" GREATER "-1") 153 | list(APPEND _cppcheck_args ${CPPCHECK_POSSIBLEERROR_ARG}) 154 | endif() 155 | 156 | list(FIND _input FAIL_ON_WARNINGS _fail_on_warn) 157 | if("${_fail_on_warn}" GREATER "-1") 158 | list(APPEND 159 | CPPCHECK_FAIL_REGULAR_EXPRESSION 160 | ${CPPCHECK_WARN_REGULAR_EXPRESSION}) 161 | list(REMOVE_AT _input ${_unused_func}) 162 | endif() 163 | 164 | get_target_property(_cppcheck_sources "${_name}" SOURCES) 165 | set(_files) 166 | foreach(_source ${_cppcheck_sources}) 167 | get_source_file_property(_cppcheck_lang "${_source}" LANGUAGE) 168 | get_source_file_property(_cppcheck_loc "${_source}" LOCATION) 169 | if("${_cppcheck_lang}" MATCHES "C") 170 | list(APPEND _files "${_cppcheck_loc}") 171 | endif() 172 | endforeach() 173 | 174 | if("1.${CMAKE_VERSION}" VERSION_LESS "1.2.8.0") 175 | # Older than CMake 2.8.0 176 | add_test(${_name}_cppcheck_test 177 | "${CPPCHECK_EXECUTABLE}" 178 | ${CPPCHECK_TEMPLATE_ARG} 179 | ${_cppcheck_args} 180 | ${_files}) 181 | else() 182 | # CMake 2.8.0 and newer 183 | add_test(NAME 184 | ${_name}_cppcheck_test 185 | COMMAND 186 | "${CPPCHECK_EXECUTABLE}" 187 | ${CPPCHECK_TEMPLATE_ARG} 188 | ${_cppcheck_args} 189 | ${_files}) 190 | endif() 191 | 192 | set_tests_properties(${_name}_cppcheck_test 193 | PROPERTIES 194 | FAIL_REGULAR_EXPRESSION 195 | "${CPPCHECK_FAIL_REGULAR_EXPRESSION}") 196 | 197 | add_custom_command(TARGET 198 | all_cppcheck 199 | PRE_BUILD 200 | COMMAND 201 | ${CPPCHECK_EXECUTABLE} 202 | ${CPPCHECK_QUIET_ARG} 203 | ${CPPCHECK_TEMPLATE_ARG} 204 | "--enable=style,information,unusedFunction" 205 | ${_cppcheck_args} 206 | ${_files} 207 | WORKING_DIRECTORY 208 | "${CMAKE_CURRENT_SOURCE_DIR}" 209 | COMMENT 210 | "${_name}_cppcheck: Running cppcheck on target ${_name}..." 211 | VERBATIM) 212 | endif() 213 | 214 | endfunction() 215 | -------------------------------------------------------------------------------- /docs/_build/conf.py.in: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # MySQL Data Dumper documentation build configuration file, created by 4 | # sphinx-quickstart on Tue Apr 26 11:44:25 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.todo'] 29 | 30 | # Add any paths that contain templates here, relative to this directory. 31 | templates_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_templates'] 32 | 33 | # The suffix of source filenames. 34 | source_suffix = '.rst' 35 | 36 | # The encoding of source files. 37 | #source_encoding = 'utf-8-sig' 38 | 39 | # The master toctree document. 40 | master_doc = 'index' 41 | 42 | # General information about the project. 43 | project = u'@PROJECT_NAME@' 44 | copyright = u'2011, Andrew Hutchings' 45 | 46 | # The version info for the project you're documenting, acts as replacement for 47 | # |version| and |release|, also used in various other places throughout the 48 | # built documents. 49 | # 50 | # The short X.Y version. 51 | version = '@VERSION@' 52 | # The full version, including alpha/beta/rc tags. 53 | release = '@VERSION@' 54 | 55 | # The language for content autogenerated by Sphinx. Refer to documentation 56 | # for a list of supported languages. 57 | #language = None 58 | 59 | # There are two options for replacing |today|: either, you set today to some 60 | # non-false value, then it is used: 61 | #today = '' 62 | # Else, today_fmt is used as the format for a strftime call. 63 | #today_fmt = '%B %d, %Y' 64 | 65 | # List of patterns, relative to source directory, that match files and 66 | # directories to ignore when looking for source files. 67 | exclude_patterns = ['_build'] 68 | 69 | # The reST default role (used for this markup: `text`) to use for all documents. 70 | #default_role = None 71 | 72 | # If true, '()' will be appended to :func: etc. cross-reference text. 73 | #add_function_parentheses = True 74 | 75 | # If true, the current module name will be prepended to all description 76 | # unit titles (such as .. function::). 77 | #add_module_names = True 78 | 79 | # If true, sectionauthor and moduleauthor directives will be shown in the 80 | # output. They are ignored by default. 81 | #show_authors = False 82 | 83 | # The name of the Pygments (syntax highlighting) style to use. 84 | pygments_style = 'sphinx' 85 | 86 | # A list of ignored prefixes for module index sorting. 87 | #modindex_common_prefix = [] 88 | 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | html_theme = 'default' 95 | 96 | # Theme options are theme-specific and customize the look and feel of a theme 97 | # further. For a list of options available for each theme, see the 98 | # documentation. 99 | #html_theme_options = {} 100 | 101 | # Add any paths that contain custom themes here, relative to this directory. 102 | #html_theme_path = [] 103 | 104 | # The name for this set of Sphinx documents. If None, it defaults to 105 | # " v documentation". 106 | #html_title = None 107 | 108 | # A shorter title for the navigation bar. Default is the same as html_title. 109 | #html_short_title = None 110 | 111 | # The name of an image file (relative to this directory) to place at the top 112 | # of the sidebar. 113 | #html_logo = None 114 | 115 | # The name of an image file (within the static path) to use as favicon of the 116 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 117 | # pixels large. 118 | #html_favicon = None 119 | 120 | # Add any paths that contain custom static files (such as style sheets) here, 121 | # relative to this directory. They are copied after the builtin static files, 122 | # so a file named "default.css" will overwrite the builtin "default.css". 123 | html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_static'] 124 | 125 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 126 | # using the given strftime format. 127 | #html_last_updated_fmt = '%b %d, %Y' 128 | 129 | # If true, SmartyPants will be used to convert quotes and dashes to 130 | # typographically correct entities. 131 | #html_use_smartypants = True 132 | 133 | # Custom sidebar templates, maps document names to template names. 134 | #html_sidebars = {} 135 | 136 | # Additional templates that should be rendered to pages, maps page names to 137 | # template names. 138 | #html_additional_pages = {} 139 | 140 | # If false, no module index is generated. 141 | #html_domain_indices = True 142 | 143 | # If false, no index is generated. 144 | #html_use_index = True 145 | 146 | # If true, the index is split into individual pages for each letter. 147 | #html_split_index = False 148 | 149 | # If true, links to the reST sources are added to the pages. 150 | #html_show_sourcelink = True 151 | 152 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 153 | #html_show_sphinx = True 154 | 155 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 156 | #html_show_copyright = True 157 | 158 | # If true, an OpenSearch description file will be output, and all pages will 159 | # contain a tag referring to it. The value of this option must be the 160 | # base URL from which the finished HTML is served. 161 | #html_use_opensearch = '' 162 | 163 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 164 | #html_file_suffix = None 165 | 166 | # Output file base name for HTML help builder. 167 | htmlhelp_basename = 'MySQLDataDumperdoc' 168 | 169 | 170 | # -- Options for LaTeX output -------------------------------------------------- 171 | 172 | # The paper size ('letter' or 'a4'). 173 | #latex_paper_size = 'letter' 174 | 175 | # The font size ('10pt', '11pt' or '12pt'). 176 | #latex_font_size = '10pt' 177 | 178 | # Grouping the document tree into LaTeX files. List of tuples 179 | # (source start file, target name, title, author, documentclass [howto/manual]). 180 | latex_documents = [ 181 | ('index', 'MySQLDataDumper.tex', u'@PROJECT_NAME@ Documentation', 182 | u'Andrew Hutchings', 'manual'), 183 | ] 184 | 185 | # The name of an image file (relative to this directory) to place at the top of 186 | # the title page. 187 | #latex_logo = None 188 | 189 | # For "manual" documents, if this is true, then toplevel headings are parts, 190 | # not chapters. 191 | #latex_use_parts = False 192 | 193 | # If true, show page references after internal links. 194 | #latex_show_pagerefs = False 195 | 196 | # If true, show URL addresses after external links. 197 | #latex_show_urls = False 198 | 199 | # Additional stuff for the LaTeX preamble. 200 | #latex_preamble = '' 201 | 202 | # Documents to append as an appendix to all manuals. 203 | #latex_appendices = [] 204 | 205 | # If false, no module index is generated. 206 | #latex_domain_indices = True 207 | 208 | 209 | # -- Options for manual page output -------------------------------------------- 210 | 211 | # One entry per manual page. List of tuples 212 | # (source start file, name, description, authors, manual section). 213 | man_pages = [ 214 | ('mydumper_usage', 'mydumper', u'@PROGRAM_DESC@', 215 | [u'Andrew Hutchings'], 1), 216 | ('myloader_usage', 'myloader', u'@PROGRAM_DESC@', 217 | [u'Andrew Hutchings'], 1) 218 | ] 219 | -------------------------------------------------------------------------------- /binlog.c: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Domas Mituzas, Facebook ( domas at fb dot com ) 16 | Mark Leith, Oracle Corporation (mark dot leith at oracle dot com) 17 | Andrew Hutchings, SkySQL (andrew at skysql dot com) 18 | 19 | */ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "mydumper.h" 31 | #include "binlog.h" 32 | 33 | #define BINLOG_MAGIC "\xfe\x62\x69\x6e" 34 | 35 | #define EVENT_HEADER_LENGTH 19 36 | #define EVENT_ROTATE_FIXED_LENGTH 8 37 | 38 | enum event_postions { 39 | EVENT_TIMESTAMP_POSITION= 0, 40 | EVENT_TYPE_POSITION= 4, 41 | EVENT_SERVERID_POSITION= 5, 42 | EVENT_LENGTH_POSITION= 9, 43 | EVENT_NEXT_POSITION= 13, 44 | EVENT_FLAGS_POSITION= 17, 45 | EVENT_EXTRA_FLAGS_POSITION= 19 // currently unused in v4 binlogs, but a good marker for end of header 46 | }; 47 | 48 | enum event_type { 49 | ROTATE_EVENT= 4, 50 | FORMAT_DESCRIPTION_EVENT= 15, 51 | EVENT_TOO_SHORT= 254 // arbitrary high number, in 5.1 the max event type number is 27 so this should be fine for a while 52 | }; 53 | 54 | extern int compress_output; 55 | extern gboolean daemon_mode; 56 | extern gboolean shutdown_triggered; 57 | 58 | FILE *new_binlog_file(char *binlog_file, const char *binlog_dir); 59 | void close_binlog_file(FILE *outfile); 60 | char *rotate_file_name(const char *buf); 61 | 62 | void get_binlogs(MYSQL *conn, struct configuration *conf) { 63 | // TODO: find logs we already have, use start position based on position of last log. 64 | MYSQL_RES *result; 65 | MYSQL_ROW row; 66 | char* last_filename = NULL; 67 | guint64 last_position; 68 | 69 | // Only snapshot dump the binlogs once in daemon mode 70 | static gboolean got_binlogs= FALSE; 71 | if (got_binlogs) 72 | return; 73 | else 74 | got_binlogs= TRUE; 75 | 76 | if (mysql_query(conn, "SHOW MASTER STATUS")) { 77 | g_critical("Error: Could not execute query: %s", mysql_error(conn)); 78 | return; 79 | } 80 | 81 | result = mysql_store_result(conn); 82 | if ((row = mysql_fetch_row(result))) { 83 | last_filename= g_strdup(row[0]); 84 | last_position= strtoll(row[1], NULL, 10); 85 | } else { 86 | g_critical("Error: Could not obtain binary log stop position"); 87 | if (last_filename != NULL) 88 | g_free(last_filename); 89 | return; 90 | } 91 | mysql_free_result(result); 92 | 93 | if (mysql_query(conn, "SHOW BINARY LOGS")) { 94 | g_critical("Error: Could not execute query: %s", mysql_error(conn)); 95 | if (last_filename != NULL) 96 | g_free(last_filename); 97 | return; 98 | } 99 | 100 | 101 | result = mysql_store_result(conn); 102 | while ((row = mysql_fetch_row(result))) { 103 | struct job *j = g_new0(struct job,1); 104 | struct binlog_job *bj = g_new0(struct binlog_job,1); 105 | j->job_data=(void*) bj; 106 | bj->filename=g_strdup(row[0]); 107 | bj->start_position=4; 108 | bj->stop_position= (!strcasecmp(row[0], last_filename)) ? last_position : 0; 109 | j->conf=conf; 110 | j->type=JOB_BINLOG; 111 | g_async_queue_push(conf->queue,j); 112 | } 113 | mysql_free_result(result); 114 | if (last_filename != NULL) 115 | g_free(last_filename); 116 | } 117 | 118 | void get_binlog_file(MYSQL *conn, char *binlog_file, const char *binlog_directory, guint64 start_position, guint64 stop_position, gboolean continuous) { 119 | // set serverID = max serverID - threadID to try an eliminate conflicts, 120 | // 0 is bad because mysqld will disconnect at the end of the last log 121 | // dupes aren't too bad since it is up to the client to check for them 122 | uchar buf[128]; 123 | // We need to read the raw network packets 124 | NET* net; 125 | net= &conn->net; 126 | unsigned long len; 127 | FILE* outfile; 128 | guint32 event_type; 129 | gboolean read_error= FALSE; 130 | gboolean read_end= FALSE; 131 | gboolean rotated= FALSE; 132 | guint32 server_id= G_MAXUINT32 - mysql_thread_id(conn); 133 | guint64 pos_counter= 0; 134 | 135 | int4store(buf, (guint32)start_position); 136 | // Binlog flags (2 byte int) 137 | int2store(buf + 4, 0); 138 | // ServerID 139 | int4store(buf + 6, server_id); 140 | memcpy(buf + 10, binlog_file, strlen(binlog_file)); 141 | #if MYSQL_VERSION_ID < 50100 142 | if (simple_command(conn, COM_BINLOG_DUMP, (const char *)buf, 143 | #else 144 | if (simple_command(conn, COM_BINLOG_DUMP, buf, 145 | #endif 146 | strlen(binlog_file) + 10, 1)) { 147 | g_critical("Error: binlog: Critical error whilst requesting binary log"); 148 | } 149 | 150 | while(1) { 151 | outfile= new_binlog_file(binlog_file, binlog_directory); 152 | if (outfile == NULL) { 153 | g_critical("Error: binlog: Could not create binlog file '%s', %d", binlog_file, errno); 154 | return; 155 | } 156 | 157 | write_binlog(outfile, BINLOG_MAGIC, 4); 158 | while(1) { 159 | len = 0; 160 | if (net->vio != 0) len=my_net_read(net); 161 | if ((len == 0) || (len == ~(unsigned long) 0)) { 162 | // Net timeout (set to 1 second) 163 | if (mysql_errno(conn) == ER_NET_READ_INTERRUPTED) { 164 | if (shutdown_triggered) { 165 | close_binlog_file(outfile); 166 | return; 167 | } else { 168 | continue; 169 | } 170 | // A real error 171 | } else { 172 | g_critical("Error: binlog: Network packet read error getting binlog file: %s", binlog_file); 173 | close_binlog_file(outfile); 174 | return; 175 | } 176 | } 177 | if (len < 8 && net->read_pos[0]) { 178 | // end of data 179 | break; 180 | } 181 | pos_counter += len; 182 | event_type= get_event((const char*)net->read_pos + 1, len -1); 183 | switch (event_type) { 184 | case EVENT_TOO_SHORT: 185 | g_critical("Error: binlog: Event too short in binlog file: %s", binlog_file); 186 | read_error= TRUE; 187 | break; 188 | case ROTATE_EVENT: 189 | if (rotated) { 190 | read_end= TRUE; 191 | } else { 192 | len= 1; 193 | rotated= TRUE; 194 | } 195 | break; 196 | default: 197 | // if we get this far this is a normal event to record 198 | break; 199 | } 200 | if (read_error) break; 201 | write_binlog(outfile, (const char*)net->read_pos + 1, len - 1); 202 | if (read_end) { 203 | if (!continuous) { 204 | break; 205 | } else { 206 | g_free(binlog_file); 207 | binlog_file= rotate_file_name((const char*)net->read_pos + 1); 208 | break; 209 | } 210 | } 211 | // stop if we are at requested end of last log 212 | if ((stop_position > 0) && (pos_counter >= stop_position)) break; 213 | } 214 | close_binlog_file(outfile); 215 | if ((!continuous) || (!read_end)) break; 216 | 217 | if (continuous && read_end) { 218 | read_end= FALSE; 219 | rotated= FALSE; 220 | } 221 | } 222 | } 223 | 224 | char *rotate_file_name(const char *buf) { 225 | guint32 event_length= 0; 226 | 227 | // event length is 4 bytes at position 9 228 | event_length= uint4korr(&buf[EVENT_LENGTH_POSITION]); 229 | // event length includes the header, plus a rotate event has a fixed 8byte part we don't need 230 | event_length= event_length - EVENT_HEADER_LENGTH - EVENT_ROTATE_FIXED_LENGTH; 231 | 232 | return g_strndup(&buf[EVENT_HEADER_LENGTH + EVENT_ROTATE_FIXED_LENGTH], event_length); 233 | } 234 | 235 | FILE *new_binlog_file(char *binlog_file, const char *binlog_dir) { 236 | FILE *outfile; 237 | char* filename; 238 | 239 | if (!compress_output) { 240 | filename= g_strdup_printf("%s/%s", binlog_dir, binlog_file); 241 | outfile= g_fopen(filename, "w"); 242 | } else { 243 | filename= g_strdup_printf("%s/%s.gz", binlog_dir, binlog_file); 244 | outfile= (void*) gzopen(filename, "w"); 245 | } 246 | g_free(filename); 247 | 248 | return outfile; 249 | } 250 | 251 | void close_binlog_file(FILE *outfile) { 252 | if (!compress_output) 253 | fclose(outfile); 254 | else 255 | gzclose((gzFile) outfile); 256 | } 257 | 258 | unsigned int get_event(const char *buf, unsigned int len) { 259 | if (len < EVENT_TYPE_POSITION) 260 | return EVENT_TOO_SHORT; 261 | return buf[EVENT_TYPE_POSITION]; 262 | 263 | // TODO: Would be good if we can check for valid event type, unfortunately this check can change from version to version 264 | } 265 | 266 | void write_binlog(FILE* file, const char* data, guint64 len) { 267 | int err; 268 | 269 | if (len > 0) { 270 | int write_result; 271 | 272 | if (!compress_output) 273 | write_result= write(fileno(file), data, len); 274 | else 275 | write_result= gzwrite((gzFile)file, data, len); 276 | 277 | if (write_result <= 0) { 278 | if (!compress_output) 279 | g_critical("Error: binlog: Error writing binary log: %s", strerror(errno)); 280 | else 281 | g_critical("Error: binlog: Error writing compressed binary log: %s", gzerror((gzFile)file, &err)); 282 | } 283 | } 284 | } 285 | -------------------------------------------------------------------------------- /myloader.c: -------------------------------------------------------------------------------- 1 | /* 2 | This program is free software: you can redistribute it and/or modify 3 | it under the terms of the GNU General Public License as published by 4 | the Free Software Foundation, either version 3 of the License, or 5 | (at your option) any later version. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program. If not, see . 14 | 15 | Authors: Andrew Hutchings, SkySQL (andrew at skysql dot com) 16 | */ 17 | 18 | #define _LARGEFILE64_SOURCE 19 | #define _FILE_OFFSET_BITS 64 20 | 21 | #include 22 | 23 | #if defined MARIADB_CLIENT_VERSION_STR && !defined MYSQL_SERVER_VERSION 24 | #define MYSQL_SERVER_VERSION MARIADB_CLIENT_VERSION_STR 25 | #endif 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include "common.h" 37 | #include "myloader.h" 38 | #include "connection.h" 39 | #include "config.h" 40 | #include "getPassword.h" 41 | 42 | guint commit_count= 1000; 43 | gchar *directory= NULL; 44 | gboolean overwrite_tables= FALSE; 45 | gboolean enable_binlog= FALSE; 46 | gchar *source_db= NULL; 47 | static GMutex *init_mutex= NULL; 48 | 49 | guint errors= 0; 50 | 51 | gboolean read_data(FILE *file, gboolean is_compressed, GString *data, gboolean *eof); 52 | void restore_data(MYSQL *conn, char *database, char *table, const char *filename, gboolean is_schema, gboolean need_use); 53 | void *process_queue(struct thread_data *td); 54 | void add_table(const gchar* filename, struct configuration *conf); 55 | void add_schema(const gchar* filename, MYSQL *conn); 56 | void restore_databases(struct configuration *conf, MYSQL *conn); 57 | void restore_schema_view(MYSQL *conn); 58 | void restore_schema_triggers(MYSQL *conn); 59 | void restore_schema_post(MYSQL *conn); 60 | void no_log(const gchar *log_domain, GLogLevelFlags log_level, const gchar *message, gpointer user_data); 61 | void set_verbose(guint verbosity); 62 | void create_database(MYSQL *conn, gchar *database); 63 | 64 | static GOptionEntry entries[] = 65 | { 66 | { "directory", 'd', 0, G_OPTION_ARG_STRING, &directory, "Directory of the dump to import", NULL }, 67 | { "queries-per-transaction", 'q', 0, G_OPTION_ARG_INT, &commit_count, "Number of queries per transaction, default 1000", NULL }, 68 | { "overwrite-tables", 'o', 0, G_OPTION_ARG_NONE, &overwrite_tables, "Drop tables if they already exist", NULL }, 69 | { "database", 'B', 0, G_OPTION_ARG_STRING, &db, "An alternative database to restore into", NULL }, 70 | { "source-db", 's', 0, G_OPTION_ARG_STRING, &source_db, "Database to restore", NULL }, 71 | { "enable-binlog", 'e', 0, G_OPTION_ARG_NONE, &enable_binlog, "Enable binary logging of the restore data", NULL }, 72 | { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL } 73 | }; 74 | 75 | void no_log(const gchar *log_domain, GLogLevelFlags log_level, const gchar *message, gpointer user_data) { 76 | (void) log_domain; 77 | (void) log_level; 78 | (void) message; 79 | (void) user_data; 80 | } 81 | 82 | void set_verbose(guint verbosity) { 83 | switch (verbosity) { 84 | case 0: 85 | g_log_set_handler(NULL, (GLogLevelFlags)(G_LOG_LEVEL_MASK), no_log, NULL); 86 | break; 87 | case 1: 88 | g_log_set_handler(NULL, (GLogLevelFlags)(G_LOG_LEVEL_WARNING | G_LOG_LEVEL_MESSAGE), no_log, NULL); 89 | break; 90 | case 2: 91 | g_log_set_handler(NULL, (GLogLevelFlags)(G_LOG_LEVEL_MESSAGE), no_log, NULL); 92 | break; 93 | default: 94 | break; 95 | } 96 | } 97 | 98 | int main(int argc, char *argv[]) { 99 | struct configuration conf= { NULL, NULL, NULL, 0 }; 100 | 101 | GError *error= NULL; 102 | GOptionContext *context; 103 | 104 | g_thread_init(NULL); 105 | 106 | init_mutex= g_mutex_new(); 107 | 108 | if(db == NULL && source_db != NULL){ 109 | db = g_strdup(source_db); 110 | } 111 | 112 | context= g_option_context_new("multi-threaded MySQL loader"); 113 | GOptionGroup *main_group= g_option_group_new("main", "Main Options", "Main Options", NULL, NULL); 114 | g_option_group_add_entries(main_group, entries); 115 | g_option_group_add_entries(main_group, common_entries); 116 | g_option_context_set_main_group(context, main_group); 117 | if (!g_option_context_parse(context, &argc, &argv, &error)) { 118 | g_print("option parsing failed: %s, try --help\n", error->message); 119 | exit(EXIT_FAILURE); 120 | } 121 | g_option_context_free(context); 122 | 123 | //prompt for password if it's NULL 124 | if ( sizeof(password) == 0 || ( password == NULL && askPassword ) ){ 125 | password = passwordPrompt(); 126 | } 127 | 128 | if (program_version) { 129 | g_print("myloader %s, built against MySQL %s\n", VERSION, MYSQL_VERSION_STR); 130 | exit(EXIT_SUCCESS); 131 | } 132 | 133 | set_verbose(verbose); 134 | 135 | if (!directory) { 136 | g_critical("a directory needs to be specified, see --help\n"); 137 | exit(EXIT_FAILURE); 138 | } else { 139 | char *p= g_strdup_printf("%s/metadata", directory); 140 | if (!g_file_test(p, G_FILE_TEST_EXISTS)) { 141 | g_critical("the specified directory is not a mydumper backup\n"); 142 | exit(EXIT_FAILURE); 143 | } 144 | } 145 | MYSQL *conn; 146 | conn= mysql_init(NULL); 147 | 148 | configure_connection(conn,"myloader"); 149 | if (!mysql_real_connect(conn, hostname, username, password, NULL, port, socket_path, 0)) { 150 | g_critical("Error connection to database: %s", mysql_error(conn)); 151 | exit(EXIT_FAILURE); 152 | } 153 | 154 | if (mysql_query(conn, "SET SESSION wait_timeout = 2147483")){ 155 | g_warning("Failed to increase wait_timeout: %s", mysql_error(conn)); 156 | } 157 | 158 | if (!enable_binlog) 159 | mysql_query(conn, "SET SQL_LOG_BIN=0"); 160 | 161 | mysql_query(conn, "/*!40014 SET FOREIGN_KEY_CHECKS=0*/"); 162 | conf.queue= g_async_queue_new(); 163 | conf.ready= g_async_queue_new(); 164 | 165 | guint n; 166 | GThread **threads= g_new(GThread*, num_threads); 167 | struct thread_data *td= g_new(struct thread_data, num_threads); 168 | for (n= 0; n < num_threads; n++) { 169 | td[n].conf= &conf; 170 | td[n].thread_id= n+1; 171 | threads[n]= g_thread_create((GThreadFunc)process_queue, &td[n], TRUE, NULL); 172 | g_async_queue_pop(conf.ready); 173 | } 174 | g_async_queue_unref(conf.ready); 175 | 176 | g_message("%d threads created", num_threads); 177 | 178 | restore_databases(&conf, conn); 179 | 180 | for (n= 0; n < num_threads; n++) { 181 | struct job *j= g_new0(struct job, 1); 182 | j->type = JOB_SHUTDOWN; 183 | g_async_queue_push(conf.queue, j); 184 | } 185 | 186 | for (n= 0; n < num_threads; n++) { 187 | g_thread_join(threads[n]); 188 | } 189 | 190 | restore_schema_post(conn); 191 | 192 | restore_schema_view(conn); 193 | 194 | restore_schema_triggers(conn); 195 | 196 | g_async_queue_unref(conf.queue); 197 | mysql_close(conn); 198 | mysql_thread_end(); 199 | mysql_library_end(); 200 | g_free(directory); 201 | g_free(td); 202 | g_free(threads); 203 | 204 | return errors ? EXIT_FAILURE : EXIT_SUCCESS; 205 | } 206 | 207 | void restore_databases(struct configuration *conf, MYSQL *conn) { 208 | GError *error= NULL; 209 | GDir* dir= g_dir_open(directory, 0, &error); 210 | 211 | if (error) { 212 | g_critical("cannot open directory %s, %s\n", directory, error->message); 213 | errors++; 214 | return; 215 | } 216 | 217 | const gchar* filename= NULL; 218 | 219 | while((filename= g_dir_read_name(dir))) { 220 | if (!source_db || g_str_has_prefix(filename, g_strdup_printf("%s.", source_db))){ 221 | if (g_strrstr(filename, "-schema.sql")) { 222 | add_schema(filename, conn); 223 | } 224 | } 225 | } 226 | 227 | g_dir_rewind(dir); 228 | 229 | while((filename= g_dir_read_name(dir))) { 230 | if (!source_db || g_str_has_prefix(filename, g_strdup_printf("%s.", source_db))){ 231 | if (!g_strrstr(filename, "-schema.sql") 232 | && !g_strrstr(filename, "-schema-view.sql") 233 | && !g_strrstr(filename, "-schema-triggers.sql") 234 | && !g_strrstr(filename, "-schema-post.sql") 235 | && !g_strrstr(filename, "-schema-create.sql") 236 | && g_strrstr(filename, ".sql")) { 237 | add_table(filename, conf); 238 | } 239 | } 240 | } 241 | 242 | g_dir_close(dir); 243 | } 244 | 245 | 246 | void restore_schema_view(MYSQL *conn){ 247 | GError *error= NULL; 248 | GDir* dir= g_dir_open(directory, 0, &error); 249 | 250 | if (error) { 251 | g_critical("cannot open directory %s, %s\n", directory, error->message); 252 | errors++; 253 | return; 254 | } 255 | 256 | const gchar* filename= NULL; 257 | 258 | while((filename= g_dir_read_name(dir))) { 259 | if (!source_db || g_str_has_prefix(filename, source_db)){ 260 | if (g_strrstr(filename, "-schema-view.sql")) { 261 | add_schema(filename, conn); 262 | } 263 | } 264 | } 265 | 266 | g_dir_close(dir); 267 | } 268 | 269 | void restore_schema_triggers(MYSQL *conn){ 270 | GError *error= NULL; 271 | GDir* dir= g_dir_open(directory, 0, &error); 272 | gchar** split_file= NULL; 273 | gchar* database=NULL; 274 | gchar** split_table= NULL; 275 | gchar* table= NULL; 276 | 277 | if (error) { 278 | g_critical("cannot open directory %s, %s\n", directory, error->message); 279 | errors++; 280 | return; 281 | } 282 | 283 | const gchar* filename= NULL; 284 | 285 | while((filename= g_dir_read_name(dir))) { 286 | if (!source_db || g_str_has_prefix(filename, source_db)){ 287 | if (g_strrstr(filename, "-schema-triggers.sql")) { 288 | split_file= g_strsplit(filename, ".", 0); 289 | database= split_file[0]; 290 | split_table= g_strsplit(split_file[1], "-schema", 0); 291 | table= split_table[0]; 292 | g_message("Restoring triggers for `%s`.`%s`", db ? db : database, table); 293 | restore_data(conn, database, table, filename, TRUE, TRUE); 294 | } 295 | } 296 | } 297 | 298 | g_strfreev(split_table); 299 | g_strfreev(split_file); 300 | g_dir_close(dir); 301 | } 302 | 303 | void restore_schema_post(MYSQL *conn){ 304 | GError *error= NULL; 305 | GDir* dir= g_dir_open(directory, 0, &error); 306 | gchar** split_file= NULL; 307 | gchar* database=NULL; 308 | //gchar* table=NULL; 309 | 310 | 311 | if (error) { 312 | g_critical("cannot open directory %s, %s\n", directory, error->message); 313 | errors++; 314 | return; 315 | } 316 | 317 | const gchar* filename= NULL; 318 | 319 | while((filename= g_dir_read_name(dir))) { 320 | if (!source_db || g_str_has_prefix(filename, source_db)){ 321 | if (g_strrstr(filename, "-schema-post.sql")) { 322 | split_file= g_strsplit(filename, "-schema-post.sql", 0); 323 | database= split_file[0]; 324 | //table= split_file[0]; //NULL 325 | g_message("Restoring routines and events for `%s`", db ? db : database); 326 | restore_data(conn, database, NULL, filename, TRUE, TRUE); 327 | } 328 | } 329 | } 330 | 331 | g_strfreev(split_file); 332 | g_dir_close(dir); 333 | } 334 | 335 | void create_database(MYSQL *conn, gchar *database){ 336 | 337 | gchar* query = NULL; 338 | 339 | if((db == NULL && source_db == NULL) || (db != NULL && source_db != NULL && !g_ascii_strcasecmp(db, source_db))){ 340 | const gchar* filename= g_strdup_printf("%s-schema-create.sql", db ? db : database); 341 | const gchar* filenamegz= g_strdup_printf("%s-schema-create.sql.gz", db ? db : database); 342 | const gchar* filepath= g_strdup_printf("%s/%s-schema-create.sql", directory, db ? db : database); 343 | const gchar* filepathgz= g_strdup_printf("%s/%s-schema-create.sql.gz", directory, db ? db : database); 344 | 345 | if (g_file_test (filepath, G_FILE_TEST_EXISTS)){ 346 | restore_data(conn, database, NULL, filename, TRUE, FALSE); 347 | }else if (g_file_test (filepathgz, G_FILE_TEST_EXISTS)){ 348 | restore_data(conn, database, NULL, filenamegz, TRUE, FALSE); 349 | }else{ 350 | query= g_strdup_printf("CREATE DATABASE `%s`", db ? db : database); 351 | mysql_query(conn, query); 352 | } 353 | }else{ 354 | query= g_strdup_printf("CREATE DATABASE `%s`", db ? db : database); 355 | mysql_query(conn, query); 356 | } 357 | 358 | g_free(query); 359 | return; 360 | } 361 | 362 | void add_schema(const gchar* filename, MYSQL *conn) { 363 | // 0 is database, 1 is table with -schema on the end 364 | gchar** split_file= g_strsplit(filename, ".", 0); 365 | gchar* database= split_file[0]; 366 | // Remove the -schema from the table name 367 | gchar** split_table= g_strsplit(split_file[1], "-schema", 0); 368 | gchar* table= split_table[0]; 369 | 370 | gchar* query= g_strdup_printf("SHOW CREATE DATABASE `%s`", db ? db : database); 371 | if (mysql_query(conn, query)) { 372 | g_message("Creating database `%s`", db ? db : database); 373 | create_database(conn, database); 374 | } else { 375 | MYSQL_RES *result= mysql_store_result(conn); 376 | // In drizzle the query succeeds with no rows 377 | my_ulonglong row_count= mysql_num_rows(result); 378 | mysql_free_result(result); 379 | if (row_count == 0) { 380 | create_database(conn, database); 381 | } 382 | } 383 | 384 | if (overwrite_tables) { 385 | g_message("Dropping table or view (if exists) `%s`.`%s`", db ? db : database, table); 386 | query= g_strdup_printf("DROP TABLE IF EXISTS `%s`.`%s`", db ? db : database, table); 387 | mysql_query(conn, query); 388 | query= g_strdup_printf("DROP VIEW IF EXISTS `%s`.`%s`", db ? db : database, table); 389 | mysql_query(conn, query); 390 | } 391 | 392 | g_free(query); 393 | 394 | g_message("Creating table `%s`.`%s`", db ? db : database, table); 395 | restore_data(conn, database, table, filename, TRUE, TRUE); 396 | g_strfreev(split_table); 397 | g_strfreev(split_file); 398 | return; 399 | } 400 | 401 | void add_table(const gchar* filename, struct configuration *conf) { 402 | struct job *j= g_new0(struct job, 1); 403 | struct restore_job *rj= g_new(struct restore_job, 1); 404 | j->job_data= (void*) rj; 405 | rj->filename= g_strdup(filename); 406 | j->type= JOB_RESTORE; 407 | gchar** split_file= g_strsplit(filename, ".", 0); 408 | rj->database= g_strdup(split_file[0]); 409 | rj->table= g_strdup(split_file[1]); 410 | rj->part= g_ascii_strtoull(split_file[2], NULL, 10); 411 | g_async_queue_push(conf->queue, j); 412 | return; 413 | } 414 | 415 | void *process_queue(struct thread_data *td) { 416 | struct configuration *conf= td->conf; 417 | g_mutex_lock(init_mutex); 418 | MYSQL *thrconn= mysql_init(NULL); 419 | g_mutex_unlock(init_mutex); 420 | 421 | configure_connection(thrconn,"myloader"); 422 | 423 | if (!mysql_real_connect(thrconn, hostname, username, password, NULL, port, socket_path, 0)) { 424 | g_critical("Failed to connect to MySQL server: %s", mysql_error(thrconn)); 425 | exit(EXIT_FAILURE); 426 | } 427 | 428 | if (mysql_query(thrconn, "SET SESSION wait_timeout = 2147483")){ 429 | g_warning("Failed to increase wait_timeout: %s", mysql_error(thrconn)); 430 | } 431 | 432 | if (!enable_binlog) 433 | mysql_query(thrconn, "SET SQL_LOG_BIN=0"); 434 | 435 | mysql_query(thrconn, "/*!40101 SET NAMES binary*/"); 436 | mysql_query(thrconn, "/*!40101 SET SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */"); 437 | mysql_query(thrconn, "/*!40014 SET UNIQUE_CHECKS=0 */"); 438 | mysql_query(thrconn, "SET autocommit=0"); 439 | 440 | g_async_queue_push(conf->ready, GINT_TO_POINTER(1)); 441 | 442 | struct job* job= NULL; 443 | struct restore_job* rj= NULL; 444 | for(;;) { 445 | job= (struct job*)g_async_queue_pop(conf->queue); 446 | 447 | switch (job->type) { 448 | case JOB_RESTORE: 449 | rj= (struct restore_job *)job->job_data; 450 | g_message("Thread %d restoring `%s`.`%s` part %d", td->thread_id, rj->database, rj->table, rj->part); 451 | restore_data(thrconn, rj->database, rj->table, rj->filename, FALSE, TRUE); 452 | if (rj->database) g_free(rj->database); 453 | if (rj->table) g_free(rj->table); 454 | if (rj->filename) g_free(rj->filename); 455 | g_free(rj); 456 | g_free(job); 457 | break; 458 | case JOB_SHUTDOWN: 459 | g_message("Thread %d shutting down", td->thread_id); 460 | if (thrconn) 461 | mysql_close(thrconn); 462 | g_free(job); 463 | mysql_thread_end(); 464 | return NULL; 465 | break; 466 | default: 467 | g_critical("Something very bad happened!"); 468 | exit(EXIT_FAILURE); 469 | } 470 | } 471 | if (thrconn) 472 | mysql_close(thrconn); 473 | mysql_thread_end(); 474 | return NULL; 475 | } 476 | 477 | void restore_data(MYSQL *conn, char *database, char *table, const char *filename, gboolean is_schema, gboolean need_use) { 478 | void *infile; 479 | gboolean is_compressed= FALSE; 480 | gboolean eof= FALSE; 481 | guint query_counter= 0; 482 | GString *data= g_string_sized_new(512); 483 | 484 | gchar* path= g_build_filename(directory, filename, NULL); 485 | 486 | if (!g_str_has_suffix(path, ".gz")) { 487 | infile= g_fopen(path, "r"); 488 | is_compressed= FALSE; 489 | } else { 490 | infile= (void*) gzopen(path, "r"); 491 | is_compressed= TRUE; 492 | } 493 | 494 | if (!infile) { 495 | g_critical("cannot open file %s (%d)", filename, errno); 496 | errors++; 497 | return; 498 | } 499 | 500 | 501 | if(need_use){ 502 | gchar *query= g_strdup_printf("USE `%s`", db ? db : database); 503 | 504 | if (mysql_query(conn, query)) { 505 | g_critical("Error switching to database %s whilst restoring table %s", db ? db : database, table); 506 | g_free(query); 507 | errors++; 508 | return; 509 | } 510 | 511 | g_free(query); 512 | } 513 | 514 | 515 | if (!is_schema) 516 | mysql_query(conn, "START TRANSACTION"); 517 | 518 | while (eof == FALSE) { 519 | if (read_data(infile, is_compressed, data, &eof)) { 520 | // Search for ; in last 5 chars of line 521 | if (g_strrstr(&data->str[data->len >= 5 ? data->len - 5 : 0], ";\n")) { 522 | if (mysql_real_query(conn, data->str, data->len)) { 523 | g_critical("Error restoring %s.%s from file %s: %s", db ? db : database, table, filename, mysql_error(conn)); 524 | errors++; 525 | return; 526 | } 527 | query_counter++; 528 | if (!is_schema &&(query_counter == commit_count)) { 529 | query_counter= 0; 530 | if (mysql_query(conn, "COMMIT")) { 531 | g_critical("Error committing data for %s.%s: %s", db ? db : database, table, mysql_error(conn)); 532 | errors++; 533 | return; 534 | } 535 | mysql_query(conn, "START TRANSACTION"); 536 | } 537 | 538 | g_string_set_size(data, 0); 539 | } 540 | } else { 541 | g_critical("error reading file %s (%d)", filename, errno); 542 | errors++; 543 | return; 544 | } 545 | } 546 | if (!is_schema && mysql_query(conn, "COMMIT")) { 547 | g_critical("Error committing data for %s.%s from file %s: %s", db ? db : database, table, filename, mysql_error(conn)); 548 | errors++; 549 | } 550 | g_string_free(data, TRUE); 551 | g_free(path); 552 | if (!is_compressed) { 553 | fclose(infile); 554 | } else { 555 | gzclose((gzFile)infile); 556 | } 557 | return; 558 | } 559 | 560 | gboolean read_data(FILE *file, gboolean is_compressed, GString *data, gboolean *eof) { 561 | char buffer[256]; 562 | 563 | do { 564 | if (!is_compressed) { 565 | if (fgets(buffer, 256, file) == NULL) { 566 | if (feof(file)) { 567 | *eof= TRUE; 568 | buffer[0]= '\0'; 569 | } else { 570 | return FALSE; 571 | } 572 | } 573 | } else { 574 | if (!gzgets((gzFile)file, buffer, 256)) { 575 | if (gzeof((gzFile)file)) { 576 | *eof= TRUE; 577 | buffer[0]= '\0'; 578 | } else { 579 | return FALSE; 580 | } 581 | } 582 | } 583 | g_string_append(data, buffer); 584 | } while ((buffer[strlen(buffer)] != '\0') && *eof == FALSE); 585 | 586 | return TRUE; 587 | } 588 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | {one line to give the program's name and a brief idea of what it does.} 635 | Copyright (C) {year} {name of author} 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | {project} Copyright (C) {year} {fullname} 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------