├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── composer.json ├── composer.lock ├── demo ├── Resources │ ├── docker │ │ ├── db-master │ │ │ ├── Dockerfile-10.1 │ │ │ ├── Dockerfile-10.2 │ │ │ ├── Dockerfile-10.3 │ │ │ ├── Dockerfile-10.4 │ │ │ ├── my.master-10.1.cnf │ │ │ ├── my.master-10.2.cnf │ │ │ ├── my.master-10.3.cnf │ │ │ ├── my.master-10.4.cnf │ │ │ └── sql │ │ │ │ ├── my.master.init_test.sql │ │ │ │ └── my.master.sql │ │ └── db-slave │ │ │ ├── Dockerfile-10.1 │ │ │ ├── Dockerfile-10.2 │ │ │ ├── Dockerfile-10.3 │ │ │ ├── Dockerfile-10.4 │ │ │ ├── my.slave-10.1.cnf │ │ │ ├── my.slave-10.2.cnf │ │ │ ├── my.slave-10.3.cnf │ │ │ ├── my.slave-10.4.cnf │ │ │ └── sql │ │ │ ├── my.slave-10.1.sql │ │ │ ├── my.slave-10.2.sql │ │ │ ├── my.slave-10.3.sql │ │ │ └── my.slave-10.4.sql │ ├── log │ │ └── test_execute_20170828105528.log │ ├── mariadb │ │ ├── change.txt │ │ ├── my.cnf.10.1.38.original │ │ ├── my.cnf.10.2.12.original │ │ ├── my.cnf.10.3.22.original │ │ └── my.cnf.10.4.13.original │ └── tools │ │ ├── generate.sh │ │ └── generate_test_sql.php ├── cron │ ├── [1h]binlog_history_gtid_updater.php │ ├── [every]binlog_collect_partitioner.php │ ├── [every]binlog_collect_worker.php │ └── binlog_collect_info.php └── include │ ├── .env.local │ └── bootstrap_binlog_collector.php ├── src └── Binlog │ └── Collector │ ├── Application │ ├── BinlogCollectorApplication.php │ └── BinlogHistoryGtidUpdaterApplication.php │ ├── BinlogCollector.php │ ├── BinlogCollectorInfo.php │ ├── BinlogCollectorService.php │ ├── BinlogEventCollector.php │ ├── BinlogEventPartitionService.php │ ├── BinlogHistoryCollector.php │ ├── BinlogHistoryGtidChildUpdater.php │ ├── BinlogHistoryService.php │ ├── BinlogPosFinder.php │ ├── Config │ ├── BinlogConfigFactory.php │ ├── BinlogConfiguration.php │ ├── BinlogEnvConfig.php │ ├── BinlogPartitionerConfig.php │ └── BinlogWorkerConfig.php │ ├── Dto │ ├── BinlogHistoryDto.php │ ├── BinlogOffsetDto.php │ ├── GtidOffsetRangeDto.php │ ├── OnlyBinlogOffsetDto.php │ └── OnlyGtidOffsetRangeDto.php │ ├── Exception │ ├── BinlogFinishedException.php │ └── MsgException.php │ ├── External │ ├── AbstractRowEventValueSkipper.php │ ├── ExceptionHandlerInterface.php │ ├── Impl │ │ ├── DefaultRowEventValueSkipper.php │ │ └── DefaultSentryExceptionHandler.php │ ├── RowEventValueSkipperInterface.php │ └── Sentry │ │ └── BinlogRavenClient.php │ ├── Interfaces │ └── BinlogHistoryServiceInterface.php │ ├── Library │ ├── DB │ │ ├── ConnectionProvider.php │ │ └── GnfConnectionProvider.php │ └── FileLock.php │ ├── Model │ ├── BinlogHistoryBaseModel.php │ ├── BinlogHistoryChildOffsetModel.php │ ├── BinlogHistoryModel.php │ ├── BinlogHistoryParentOffsetModel.php │ ├── OnceBinlogHistoryChildOffsetModel.php │ ├── OnceBinlogHistoryModel.php │ ├── OnceBinlogHistoryParentOffsetModel.php │ └── ReplicationDbModel.php │ ├── Monitor │ ├── Constants │ │ └── TimeMonitorConst.php │ ├── Dto │ │ └── TimeMonitorDto.php │ ├── Model │ │ └── BinlogTimeMonitorModel.php │ ├── TimeMonitor.php │ └── TimeMonitorService.php │ ├── OnceBinlogHistoryService.php │ ├── ReplicationQuery.php │ ├── Subscriber │ ├── GetInitBinlogDateSubscriber.php │ └── InsertBinlogSubscriber.php │ └── Utils │ └── BinlogUtils.php └── tests ├── .env.test ├── Binlog ├── BinlogCollectorTest.php ├── BinlogHistoryGtidChildUpdaterTest.php └── Utils │ ├── BinlogConfigurationTest.php │ └── BinlogUtilsTest.php ├── bootstrap.php ├── config.test.php └── phpunit.xml /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | .idea/ 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | 3 | php: 4 | - '7.4' 5 | 6 | env: 7 | - DB=mariadb-10.1 8 | - DB=mariadb-10.2 9 | - DB=mariadb-10.3 10 | - DB=mariadb-10.4 11 | services: 12 | - docker 13 | 14 | cache: 15 | apt: true 16 | bundler: true 17 | directories: 18 | - $HOME/.composer/cache 19 | 20 | before_script: 21 | - "if [ $DB = 'mariadb-10.1' ]; then docker build -t db-master -f ./demo/Resources/docker/db-master/Dockerfile-10.1 ./demo/Resources/docker/db-master; fi" 22 | - "if [ $DB = 'mariadb-10.1' ]; then docker build -t db-slave -f ./demo/Resources/docker/db-slave/Dockerfile-10.1 ./demo/Resources/docker/db-slave; fi" 23 | - "if [ $DB = 'mariadb-10.2' ]; then docker build -t db-master -f ./demo/Resources/docker/db-master/Dockerfile-10.2 ./demo/Resources/docker/db-master; fi" 24 | - "if [ $DB = 'mariadb-10.2' ]; then docker build -t db-slave -f ./demo/Resources/docker/db-slave/Dockerfile-10.2 ./demo/Resources/docker/db-slave; fi" 25 | - "if [ $DB = 'mariadb-10.3' ]; then docker build -t db-master -f ./demo/Resources/docker/db-master/Dockerfile-10.3 ./demo/Resources/docker/db-master; fi" 26 | - "if [ $DB = 'mariadb-10.3' ]; then docker build -t db-slave -f ./demo/Resources/docker/db-slave/Dockerfile-10.3 ./demo/Resources/docker/db-slave; fi" 27 | - "if [ $DB = 'mariadb-10.4' ]; then docker build -t db-master -f ./demo/Resources/docker/db-master/Dockerfile-10.4 ./demo/Resources/docker/db-master; fi" 28 | - "if [ $DB = 'mariadb-10.4' ]; then docker build -t db-slave -f ./demo/Resources/docker/db-slave/Dockerfile-10.4 ./demo/Resources/docker/db-slave; fi" 29 | - docker run -p 13306:3306 --name db-master -e MYSQL_ROOT_PASSWORD=1234 -d db-master 30 | - docker run -p 13307:3306 --name db-slave --link db-master:master -e MYSQL_ROOT_PASSWORD=1234 -d db-slave 31 | - sleep 60; 32 | - docker exec -it db-master bash -c "mysql -uroot -p1234 < temp/my.master.sql" 33 | - sleep 5; 34 | - docker exec -it db-slave bash -c "mysql -uroot -p1234 < temp/my.slave.sql" 35 | - sleep 5; 36 | - docker exec -it db-master bash -c "mysql -uroot -p1234 < temp/my.master.init_test.sql" 37 | - sleep 5 38 | 39 | install: 40 | travis_retry composer install --no-interaction --prefer-dist; 41 | 42 | script: 43 | - "if [ $DB = 'mariadb-10.1' ]; then php ./demo/cron/[every]binlog_collect_partitioner.php change_pos mariadb-bin.000005 4; fi" 44 | - "if [ $DB = 'mariadb-10.2' ]; then php ./demo/cron/[every]binlog_collect_partitioner.php change_pos mariadb-bin.000003 4; fi" 45 | - "if [ $DB = 'mariadb-10.3' ]; then php ./demo/cron/[every]binlog_collect_partitioner.php change_pos mariadb-bin.000003 4; fi" 46 | - "if [ $DB = 'mariadb-10.4' ]; then php ./demo/cron/[every]binlog_collect_partitioner.php change_pos mariadb-bin.000003 4; fi" 47 | - php ./demo/cron/\[every\]binlog_collect_partitioner.php continue 48 | - php ./demo/cron/\[every\]binlog_collect_worker.php 49 | - php ./demo/cron/\[every\]binlog_collect_worker.php 50 | - php ./demo/cron/\[every\]binlog_collect_worker.php 51 | - cd tests 52 | - ../vendor/bin/phpunit 53 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) 5 | and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). 6 | 7 | ### CHANGED 8 | - add test env=mariadb-10.0 -> 10.4 9 | 10 | ## [1.4.0] - 2020-04-21 11 | ### CHANGED 12 | - upgrade php-mysql-replication v6.2.2 13 | - fixed Dto/TimeMonitorDto.php 14 | 15 | ### ADDED 16 | - add test env=mariadb-10.2, 10.3 17 | 18 | ## [1.3.0] - 2019-09-18 19 | ### CHANGED 20 | - change env variable 21 | -- ADD prefix BINLOG_XXX 22 | 23 | ## [1.2.0] - 2019-08-09 24 | ### CHANGED 25 | - upgrade php-mysql-replication v6.0.1 26 | - add test env=mariadb-10.1 27 | 28 | ## [1.1.1] - 2018-11-12 29 | ### CHANGED 30 | - upgrade php-mysql-replication v5.0.5 31 | -- fixed support to recive more than 16Mbyte + tests 32 | - change mariadb's confs 33 | 34 | ## [1.1.0] - 2018-08-13 35 | 36 | ### CHANGED 37 | - upgrade php-mysql-replication v5.0.4 38 | -- configService -> ConfigFactory 39 | -- mySQLReplicationFactory() -> MySQLReplicationFactory($config); 40 | -- update using BinlogException 41 | - refactoring based on php7.1 42 | - binlogId's type string to int 43 | 44 | ## [1.0.3] - 2018-01-02 45 | 46 | ### Fixed 47 | - fix setting of logger 48 | - pass all test 49 | 50 | ## [1.0.2] - 2018-01-02 51 | 52 | ### Added 53 | - add .travis.yml, LICENSE 54 | - upload github 55 | 56 | ## [1.0.1] - 2017-12-21 57 | 58 | ### Changed 59 | - remove unnecessary gtid converting in `BinlogHistoryGtidChildUpdater` 60 | - Change tab to space 61 | 62 | ## [1.0.0] 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 RIDI Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | composer: 2 | composer install --optimize-autoloader 3 | 4 | update-composer: 5 | composer update --no-dev --optimize-autoloader 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | php-binlog-collector 2 | ========= 3 | [![Build Status](https://travis-ci.org/ridi/php-binlog-collector.svg?branch=master)](https://travis-ci.org/ridi/php-binlog-collector) 4 | [![Latest Stable Version](https://poser.pugx.org/ridibooks/php-binlog-collector/v/stable)](https://packagist.org/packages/ridibooks/php-binlog-collector) 5 | [![License](https://poser.pugx.org/ridibooks/php-binlog-collector/license)](https://packagist.org/packages/ridibooks/php-binlog-collector) 6 | 7 | PHP로 만들어진 MySQL/MariaDB 복제 프로토콜 라이브러리를 이용한, MariaDB Binlog 기반의 변경사항 수집 응용 어플리케이션으로 분석 범위를 분할하는 Partitioner와 실제 변경사항 데이터를 수집하는 Worker로 구성되어 있습니다. 8 | 9 | 주요 사용 라이브러리: [php-mysql-replication](https://github.com/krowinski/php-mysql-replication) 10 | 11 | Installation 12 | ========= 13 | 14 | 1. Composer 사용 15 | 16 | ```sh 17 | composer require ridibooks/php-binlog-collector 18 | ``` 19 | 20 | 2. 직접 Github 사용 21 | 22 | ```sh 23 | git clone https://github.com/ridi/php-binlog-collector.git 24 | composer install -o 25 | ``` 26 | 27 | 28 | 29 | MariaDB 사용 제약 30 | ========= 31 | 32 | [php-mysql-replication](https://github.com/krowinski/php-mysql-replication) 와 동일합니다. 33 | 34 | TARGET_DB는 **binlog-format: row** 로 설정되어 운영중이여야 합니다. 35 | 36 | TARGET_DB 접근 권한은 **REPLICATION SLAVE, REPLICATION CLIENT, SELECT** 가 필요합니다. 37 | 38 | 39 | 40 | # DB 연결 설정 41 | 42 | [.env.local](https://github.com/ridi/php-binlog-collector/tree/master/demo/include/.env.local)는 아래와 같습니다. 43 | 44 | BINLOG_MYSQL_TARGET는 분석할 DB, BINLOG_MYSQL_HISTORY_WRITE는 분석 위치 및 이력 데이터를 저장할 DB 입니다. 45 | 46 | ``` 47 | BINLOG_ENABLE_SENTRY=0 48 | BINLOG_SENTRY_KEY= 49 | BINLOG_MYSQL_TARGET_HOST=127.0.0.1 50 | BINLOG_MYSQL_TARGET_USER=repl 51 | BINLOG_MYSQL_TARGET_PORT=3307 52 | BINLOG_MYSQL_TARGET_PASSWORD=1234 53 | BINLOG_MYSQL_TARGET_DBNAME=php_mysql_replication 54 | BINLOG_MYSQL_TARGET_DRIVER=pdo_mysql 55 | BINLOG_MYSQL_TARGET_CHARSET=utf8 56 | 57 | BINLOG_MYSQL_HISTORY_WRITE_HOST=127.0.0.1 58 | BINLOG_MYSQL_HISTORY_WRITE_USER=testUser 59 | BINLOG_MYSQL_HISTORY_WRITE_PORT=3308 60 | BINLOG_MYSQL_HISTORY_WRITE_PASSWORD=testUser 61 | BINLOG_MYSQL_HISTORY_WRITE_DBNAME=platform 62 | BINLOG_MYSQL_HISTORY_WRITE_DRIVER=pdo_mysql 63 | BINLOG_MYSQL_HISTORY_WRITE_CHARSET=utf8 64 | ``` 65 | 66 | 해당 설정 정보는 demo에 docker 기반 환경에 들어 있는 기본 연결 정보입니다. 67 | 68 | 만약 .env 파일이 있으면 해당 파일로 교체됩니다. 69 | 70 | 71 | 72 | Partitioner/Worker 설정 73 | ========= 74 | default 설정정보이며, 외부에서 설정을 변경할 수 있습니다. 75 | 76 | | 설정 정보 | 디폴트 값 | 사용 | 설명 | 77 | | -------------------------------------- | ----- | ----------- | ---------------------- | 78 | | gtid_partition_max_count | 1000 | Partitioner | 파티션 최대 개수 | 79 | | jump_offset_for_next_partition | 10000 | Partitioner | 파티션을 나누기 위한 값 | 80 | | tables_only | X | Worker | 타겟 테이블명들 | 81 | | databases_only | X | Worker | 타겟 데이터베이스명들 | 82 | | child_process_max_count | 10 | Worker | 동시에 실행할 워커 수 | 83 | | once_processed_max_event_count_in_gtid | 100 | Worker | 한 트랜잭션에 한 번에 저장할 이벤트 수 | 84 | | gtid_count_for_persist_per_partition | 500 | Worker | 분석시 위치 갱신을 위한 Gtid 갯수 | 85 | | is_all_print_event | false | ALL | 디버깅 사용여부 | 86 | 87 | 88 | 89 | Partitioner/Worker 실행 90 | ========= 91 | 92 | 1. Binlog Collector Partitioner에서 최초에 분석을 시작할 위치를 입력하여 분석할 범위를 나눕니다. 93 | ```sh 94 | php [every]binlog_collect_partitioner.php change_pos [binLogFileName] [binLogPosition] 95 | 'ex) php [every]binlog_collect_partitioner.php change_pos mariadb-bin.000003 36755' 96 | ``` 97 | 98 | 2. 이미 분석을 시작했다면, 아래 명령어로 DB에 저장된 위치로부터 계속 분석 범위를 나눕니다. 99 | ```sh 100 | php [every]binlog_collect_partitioner.php continue 101 | ``` 102 | 103 | 3. Partitioner가 나눈 범위에 대해서 Binlog Collector Worker가 병렬적으로 처리합니다. 104 | ```sh 105 | php [every]binlog_collector_worker.php 106 | ``` 107 | 108 | 109 | 데모 110 | ========= 111 | 112 | 모든 데모들은 [demo directory](https://github.com/ridi/php-binlog-collector/tree/master/demo) 에서 이용할 수 있고, docker 기반의 mariadb 환경 및 실제 구동은 travis-ci를 통해서 테스트 해볼 수 있습니다. 113 | [MariaDB Binlog을 이용한 변경사항 추적](https://www.ridicorp.com/blog/2017/10/30/binlog-collector) 에서 설계 내용을 확인 할 수 있습니다. 114 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ridibooks/php-binlog-collector", 3 | "description": "Ridibooks PHP Binlog Collector", 4 | "type": "library", 5 | "license": "MIT", 6 | "authors": [ 7 | { 8 | "name": "오길원", 9 | "email": "kobi97@ridi.com" 10 | } 11 | ], 12 | "require": { 13 | "illuminate/support": "^5.4", 14 | "krowinski/php-mysql-replication": "6.2.2", 15 | "monolog/monolog": "^1.18.0", 16 | "ridibooks/platform-gnfdb": "^0.1", 17 | "vlucas/phpdotenv": "^2.4", 18 | "sentry/sentry": "^1.5" 19 | }, 20 | "require-dev": { 21 | "phpunit/phpunit": "^7.0" 22 | }, 23 | "autoload": { 24 | "psr-4": { 25 | "Binlog\\": "src/Binlog" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/Dockerfile-10.1: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.1.38 2 | COPY my.master-10.1.cnf /etc/mysql/my.cnf 3 | COPY ./sql /temp 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/Dockerfile-10.2: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.2.12 2 | COPY my.master-10.2.cnf /etc/mysql/my.cnf 3 | COPY ./sql /temp 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/Dockerfile-10.3: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.3.22 2 | COPY my.master-10.3.cnf /etc/mysql/my.cnf 3 | COPY ./sql /temp 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/Dockerfile-10.4: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.4.13 2 | COPY my.master-10.4.cnf /etc/mysql/my.cnf 3 | COPY ./sql /temp 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/my.master-10.1.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | #relay_log = /var/log/mysql/relay-bin 121 | #relay_log_index = /var/log/mysql/relay-bin.index 122 | #relay_log_info_file = /var/log/mysql/relay-bin.info 123 | #log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | # you can't just change log file size, requires special procedure 136 | innodb_log_file_size = 250M 137 | innodb_buffer_pool_size = 256M 138 | innodb_log_buffer_size = 8M 139 | innodb_file_per_table = 1 140 | innodb_open_files = 400 141 | innodb_io_capacity = 400 142 | innodb_flush_method = O_DIRECT 143 | # 144 | # * Security Features 145 | # 146 | # Read the manual, too, if you want chroot! 147 | # chroot = /var/lib/mysql/ 148 | # 149 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 150 | # 151 | # ssl-ca=/etc/mysql/cacert.pem 152 | # ssl-cert=/etc/mysql/server-cert.pem 153 | # ssl-key=/etc/mysql/server-key.pem 154 | 155 | # 156 | # * Galera-related settings 157 | # 158 | [galera] 159 | # Mandatory settings 160 | #wsrep_on=ON 161 | #wsrep_provider= 162 | #wsrep_cluster_address= 163 | #binlog_format=row 164 | #default_storage_engine=InnoDB 165 | #innodb_autoinc_lock_mode=2 166 | # 167 | # Allow server to accept connections on all interfaces. 168 | # 169 | #bind-address=0.0.0.0 170 | # 171 | # Optional setting 172 | #wsrep_slave_threads=1 173 | #innodb_flush_log_at_trx_commit=0 174 | 175 | [mysqldump] 176 | quick 177 | quote-names 178 | max_allowed_packet = 16M 179 | 180 | [mysql] 181 | #no-auto-rehash # faster start of mysql but no tab completion 182 | 183 | [isamchk] 184 | key_buffer = 16M 185 | 186 | # 187 | # * IMPORTANT: Additional settings that can override those from this file! 188 | # The files must end with '.cnf', otherwise they'll be ignored. 189 | # 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/my.master-10.2.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | #relay_log = /var/log/mysql/relay-bin 121 | #relay_log_index = /var/log/mysql/relay-bin.index 122 | #relay_log_info_file = /var/log/mysql/relay-bin.info 123 | #log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | # you can't just change log file size, requires special procedure 136 | innodb_log_file_size = 250M 137 | innodb_buffer_pool_size = 256M 138 | innodb_log_buffer_size = 8M 139 | innodb_file_per_table = 1 140 | innodb_open_files = 400 141 | innodb_io_capacity = 400 142 | innodb_flush_method = O_DIRECT 143 | # 144 | # * Security Features 145 | # 146 | # Read the manual, too, if you want chroot! 147 | # chroot = /var/lib/mysql/ 148 | # 149 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 150 | # 151 | # ssl-ca=/etc/mysql/cacert.pem 152 | # ssl-cert=/etc/mysql/server-cert.pem 153 | # ssl-key=/etc/mysql/server-key.pem 154 | 155 | # 156 | # * Galera-related settings 157 | # 158 | [galera] 159 | # Mandatory settings 160 | #wsrep_on=ON 161 | #wsrep_provider= 162 | #wsrep_cluster_address= 163 | #binlog_format=row 164 | #default_storage_engine=InnoDB 165 | #innodb_autoinc_lock_mode=2 166 | # 167 | # Allow server to accept connections on all interfaces. 168 | # 169 | #bind-address=0.0.0.0 170 | # 171 | # Optional setting 172 | #wsrep_slave_threads=1 173 | #innodb_flush_log_at_trx_commit=0 174 | 175 | [mysqldump] 176 | quick 177 | quote-names 178 | max_allowed_packet = 16M 179 | 180 | [mysql] 181 | #no-auto-rehash # faster start of mysql but no tab completion 182 | 183 | [isamchk] 184 | key_buffer = 16M 185 | 186 | # 187 | # * IMPORTANT: Additional settings that can override those from this file! 188 | # The files must end with '.cnf', otherwise they'll be ignored. 189 | # 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/my.master-10.3.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | #relay_log = /var/log/mysql/relay-bin 121 | #relay_log_index = /var/log/mysql/relay-bin.index 122 | #relay_log_info_file = /var/log/mysql/relay-bin.info 123 | #log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | innodb_log_file_size = 250M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !include /etc/mysql/mariadb.cnf 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/my.master-10.4.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | #relay_log = /var/log/mysql/relay-bin 121 | #relay_log_index = /var/log/mysql/relay-bin.index 122 | #relay_log_info_file = /var/log/mysql/relay-bin.info 123 | #log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | innodb_log_file_size = 250M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !include /etc/mysql/mariadb.cnf 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-master/sql/my.master.sql: -------------------------------------------------------------------------------- 1 | USE mysql; 2 | CREATE USER 'repl'@'%' IDENTIFIED BY '1234'; 3 | GRANT REPLICATION SLAVE ON *.* to 'repl'; 4 | GRANT REPLICATION CLIENT ON *.* to 'repl'; 5 | GRANT SELECT ON *.* to 'repl'; 6 | CREATE USER 'testUser'@'%' identified by 'testUser'; 7 | GRANT ALL PRIVILEGES ON *.* TO 'testUser'@'%'; 8 | FLUSH PRIVILEGES; 9 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/Dockerfile-10.1: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.1.38 2 | COPY my.slave-10.1.cnf /etc/mysql/my.cnf 3 | COPY ./sql/my.slave-10.1.sql /temp/my.slave.sql 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/Dockerfile-10.2: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.2.12 2 | COPY my.slave-10.2.cnf /etc/mysql/my.cnf 3 | COPY ./sql/my.slave-10.2.sql /temp/my.slave.sql 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/Dockerfile-10.3: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.3.22 2 | COPY my.slave-10.3.cnf /etc/mysql/my.cnf 3 | COPY ./sql/my.slave-10.3.sql /temp/my.slave.sql 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/Dockerfile-10.4: -------------------------------------------------------------------------------- 1 | FROM mariadb:10.4.13 2 | COPY my.slave-10.4.cnf /etc/mysql/my.cnf 3 | COPY ./sql/my.slave-10.4.sql /temp/my.slave.sql 4 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/my.slave-10.1.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 2 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | relay_log = /var/log/mysql/relay-bin 121 | relay_log_index = /var/log/mysql/relay-bin.index 122 | relay_log_info_file = /var/log/mysql/relay-bin.info 123 | log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | # you can't just change log file size, requires special procedure 136 | innodb_log_file_size = 250M 137 | innodb_buffer_pool_size = 256M 138 | innodb_log_buffer_size = 8M 139 | innodb_file_per_table = 1 140 | innodb_open_files = 400 141 | innodb_io_capacity = 400 142 | innodb_flush_method = O_DIRECT 143 | # 144 | # * Security Features 145 | # 146 | # Read the manual, too, if you want chroot! 147 | # chroot = /var/lib/mysql/ 148 | # 149 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 150 | # 151 | # ssl-ca=/etc/mysql/cacert.pem 152 | # ssl-cert=/etc/mysql/server-cert.pem 153 | # ssl-key=/etc/mysql/server-key.pem 154 | 155 | # 156 | # * Galera-related settings 157 | # 158 | [galera] 159 | # Mandatory settings 160 | #wsrep_on=ON 161 | #wsrep_provider= 162 | #wsrep_cluster_address= 163 | #binlog_format=row 164 | #default_storage_engine=InnoDB 165 | #innodb_autoinc_lock_mode=2 166 | # 167 | # Allow server to accept connections on all interfaces. 168 | # 169 | #bind-address=0.0.0.0 170 | # 171 | # Optional setting 172 | #wsrep_slave_threads=1 173 | #innodb_flush_log_at_trx_commit=0 174 | 175 | [mysqldump] 176 | quick 177 | quote-names 178 | max_allowed_packet = 16M 179 | 180 | [mysql] 181 | #no-auto-rehash # faster start of mysql but no tab completion 182 | 183 | [isamchk] 184 | key_buffer = 16M 185 | 186 | # 187 | # * IMPORTANT: Additional settings that can override those from this file! 188 | # The files must end with '.cnf', otherwise they'll be ignored. 189 | # 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/my.slave-10.2.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 2 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | relay_log = /var/log/mysql/relay-bin 121 | relay_log_index = /var/log/mysql/relay-bin.index 122 | relay_log_info_file = /var/log/mysql/relay-bin.info 123 | log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | # you can't just change log file size, requires special procedure 136 | innodb_log_file_size = 250M 137 | innodb_buffer_pool_size = 256M 138 | innodb_log_buffer_size = 8M 139 | innodb_file_per_table = 1 140 | innodb_open_files = 400 141 | innodb_io_capacity = 400 142 | innodb_flush_method = O_DIRECT 143 | # 144 | # * Security Features 145 | # 146 | # Read the manual, too, if you want chroot! 147 | # chroot = /var/lib/mysql/ 148 | # 149 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 150 | # 151 | # ssl-ca=/etc/mysql/cacert.pem 152 | # ssl-cert=/etc/mysql/server-cert.pem 153 | # ssl-key=/etc/mysql/server-key.pem 154 | 155 | # 156 | # * Galera-related settings 157 | # 158 | [galera] 159 | # Mandatory settings 160 | #wsrep_on=ON 161 | #wsrep_provider= 162 | #wsrep_cluster_address= 163 | #binlog_format=row 164 | #default_storage_engine=InnoDB 165 | #innodb_autoinc_lock_mode=2 166 | # 167 | # Allow server to accept connections on all interfaces. 168 | # 169 | #bind-address=0.0.0.0 170 | # 171 | # Optional setting 172 | #wsrep_slave_threads=1 173 | #innodb_flush_log_at_trx_commit=0 174 | 175 | [mysqldump] 176 | quick 177 | quote-names 178 | max_allowed_packet = 16M 179 | 180 | [mysql] 181 | #no-auto-rehash # faster start of mysql but no tab completion 182 | 183 | [isamchk] 184 | key_buffer = 16M 185 | 186 | # 187 | # * IMPORTANT: Additional settings that can override those from this file! 188 | # The files must end with '.cnf', otherwise they'll be ignored. 189 | # 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/my.slave-10.3.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 2 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | relay_log = /var/log/mysql/relay-bin 121 | relay_log_index = /var/log/mysql/relay-bin.index 122 | relay_log_info_file = /var/log/mysql/relay-bin.info 123 | log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | innodb_log_file_size = 250M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !include /etc/mysql/mariadb.cnf 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/docker/db-slave/my.slave-10.4.cnf: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 64M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | server-id = 2 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | log_bin = /var/log/mysql/mariadb-bin 113 | log_bin_index = /var/log/mysql/mariadb-bin.index 114 | binlog-format = row 115 | # not fab for performance, but safer 116 | #sync_binlog = 1 117 | expire_logs_days = 10 118 | max_binlog_size = 100M 119 | # slaves 120 | relay_log = /var/log/mysql/relay-bin 121 | relay_log_index = /var/log/mysql/relay-bin.index 122 | relay_log_info_file = /var/log/mysql/relay-bin.info 123 | log_slave_updates 124 | #read_only 125 | # 126 | # If applications support it, this stricter sql_mode prevents some 127 | # mistakes like inserting invalid dates etc. 128 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 129 | # 130 | # * InnoDB 131 | # 132 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 133 | # Read the manual for more InnoDB related options. There are many! 134 | default_storage_engine = InnoDB 135 | innodb_log_file_size = 250M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !include /etc/mysql/mariadb.cnf 190 | !includedir /etc/mysql/conf.d/ 191 | -------------------------------------------------------------------------------- /demo/Resources/mariadb/change.txt: -------------------------------------------------------------------------------- 1 | my.cnf -> my.master.cnf 2 | server-id = 1 3 | log_bin = /var/log/mysql/mariadb-bin 4 | log_bin_index = /var/log/mysql/mariadb-bin.index 5 | binlog-format = row 6 | innodb_log_file_size = 250M 7 | max_allowed_packet = 64M 8 | 9 | my.cnf -> my.slave.cnf 10 | server-id = 2 11 | log_bin = /var/log/mysql/mariadb-bin 12 | log_bin_index = /var/log/mysql/mariadb-bin.index 13 | # slaves 14 | relay_log = /var/log/mysql/relay-bin 15 | relay_log_index = /var/log/mysql/relay-bin.index 16 | relay_log_info_file = /var/log/mysql/relay-bin.info 17 | log_slave_updates 18 | binlog-format = row 19 | innodb_log_file_size = 250M 20 | max_allowed_packet = 64M 21 | -------------------------------------------------------------------------------- /demo/Resources/mariadb/my.cnf.10.1.38.original: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | # you can't just change log file size, requires special procedure 135 | #innodb_log_file_size = 50M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !includedir /etc/mysql/conf.d/ 190 | -------------------------------------------------------------------------------- /demo/Resources/mariadb/my.cnf.10.2.12.original: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | # you can't just change log file size, requires special procedure 135 | #innodb_log_file_size = 50M 136 | innodb_buffer_pool_size = 256M 137 | innodb_log_buffer_size = 8M 138 | innodb_file_per_table = 1 139 | innodb_open_files = 400 140 | innodb_io_capacity = 400 141 | innodb_flush_method = O_DIRECT 142 | # 143 | # * Security Features 144 | # 145 | # Read the manual, too, if you want chroot! 146 | # chroot = /var/lib/mysql/ 147 | # 148 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 149 | # 150 | # ssl-ca=/etc/mysql/cacert.pem 151 | # ssl-cert=/etc/mysql/server-cert.pem 152 | # ssl-key=/etc/mysql/server-key.pem 153 | 154 | # 155 | # * Galera-related settings 156 | # 157 | [galera] 158 | # Mandatory settings 159 | #wsrep_on=ON 160 | #wsrep_provider= 161 | #wsrep_cluster_address= 162 | #binlog_format=row 163 | #default_storage_engine=InnoDB 164 | #innodb_autoinc_lock_mode=2 165 | # 166 | # Allow server to accept connections on all interfaces. 167 | # 168 | #bind-address=0.0.0.0 169 | # 170 | # Optional setting 171 | #wsrep_slave_threads=1 172 | #innodb_flush_log_at_trx_commit=0 173 | 174 | [mysqldump] 175 | quick 176 | quote-names 177 | max_allowed_packet = 16M 178 | 179 | [mysql] 180 | #no-auto-rehash # faster start of mysql but no tab completion 181 | 182 | [isamchk] 183 | key_buffer = 16M 184 | 185 | # 186 | # * IMPORTANT: Additional settings that can override those from this file! 187 | # The files must end with '.cnf', otherwise they'll be ignored. 188 | # 189 | !includedir /etc/mysql/conf.d/ 190 | -------------------------------------------------------------------------------- /demo/Resources/mariadb/my.cnf.10.3.22.original: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | innodb_buffer_pool_size = 256M 135 | innodb_log_buffer_size = 8M 136 | innodb_file_per_table = 1 137 | innodb_open_files = 400 138 | innodb_io_capacity = 400 139 | innodb_flush_method = O_DIRECT 140 | # 141 | # * Security Features 142 | # 143 | # Read the manual, too, if you want chroot! 144 | # chroot = /var/lib/mysql/ 145 | # 146 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 147 | # 148 | # ssl-ca=/etc/mysql/cacert.pem 149 | # ssl-cert=/etc/mysql/server-cert.pem 150 | # ssl-key=/etc/mysql/server-key.pem 151 | 152 | # 153 | # * Galera-related settings 154 | # 155 | [galera] 156 | # Mandatory settings 157 | #wsrep_on=ON 158 | #wsrep_provider= 159 | #wsrep_cluster_address= 160 | #binlog_format=row 161 | #default_storage_engine=InnoDB 162 | #innodb_autoinc_lock_mode=2 163 | # 164 | # Allow server to accept connections on all interfaces. 165 | # 166 | #bind-address=0.0.0.0 167 | # 168 | # Optional setting 169 | #wsrep_slave_threads=1 170 | #innodb_flush_log_at_trx_commit=0 171 | 172 | [mysqldump] 173 | quick 174 | quote-names 175 | max_allowed_packet = 16M 176 | 177 | [mysql] 178 | #no-auto-rehash # faster start of mysql but no tab completion 179 | 180 | [isamchk] 181 | key_buffer = 16M 182 | 183 | # 184 | # * IMPORTANT: Additional settings that can override those from this file! 185 | # The files must end with '.cnf', otherwise they'll be ignored. 186 | # 187 | !include /etc/mysql/mariadb.cnf 188 | !includedir /etc/mysql/conf.d/ 189 | -------------------------------------------------------------------------------- /demo/Resources/mariadb/my.cnf.10.4.13.original: -------------------------------------------------------------------------------- 1 | # MariaDB database server configuration file. 2 | # 3 | # You can copy this file to one of: 4 | # - "/etc/mysql/my.cnf" to set global options, 5 | # - "~/.my.cnf" to set user-specific options. 6 | # 7 | # One can use all long options that the program supports. 8 | # Run program with --help to get a list of available options and with 9 | # --print-defaults to see which it would actually understand and use. 10 | # 11 | # For explanations see 12 | # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 13 | 14 | # This will be passed to all mysql clients 15 | # It has been reported that passwords should be enclosed with ticks/quotes 16 | # escpecially if they contain "#" chars... 17 | # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 18 | [client] 19 | port = 3306 20 | socket = /var/run/mysqld/mysqld.sock 21 | 22 | # Here is entries for some specific programs 23 | # The following values assume you have at least 32M ram 24 | 25 | # This was formally known as [safe_mysqld]. Both versions are currently parsed. 26 | [mysqld_safe] 27 | socket = /var/run/mysqld/mysqld.sock 28 | nice = 0 29 | 30 | [mysqld] 31 | # 32 | # * Basic Settings 33 | # 34 | #user = mysql 35 | pid-file = /var/run/mysqld/mysqld.pid 36 | socket = /var/run/mysqld/mysqld.sock 37 | port = 3306 38 | basedir = /usr 39 | datadir = /var/lib/mysql 40 | tmpdir = /tmp 41 | lc_messages_dir = /usr/share/mysql 42 | lc_messages = en_US 43 | skip-external-locking 44 | # 45 | # Instead of skip-networking the default is now to listen only on 46 | # localhost which is more compatible and is not less secure. 47 | #bind-address = 127.0.0.1 48 | # 49 | # * Fine Tuning 50 | # 51 | max_connections = 100 52 | connect_timeout = 5 53 | wait_timeout = 600 54 | max_allowed_packet = 16M 55 | thread_cache_size = 128 56 | sort_buffer_size = 4M 57 | bulk_insert_buffer_size = 16M 58 | tmp_table_size = 32M 59 | max_heap_table_size = 32M 60 | # 61 | # * MyISAM 62 | # 63 | # This replaces the startup script and checks MyISAM tables if needed 64 | # the first time they are touched. On error, make copy and try a repair. 65 | myisam_recover_options = BACKUP 66 | key_buffer_size = 128M 67 | #open-files-limit = 2000 68 | table_open_cache = 400 69 | myisam_sort_buffer_size = 512M 70 | concurrent_insert = 2 71 | read_buffer_size = 2M 72 | read_rnd_buffer_size = 1M 73 | # 74 | # * Query Cache Configuration 75 | # 76 | # Cache only tiny result sets, so we can fit more in the query cache. 77 | query_cache_limit = 128K 78 | query_cache_size = 64M 79 | # for more write intensive setups, set to DEMAND or OFF 80 | #query_cache_type = DEMAND 81 | # 82 | # * Logging and Replication 83 | # 84 | # Both location gets rotated by the cronjob. 85 | # Be aware that this log type is a performance killer. 86 | # As of 5.1 you can enable the log at runtime! 87 | #general_log_file = /var/log/mysql/mysql.log 88 | #general_log = 1 89 | # 90 | # Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf. 91 | # 92 | # we do want to know about network errors and such 93 | #log_warnings = 2 94 | # 95 | # Enable the slow query log to see queries with especially long duration 96 | #slow_query_log[={0|1}] 97 | slow_query_log_file = /var/log/mysql/mariadb-slow.log 98 | long_query_time = 10 99 | #log_slow_rate_limit = 1000 100 | #log_slow_verbosity = query_plan 101 | 102 | #log-queries-not-using-indexes 103 | #log_slow_admin_statements 104 | # 105 | # The following can be used as easy to replay backup logs or for replication. 106 | # note: if you are setting up a replication slave, see README.Debian about 107 | # other settings you may need to change. 108 | #server-id = 1 109 | #report_host = master1 110 | #auto_increment_increment = 2 111 | #auto_increment_offset = 1 112 | #log_bin = /var/log/mysql/mariadb-bin 113 | #log_bin_index = /var/log/mysql/mariadb-bin.index 114 | # not fab for performance, but safer 115 | #sync_binlog = 1 116 | expire_logs_days = 10 117 | max_binlog_size = 100M 118 | # slaves 119 | #relay_log = /var/log/mysql/relay-bin 120 | #relay_log_index = /var/log/mysql/relay-bin.index 121 | #relay_log_info_file = /var/log/mysql/relay-bin.info 122 | #log_slave_updates 123 | #read_only 124 | # 125 | # If applications support it, this stricter sql_mode prevents some 126 | # mistakes like inserting invalid dates etc. 127 | #sql_mode = NO_ENGINE_SUBSTITUTION,TRADITIONAL 128 | # 129 | # * InnoDB 130 | # 131 | # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 132 | # Read the manual for more InnoDB related options. There are many! 133 | default_storage_engine = InnoDB 134 | innodb_buffer_pool_size = 256M 135 | innodb_log_buffer_size = 8M 136 | innodb_file_per_table = 1 137 | innodb_open_files = 400 138 | innodb_io_capacity = 400 139 | innodb_flush_method = O_DIRECT 140 | # 141 | # * Security Features 142 | # 143 | # Read the manual, too, if you want chroot! 144 | # chroot = /var/lib/mysql/ 145 | # 146 | # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 147 | # 148 | # ssl-ca=/etc/mysql/cacert.pem 149 | # ssl-cert=/etc/mysql/server-cert.pem 150 | # ssl-key=/etc/mysql/server-key.pem 151 | 152 | # 153 | # * Galera-related settings 154 | # 155 | [galera] 156 | # Mandatory settings 157 | #wsrep_on=ON 158 | #wsrep_provider= 159 | #wsrep_cluster_address= 160 | #binlog_format=row 161 | #default_storage_engine=InnoDB 162 | #innodb_autoinc_lock_mode=2 163 | # 164 | # Allow server to accept connections on all interfaces. 165 | # 166 | #bind-address=0.0.0.0 167 | # 168 | # Optional setting 169 | #wsrep_slave_threads=1 170 | #innodb_flush_log_at_trx_commit=0 171 | 172 | [mysqldump] 173 | quick 174 | quote-names 175 | max_allowed_packet = 16M 176 | 177 | [mysql] 178 | #no-auto-rehash # faster start of mysql but no tab completion 179 | 180 | [isamchk] 181 | key_buffer = 16M 182 | 183 | # 184 | # * IMPORTANT: Additional settings that can override those from this file! 185 | # The files must end with '.cnf', otherwise they'll be ignored. 186 | # 187 | !include /etc/mysql/mariadb.cnf 188 | !includedir /etc/mysql/conf.d/ 189 | -------------------------------------------------------------------------------- /demo/Resources/tools/generate.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | php generate_test_sql.php > ../docker/db-master/sql/my.master.init_test.sql 3 | -------------------------------------------------------------------------------- /demo/cron/[1h]binlog_history_gtid_updater.php: -------------------------------------------------------------------------------- 1 | tryLock()) { 13 | die(); 14 | } 15 | $env_config = BinlogEnvConfig::importDefaultConfig(); 16 | $exception_handler = new DefaultSentryExceptionHandler('./', 'binlog_history_gtid_updater', $env_config); 17 | $configuration = BinlogConfiguration::newInstance($argv, $env_config, $exception_handler); 18 | 19 | $application = new BinlogHistoryGtidUpdaterApplication($configuration); 20 | $application->executeUpdate(); 21 | 22 | $lock->unlock(); 23 | -------------------------------------------------------------------------------- /demo/cron/[every]binlog_collect_partitioner.php: -------------------------------------------------------------------------------- 1 | tryLock()) { 15 | die(); 16 | } 17 | $env_config = BinlogEnvConfig::importDefaultConfig(); 18 | $exception_handler = new DefaultSentryExceptionHandler('./', 'binlog_collector', $env_config); 19 | $configuration = BinlogConfiguration::newInstance($argv, $env_config, $exception_handler); 20 | 21 | $function = function () use ($configuration): void { 22 | $application = new BinlogCollectorApplication($configuration); 23 | $application->executePartitioning(); 24 | }; 25 | TimeMonitor::benchmark(TimeMonitorConst::TYPE_BINLOG_PARTITIONER, $function); 26 | 27 | $lock->unlock(); 28 | -------------------------------------------------------------------------------- /demo/cron/[every]binlog_collect_worker.php: -------------------------------------------------------------------------------- 1 | tryLock()) { 16 | die(); 17 | } 18 | 19 | $tables_only = ['test_target1', 'test_target2', 'test_target3', 'test_target4']; 20 | $databasesOnly = ['binlog_sample1','binlog_sample2','binlog_sample3','binlog_sample4']; 21 | 22 | $env_config = BinlogEnvConfig::extendDefaultConfig( 23 | [], 24 | new DefaultRowEventValueSkipper($tables_only, $databasesOnly) 25 | ); 26 | 27 | $env_config->validateTarget(); 28 | $exception_handler = new DefaultSentryExceptionHandler('./', 'binlog_collector', $env_config); 29 | $configuration = BinlogConfiguration::newInstance($argv, $env_config, $exception_handler); 30 | 31 | $function = function () use ($configuration): void { 32 | $application = new BinlogCollectorApplication($configuration); 33 | $application->executeWorking(); 34 | }; 35 | TimeMonitor::benchmark(TimeMonitorConst::TYPE_BINLOG_WORKER, $function); 36 | 37 | $lock->unlock(); 38 | -------------------------------------------------------------------------------- /demo/cron/binlog_collect_info.php: -------------------------------------------------------------------------------- 1 | tryLock()) { 13 | die(); 14 | } 15 | $env_config = BinlogEnvConfig::extendDefaultConfig( 16 | [ 17 | 'gtid_partition_max_count' => 250, 18 | 'jump_offset_for_next_partition' => 1000, 19 | ] 20 | ); 21 | $exception_handler = new DefaultSentryExceptionHandler('./', 'binlog_collector', $env_config); 22 | $configuration = BinlogConfiguration::newInstance($argv, $env_config, $exception_handler); 23 | 24 | $application = new BinlogCollectorApplication($configuration); 25 | $application->getInfo(); 26 | 27 | $lock->unlock(); 28 | -------------------------------------------------------------------------------- /demo/include/.env.local: -------------------------------------------------------------------------------- 1 | BINLOG_ENABLE_SENTRY=0 2 | #BINLOG_SENTRY_KEY= 3 | 4 | BINLOG_MYSQL_TARGET_HOST=127.0.0.1 5 | BINLOG_MYSQL_TARGET_USER=repl 6 | BINLOG_MYSQL_TARGET_PORT=13306 7 | BINLOG_MYSQL_TARGET_PASSWORD=1234 8 | BINLOG_MYSQL_TARGET_DBNAME=php_mysql_replication 9 | BINLOG_MYSQL_TARGET_DRIVER=pdo_mysql 10 | BINLOG_MYSQL_TARGET_CHARSET=utf8 11 | 12 | BINLOG_MYSQL_HISTORY_WRITE_HOST=127.0.0.1 13 | BINLOG_MYSQL_HISTORY_WRITE_USER=testUser 14 | BINLOG_MYSQL_HISTORY_WRITE_PORT=13307 15 | BINLOG_MYSQL_HISTORY_WRITE_PASSWORD=testUser 16 | BINLOG_MYSQL_HISTORY_WRITE_DBNAME=platform 17 | BINLOG_MYSQL_HISTORY_WRITE_DRIVER=pdo_mysql 18 | BINLOG_MYSQL_HISTORY_WRITE_CHARSET=utf8 19 | -------------------------------------------------------------------------------- /demo/include/bootstrap_binlog_collector.php: -------------------------------------------------------------------------------- 1 | load(); 8 | } elseif (is_readable(__DIR__ . '/'. '.env.local')) { 9 | $dotenv = new Dotenv\Dotenv(__DIR__, '.env.local'); 10 | $dotenv->load(); 11 | } 12 | 13 | date_default_timezone_set('Asia/Seoul'); 14 | error_reporting(E_ALL & ~E_NOTICE); 15 | set_time_limit(0); 16 | ini_set('mysql.connect_timeout', -1); 17 | ini_set('default_socket_timeout', -1); 18 | ini_set('max_execution_time', -1); 19 | ini_set('memory_limit', '2048M'); 20 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogCollectorInfo.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 19 | try { 20 | $this->assertInfoCommand($argv); 21 | } catch (MsgException $e) { 22 | print('error: ' . $e->getMessage() . "\n"); 23 | $this->printGetInfoUsage($argv[0]); 24 | exit(); 25 | } 26 | 27 | switch ($argv[1]) { 28 | case 'master_status': 29 | $replication_db_model = new ReplicationDbModel($partitioner_config->connect_config); 30 | $replication_query = new ReplicationQuery($replication_db_model); 31 | $master_binlog_offset_dto = $replication_query->getMasterBinlogOffset(); 32 | $logger->info("only Query: Master Bin_Log_Offset: {$master_binlog_offset_dto}"); 33 | exit; 34 | case 'gtid_to_binlog_pos': 35 | $this->printGtidToBinlogPos($partitioner_config->binlog_connect_array, $argv); 36 | exit; 37 | } 38 | } 39 | 40 | private function printGtidToBinlogPos(array $connect_array, array $argv): void 41 | { 42 | $replace_connect_array = []; 43 | if (count($argv) > 3 && $argv[3] !== 'current') { 44 | $replace_connect_array['ip'] = $argv[3]; 45 | } 46 | $connect_config = BinlogConfiguration::createCustomConnectConfigWithReplace( 47 | $connect_array, 48 | $replace_connect_array 49 | ); 50 | try { 51 | $replication_db_model = new ReplicationDbModel($connect_config); 52 | $replication_query = new ReplicationQuery($replication_db_model); 53 | // check connection and auth 54 | $replication_query->assertCheckAuth(); 55 | $master_binlog_offset_dto = $replication_query->getMasterBinlogOffset(); 56 | 57 | $pos_finder = new BinlogPosFinder($this->logger, $replication_db_model); 58 | 59 | $finder_dto = null; 60 | switch (count($argv)) { 61 | case 3: 62 | case 4: 63 | $finder_dto = $pos_finder->findBinlogOffsetDto($master_binlog_offset_dto, $argv[2]); 64 | break; 65 | case 5: 66 | $finder_dto = $pos_finder->findBinlogOffsetDto($master_binlog_offset_dto, $argv[2], $argv[4]); 67 | break; 68 | case 6: 69 | $finder_dto = $pos_finder->findBinlogOffsetDto( 70 | $master_binlog_offset_dto, 71 | $argv[2], 72 | $argv[4], 73 | intval($argv[5]) 74 | ); 75 | break; 76 | } 77 | $this->logger->info("gtid_to_binlog_pos's result: {$finder_dto}"); 78 | $replication_db_model->close(); 79 | } catch (\Throwable $e) { 80 | $this->logger->info("error: " . $e->getMessage()); 81 | throw new MsgException($e->getMessage()); 82 | } 83 | } 84 | 85 | private function assertInfoCommand(array $argv): void 86 | { 87 | $count = count($argv); 88 | 89 | if ($count >= 2) { 90 | switch ($argv[1]) { 91 | case 'gtid_to_binlog_pos': 92 | if ($count < 3 || 6 < $count) { 93 | throw new MsgException('wrong command'); 94 | } 95 | 96 | return; 97 | case 'master_status': 98 | return; 99 | default: 100 | throw new MsgException('wrong command'); 101 | } 102 | } 103 | 104 | throw new MsgException('wrong command'); 105 | } 106 | 107 | private function printGetInfoUsage(string $php_file): void 108 | { 109 | print("##########################################################################################\n"); 110 | print("Usage:\n"); 111 | 112 | print("1. master 현재 위치 조회\n"); 113 | print(" php {$php_file} master_status\n"); 114 | print("\n"); 115 | 116 | print("2. Gtid To Binlog Pos 변환\n"); 117 | print(" php {$php_file} gtid_to_binlog_pos [gtid] [ip=current] [skip_server_id=none] [limitPreviousFileCount=5]\n"); 118 | print("ex) php {$php_file} gtid_to_binlog_pos 0-43-14543397 current none 10\n"); 119 | print("##########################################################################################\n"); 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogCollectorService.php: -------------------------------------------------------------------------------- 1 | getMinCurrentBinlogPositionDate(); 13 | 14 | if ($min_current_binlog_position_date !== null) { 15 | return $min_current_binlog_position_date; 16 | } 17 | 18 | $parent_binlog_date = $binlog_history_service->getParentBinlogDate(); 19 | 20 | if ($parent_binlog_date === null) { 21 | throw new MsgException('not existed parent_binlog_date'); 22 | } 23 | 24 | return $parent_binlog_date; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogEventPartitionService.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 26 | $this->replication_db_model = $replication_db_model; 27 | $this->jump_offset = $jump_offset_for_next_partition; 28 | } 29 | 30 | /** 31 | * Binlog 기준 위치로부터 32 | * 입력으로 받은 Jump Offset Row 만큼 이동하여, 첫번째 Event_type:Gtid의 End_log_pos의 위치를 반복적으로 수집한다. 33 | * (단, Binlog 이벤트 row가 없으면, 다음 시퀀스 파일의 처음 Jump Offset 만큼에 대해서도 체크해보고 있으면 계속 수집을 진행한다.) 34 | * 참고) 여기서 보장을 하는 것은 Jump Offset Row의 개수와 상관없이 무조건 최소 한개 이후의 GTID를 수집. 35 | * 36 | * @param int $partition_max_count 37 | * @param BinlogOffsetDto $start_binlog_offset_dto 38 | * @param BinlogOffsetDto|null $end_binlog_offset_dto (해당 범위를 넘어서면 종료) 39 | * 40 | * @return OnlyBinlogOffsetDto[] 41 | */ 42 | public function calculateGtidOffsetDtos( 43 | int $partition_max_count, 44 | BinlogOffsetDto $start_binlog_offset_dto, 45 | BinlogOffsetDto $end_binlog_offset_dto = null 46 | ): array { 47 | $this->logger->info("Limit Extra PartitionMaxCount: {$partition_max_count}"); 48 | $dtos = []; 49 | $current_partition_count = 0; 50 | 51 | [$dicts, $current_binlog_file_name] = $this->getNextJumpEventDictsAndFile( 52 | $start_binlog_offset_dto->file_name, 53 | $start_binlog_offset_dto->position, 54 | true 55 | ); 56 | 57 | $dict_count = count($dicts); 58 | while ($current_partition_count < $partition_max_count && $dict_count > 0) { 59 | foreach ($dicts as $dict) { 60 | if ($dict['Event_type'] === 'Gtid') { 61 | $gtid_end_pos = $dict['End_log_pos']; 62 | $current_partition_count++; 63 | $this->dumpPartitionInfo($current_partition_count, $current_binlog_file_name, $gtid_end_pos); 64 | $dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset($current_binlog_file_name, $gtid_end_pos); 65 | $dtos[] = $dto; 66 | 67 | // end 범위를 넣어서면 강제 종료 68 | if ($end_binlog_offset_dto !== null && $end_binlog_offset_dto->compareTo($dto) <= 0) { 69 | return $dtos; 70 | } 71 | break; 72 | } 73 | } 74 | 75 | $next_pos = $dicts[$dict_count - 1]['End_log_pos']; 76 | [$dicts, $current_binlog_file_name] = $this->getNextJumpEventDictsAndFile( 77 | $current_binlog_file_name, 78 | $next_pos, 79 | true 80 | ); 81 | $dict_count = count($dicts); 82 | } 83 | 84 | return $dtos; 85 | } 86 | 87 | /** 88 | * 현재 주어진 binlog 위치로부터, jump offset row 이후, 디폴트 개수(1000개) 만큼 가져옴. 89 | * 단 $use_next_seq_file를 사용하고, 데이터가 없으면 다음 시퀀스 파일의 처음 jump offset row만큼 가져옴 90 | * 91 | * @param string $binlog_file_name 92 | * @param int $pos 93 | * @param bool $use_next_seq_file 94 | * 95 | * @return array 96 | */ 97 | private function getNextJumpEventDictsAndFile(string $binlog_file_name, int $pos, bool $use_next_seq_file): array 98 | { 99 | $dicts = $this->replication_db_model->showBinlogEvents($binlog_file_name, $pos, $this->jump_offset); 100 | if ($use_next_seq_file && count($dicts) === 0) { 101 | $binlog_file_name = BinlogUtils::calculateNextSeqFile($binlog_file_name); 102 | $row_count = $this->jump_offset; 103 | $dicts = $this->replication_db_model->showBinlogEventsFromInit($binlog_file_name, 0, $row_count); 104 | } 105 | 106 | return [$dicts, $binlog_file_name]; 107 | } 108 | 109 | public function calculateLastSecondGtidOffsetDto( 110 | int $current_partition_count, 111 | OnlyBinlogOffsetDto $start_binlog_offset_dto 112 | ): ?OnlyBinlogOffsetDto { 113 | [$dicts, $current_binlog_file_name] = $this->getNextEventDictsAndFile( 114 | $start_binlog_offset_dto->file_name, 115 | $start_binlog_offset_dto->position, 116 | true 117 | ); 118 | 119 | $current_gtid_count = 0; 120 | for ($i = count($dicts) - 1; $i > 0; $i--) { 121 | $dict = $dicts[$i]; 122 | if ($dict['Event_type'] === 'Gtid') { 123 | $current_gtid_count++; 124 | //가장 마지막의 gtid는 계속 처리중일지 몰라서 혹시 몰라서 last second로 125 | if ($current_gtid_count === 2) { 126 | $gtid_end_pos = $dict['End_log_pos']; 127 | $this->dumpPartitionInfo($current_partition_count, $current_binlog_file_name, $gtid_end_pos); 128 | 129 | return OnlyBinlogOffsetDto::importOnlyBinlogOffset($current_binlog_file_name, $gtid_end_pos); 130 | } 131 | } 132 | } 133 | 134 | return null; 135 | } 136 | 137 | private function getNextEventDictsAndFile(string $binlog_file_name, int $pos, bool $use_next_seq_file): array 138 | { 139 | $row_count = $this->jump_offset; 140 | $dicts = $this->replication_db_model->showBinlogEvents($binlog_file_name, $pos, 0, $row_count); 141 | if ($use_next_seq_file && count($dicts) === 0) { 142 | $binlog_file_name = BinlogUtils::calculateNextSeqFile($binlog_file_name); 143 | $dicts = $this->replication_db_model->showBinlogEventsFromInit($binlog_file_name, 0, $row_count); 144 | } 145 | 146 | return [$dicts, $binlog_file_name]; 147 | } 148 | 149 | private function dumpPartitionInfo(int $partition_count, string $binlog_file_name, int $gtid_end_log_pos): void 150 | { 151 | if ($partition_count % 10 !== 0) { 152 | return; 153 | } 154 | $this->logger->info("~ {$partition_count} PartitionCount({$binlog_file_name}/{$gtid_end_log_pos})"); 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogHistoryCollector.php: -------------------------------------------------------------------------------- 1 | row_event_value_skipper = $row_event_value_skipper; 22 | } 23 | 24 | /** 25 | * @param OnlyBinlogOffsetDto $binlog_offset_dto 26 | * @param EventDTO[] $events 27 | * 28 | * @return BinlogHistoryDto[] 29 | */ 30 | public function collect(OnlyBinlogOffsetDto $binlog_offset_dto, array $events): array 31 | { 32 | $binlog_history_dtos = []; 33 | foreach ($events as $event_index => $event) { 34 | if ($event instanceof RowsDTO) { 35 | $dtos = $this->collectUniversalHistoryDtos($binlog_offset_dto, $event, $event_index); 36 | $binlog_history_dtos = array_merge($binlog_history_dtos, $dtos); 37 | } 38 | } 39 | 40 | return $binlog_history_dtos; 41 | } 42 | 43 | /** 44 | * @param OnlyBinlogOffsetDto $binlog_offset_dto 45 | * @param RowsDTO $event 46 | * @param int $event_index 47 | * 48 | * @return BinlogHistoryDto[] 49 | */ 50 | private function collectUniversalHistoryDtos( 51 | OnlyBinlogOffsetDto $binlog_offset_dto, 52 | RowsDTO $event, 53 | int $event_index 54 | ): array { 55 | $ret_dtos = []; 56 | $table = $event->getTableMap()->getTable(); 57 | $schema_name = $event->getTableMap()->getDatabase() . '.' . $table; 58 | $values = $event->getValues(); 59 | $timestamp = $event->getEventInfo()->getTimestamp(); 60 | $reg_date = (new \DateTime())->setTimestamp($timestamp)->format('Y-m-d H:i:s'); 61 | $action = $event->getType(); 62 | 63 | foreach ($values as $value) { 64 | $timestamp = $event->getEventInfo()->getTimestamp(); 65 | if (!$this->row_event_value_skipper->isTargetEventValue($timestamp, $table, $action, $value)) { 66 | continue; 67 | } 68 | 69 | if ($action === ConstEventsNames::UPDATE) { 70 | $pk_ids = $this->findPrimaryValues($event->getTableMap()->getColumnDTOCollection(), $value['before']); 71 | $value = $this->getOnlyChangedValue($value); 72 | } else { 73 | $pk_ids = $this->findPrimaryValues($event->getTableMap()->getColumnDTOCollection(), $value); 74 | } 75 | $ret_dtos[] = BinlogHistoryDto::importFromLog( 76 | $binlog_offset_dto, 77 | $schema_name, 78 | $pk_ids, 79 | $action, 80 | $reg_date, 81 | $value, 82 | $event_index 83 | ); 84 | } 85 | 86 | return $ret_dtos; 87 | } 88 | 89 | private function findPrimaryValues(ColumnDTOCollection $column_dto_collection, array $row_values): string 90 | { 91 | $pk_ids = []; 92 | /** @var ColumnDto $column_dto */ 93 | foreach ($column_dto_collection as $column_dto) { 94 | if ($column_dto->isPrimary()) { 95 | $pk_ids[] = $row_values[$column_dto->getName()]; 96 | } 97 | } 98 | if (empty($pk_ids)) { 99 | return '?'; 100 | } 101 | 102 | return implode(',', $pk_ids); 103 | } 104 | 105 | private function getOnlyChangedValue(array $value): array 106 | { 107 | $new_value = []; 108 | $before = $value['before']; 109 | $after = $value['after']; 110 | 111 | $new_after = []; 112 | 113 | foreach ($before as $key => $val) { 114 | if ($before[$key] !== $after[$key]) { 115 | $new_after[$key] = $after[$key]; 116 | } 117 | } 118 | $new_value['after'] = $new_after; 119 | $new_value['before'] = $before; 120 | 121 | return $new_value; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogHistoryGtidChildUpdater.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 29 | $this->replication_db_model = new ReplicationDbModel($config); 30 | $this->replication_query = new ReplicationQuery($this->replication_db_model); 31 | $this->binlog_history_service = new BinlogHistoryService(); 32 | } 33 | 34 | private function getFetchCount(): int 35 | { 36 | if ($this->remain_binlog_count <= 0) { 37 | return 0; 38 | } 39 | $fetch_count = self::CHILD_ONCE_BINLOG_FETCH_LIMIT; 40 | if ($this->remain_binlog_count < self::CHILD_ONCE_BINLOG_FETCH_LIMIT) { 41 | $fetch_count = $this->remain_binlog_count; 42 | } 43 | $this->remain_binlog_count -= $fetch_count; 44 | 45 | return $fetch_count; 46 | } 47 | 48 | public function execute(int $last_binlog_id, int $max_binlog_count): void 49 | { 50 | $this->remain_binlog_count = $max_binlog_count; 51 | $fetch_count = $this->getFetchCount(); 52 | $is_first = true; 53 | while ($fetch_count > 0) { 54 | //최초 $last_binlog_id 포함 55 | if ($is_first) { 56 | $dicts = $this->binlog_history_service->getEmptyGtidBinlogDictsByLesserEqualId( 57 | $last_binlog_id, 58 | $fetch_count 59 | ); 60 | $is_first = false; 61 | } else { 62 | $dicts = $this->binlog_history_service->getEmptyGtidBinlogDictsByLesserId( 63 | $last_binlog_id, 64 | $fetch_count 65 | ); 66 | } 67 | $is_all_updated = $this->calculateGtidAndUpdate($dicts); 68 | if (!$is_all_updated) { 69 | break; 70 | } 71 | $dict_count = count($dicts); 72 | $last_binlog_id = $dicts[$dict_count - 1]['id']; 73 | $fetch_count = $this->getFetchCount(); 74 | } 75 | } 76 | 77 | private function calculateGtidAndUpdate(array $dicts): bool 78 | { 79 | $new_dicts = self::sortDescendingByBinlogFileNameAndGtidEndPos($dicts); 80 | 81 | foreach ($new_dicts as $dict) { 82 | $id = intval($dict['id']); 83 | $binlog_file_name = $dict['binlog_filename']; 84 | $binlog_position = intval($dict['gtid_end_pos']); 85 | try { 86 | $binlog_offset_dto = $this->replication_query->convertToBinlogOffsetDto( 87 | $binlog_file_name, 88 | $binlog_position 89 | ); 90 | $this->binlog_history_service->updateBinlogGtid($id, $binlog_offset_dto->mariadb_gtid); 91 | } catch (\Exception $exception) { 92 | $this->logger->info( 93 | "Gtid ({$id}:{$binlog_file_name},{$binlog_position}) " . 94 | "변환 실패(오래된 Binlog Offset인 경우 발생 할 수 있습니다.)" 95 | ); 96 | 97 | return false; 98 | } 99 | } 100 | 101 | return true; 102 | } 103 | 104 | public static function sortDescendingByBinlogFileNameAndGtidEndPos(array $dicts): array 105 | { 106 | usort( 107 | $dicts, 108 | function ($a, $b) { 109 | $ret_val = $b['binlog_filename'] <=> $a['binlog_filename']; 110 | if ($ret_val === 0) { 111 | return $b['gtid_end_pos'] <=> $a['gtid_end_pos']; 112 | } 113 | 114 | return $ret_val; 115 | } 116 | ); 117 | 118 | return $dicts; 119 | } 120 | 121 | public function close(): void 122 | { 123 | $this->replication_db_model->close(); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogHistoryService.php: -------------------------------------------------------------------------------- 1 | transactional($callable); 31 | } 32 | 33 | /** 34 | * @return OnlyGtidOffsetRangeDto[] 35 | */ 36 | public function getChildGtidOffsetRanges(): array 37 | { 38 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 39 | 40 | return $child_offset_model->getChildGtidOffsetRanges(); 41 | } 42 | 43 | public function getChildGtidOffsetRangeCount(): int 44 | { 45 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 46 | 47 | return $child_offset_model->getChildGtidOffsetRangeCount(); 48 | } 49 | 50 | public function upsertChildGtidOffsetRange( 51 | int $child_index, 52 | OnlyBinlogOffsetDto $current_binlog_offset_dto, 53 | OnlyBinlogOffsetDto $end_gtid_offset_dto, 54 | string $current_binlog_offset_date 55 | ): int { 56 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 57 | 58 | $affected_rows = $child_offset_model->upsertChildGtidOffsetRange( 59 | $child_index, 60 | $current_binlog_offset_dto, 61 | $end_gtid_offset_dto, 62 | $current_binlog_offset_date 63 | ); 64 | 65 | return $affected_rows; 66 | } 67 | 68 | public function insertChildGtidOffsetRange(OnlyGtidOffsetRangeDto $gtid_offset_range_dto): int 69 | { 70 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 71 | 72 | return $child_offset_model->insertChildGtidOffsetRange($gtid_offset_range_dto); 73 | } 74 | 75 | public function deleteAllChildGtidOffsetRanges(): int 76 | { 77 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 78 | 79 | return $child_offset_model->deleteAllChildGtidOffsetRanges(); 80 | } 81 | 82 | public function deleteChildGtidOffsetRangeById(int $child_index): int 83 | { 84 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 85 | 86 | return $child_offset_model->deleteChildGtidOffsetRangeById($child_index); 87 | } 88 | 89 | public function getMinCurrentBinlogPositionDate(): ?string 90 | { 91 | $child_offset_model = BinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 92 | 93 | return $child_offset_model->getMinCurrentBinlogPositionDate(); 94 | } 95 | 96 | /** 97 | * @param BinlogHistoryDto[] $dtos 98 | * 99 | * @return int 100 | */ 101 | public function insertHistoryBulk(array $dtos): int 102 | { 103 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 104 | 105 | return $binlog_history_model->insertHistoryBulk($dtos); 106 | } 107 | 108 | public function getEmptyGtidBinlogCount(): int 109 | { 110 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 111 | 112 | return $binlog_history_model->getEmptyGtidBinlogCount(); 113 | } 114 | 115 | public function getRecentEmptyGtidBinlogId(): int 116 | { 117 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 118 | 119 | return $binlog_history_model->getRecentEmptyGtidBinlogId(); 120 | } 121 | 122 | public function getEmptyGtidBinlogDictsByLesserEqualId(int $id, int $limit): array 123 | { 124 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 125 | 126 | return $binlog_history_model->getEmptyGtidBinlogDictsByLesserEqualId($id, $limit); 127 | } 128 | 129 | public function getEmptyGtidBinlogDictsByLesserId(int $id, int $limit): array 130 | { 131 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 132 | 133 | return $binlog_history_model->getEmptyGtidBinlogDictsByLesserId($id, $limit); 134 | } 135 | 136 | public function getEmptyGtidBinlogIdByLesserIdAndOffset(int $id, int $offset): int 137 | { 138 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 139 | 140 | return $binlog_history_model->getEmptyGtidBinlogIdByLesserIdAndOffset($id, $offset); 141 | } 142 | 143 | public function updateBinlogGtid(int $id, string $gtid): void 144 | { 145 | $binlog_history_model = BinlogHistoryModel::createBinlogHistoryWrite(); 146 | 147 | $binlog_history_model->updateBinlogGtid($id, $gtid); 148 | } 149 | 150 | public function getParentBinlogOffset(): OnlyBinlogOffsetDto 151 | { 152 | $parent_offset_model = BinlogHistoryParentOffsetModel::createBinlogHistoryWrite(); 153 | 154 | return $parent_offset_model->getParentBinlogOffset(); 155 | } 156 | 157 | public function getParentBinlogDate(): ?string 158 | { 159 | $parent_offset_model = BinlogHistoryParentOffsetModel::createBinlogHistoryWrite(); 160 | 161 | return $parent_offset_model->getParentBinlogDate(); 162 | } 163 | 164 | public function upsertParentBinlogOffset(OnlyBinlogOffsetDto $binlog_offset_dto, string $binlog_date = null): int 165 | { 166 | $parent_offset_model = BinlogHistoryParentOffsetModel::createBinlogHistoryWrite(); 167 | 168 | return $parent_offset_model->upsertParentBinlogOffset($binlog_offset_dto, $binlog_date); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /src/Binlog/Collector/BinlogPosFinder.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 28 | $this->replication_db_model = $replication_db_model; 29 | } 30 | 31 | /** 32 | * @param BinlogOffsetDto $start_binlog_offset_dto 33 | * 34 | * @param string $target_gtid ex) 0-146-1683252403,12-12-19056634,11-11-1887613118 35 | * @param string $skip_server_id 36 | * @param int $limit_previous_file_count 37 | * 38 | * @return BinlogOffsetDto|null 39 | */ 40 | public function findBinlogOffsetDto( 41 | BinlogOffsetDto $start_binlog_offset_dto, 42 | string $target_gtid, 43 | string $skip_server_id = self::NO_SKIP_SERVER_ID, 44 | int $limit_previous_file_count = 5 45 | ): ?BinlogOffsetDto { 46 | $target_gtids = explode(',', $target_gtid); 47 | $this->previous_file_count = 1; 48 | $this->limit_previous_file_count = $limit_previous_file_count; 49 | $binlog_file_name = $start_binlog_offset_dto->file_name; 50 | $dicts = $this->replication_db_model->showBinlogEventsFromInit($binlog_file_name, 0, self::ROW_COUNT); 51 | 52 | $dict_count = count($dicts); 53 | $previous_pos = 'init'; 54 | $previous_binlog_file_name = $binlog_file_name; 55 | $gtid = ''; 56 | while ($dict_count > 0) { 57 | foreach ($dicts as $dict) { 58 | if ($dict['Event_type'] === 'Gtid') { 59 | $gtid_end_log_pos = $dict['End_log_pos']; 60 | 61 | //BEGIN GTID 0-43-14532884 cid=97361316: 62 | //GTID 0-43-14535494: 63 | //BEGIN GTID 0-43-14535495: 64 | $gtid = explode(' ', trim(str_replace(['BEGIN', 'GTID'], '', $dict['Info'])))[0]; 65 | 66 | if (in_array($gtid, $target_gtids)) { 67 | $found_gtid = $this->replication_db_model->getBinlogGtidPos( 68 | $binlog_file_name, 69 | $gtid_end_log_pos 70 | ); 71 | if ($this->isSameGtidExceptSkipSeverId($target_gtid, $found_gtid, $skip_server_id)) { 72 | return BinlogOffsetDto::importBinlogOffset( 73 | $found_gtid, 74 | $binlog_file_name, 75 | $gtid_end_log_pos 76 | ); 77 | } 78 | } 79 | } 80 | } 81 | 82 | $next_pos = $dicts[$dict_count - 1]['End_log_pos']; 83 | if ($previous_binlog_file_name !== $binlog_file_name) { 84 | $previous_pos = 'init'; 85 | } 86 | $this->logger->info("no finding gtid: {$binlog_file_name} ({$previous_pos} ~ {$next_pos}) [{$gtid}]"); 87 | 88 | $previous_pos = $next_pos; 89 | $previous_binlog_file_name = $binlog_file_name; 90 | [$dicts, $binlog_file_name] = $this->getNextEventDictsAndPreviousFile($binlog_file_name, $next_pos, true); 91 | 92 | $dict_count = count($dicts); 93 | } 94 | 95 | return null; 96 | } 97 | 98 | public function isSameGtidExceptSkipSeverId(string $target_gtid, string $found_gtid, string $skip_server_id): bool 99 | { 100 | if ($skip_server_id === self::NO_SKIP_SERVER_ID) { 101 | return self::sortGtidList($target_gtid) === self::sortGtidList($found_gtid); 102 | } 103 | return self::removeSkipServerId($target_gtid, $skip_server_id) 104 | === self::removeSkipServerId($found_gtid, $skip_server_id); 105 | } 106 | 107 | public static function sortGtidList(string $gtid_list): string 108 | { 109 | $new_gtids = explode(',', $gtid_list); 110 | sort($new_gtids); 111 | 112 | return implode(',', $new_gtids); 113 | } 114 | 115 | public static function removeSkipServerId(string $gtid, string $skip_server_id): string 116 | { 117 | $target_gtids = explode(',', $gtid); 118 | 119 | $pattern = '/^(?:[0-9]+)?-([0-9]+)-(?:[0-9]+)?$/'; 120 | 121 | $new_gtids = []; 122 | foreach ($target_gtids as $target_gtid) { 123 | $is_found = (preg_match($pattern, $target_gtid, $matches)); 124 | if ($is_found && $skip_server_id === $matches[1]) { 125 | continue; 126 | } 127 | $new_gtids[] = $target_gtid; 128 | } 129 | sort($new_gtids); 130 | 131 | return implode(',', $new_gtids); 132 | } 133 | 134 | private function getNextEventDictsAndPreviousFile( 135 | string $binlog_file_name, 136 | int $pos, 137 | bool $use_previous_seq_if_failed 138 | ): array { 139 | $dicts = $this->replication_db_model->showBinlogEvents($binlog_file_name, $pos, 0, self::ROW_COUNT); 140 | if (count($dicts) === 0 141 | && $use_previous_seq_if_failed 142 | && $this->previous_file_count < $this->limit_previous_file_count 143 | ) { 144 | $this->previous_file_count++; 145 | $binlog_file_name = BinlogUtils::calculatePreviousSeqFile($binlog_file_name); 146 | $dicts = $this->replication_db_model->showBinlogEventsFromInit($binlog_file_name, 0, self::ROW_COUNT); 147 | } 148 | 149 | return [$dicts, $binlog_file_name]; 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Config/BinlogConfigFactory.php: -------------------------------------------------------------------------------- 1 | $v) 18 | { 19 | if ('user' === $k) 20 | { 21 | $configBuilder->withUser($v); 22 | } 23 | if ('ip' === $k || 'host' === $k) 24 | { 25 | $configBuilder->withHost($v); 26 | } 27 | if ('port' === $k) 28 | { 29 | $configBuilder->withPort($v); 30 | } 31 | if ('password' === $k) 32 | { 33 | $configBuilder->withPassword($v); 34 | } 35 | if ('charset' === $k) 36 | { 37 | $configBuilder->withCharset($v); 38 | } 39 | if ('gtid' === $k) 40 | { 41 | $configBuilder->withGtid($v); 42 | } 43 | if ('slaveId' === $k) 44 | { 45 | $configBuilder->withSlaveId($v); 46 | } 47 | if ('binLogFileName' === $k) 48 | { 49 | $configBuilder->withBinLogFileName($v); 50 | } 51 | if ('binLogPosition' === $k) 52 | { 53 | $configBuilder->withBinLogPosition($v); 54 | } 55 | if ('eventsOnly' === $k) 56 | { 57 | $configBuilder->withEventsOnly($v); 58 | } 59 | if ('eventsIgnore' === $k) 60 | { 61 | $configBuilder->withEventsIgnore($v); 62 | } 63 | if ('tablesOnly' === $k) 64 | { 65 | $configBuilder->withTablesOnly($v); 66 | } 67 | if ('databasesOnly' === $k) 68 | { 69 | $configBuilder->withDatabasesOnly($v); 70 | } 71 | if ('mariaDbGtid' === $k) 72 | { 73 | $configBuilder->withMariaDbGtid($v); 74 | } 75 | if ('tableCacheSize' === $k) 76 | { 77 | $configBuilder->withTableCacheSize($v); 78 | } 79 | if ('custom' === $k) 80 | { 81 | $configBuilder->withCustom($v); 82 | } 83 | if ('heartbeatPeriod' === $k) 84 | { 85 | $configBuilder->withHeartbeatPeriod($v); 86 | } 87 | } 88 | 89 | return $configBuilder->build(); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Config/BinlogConfiguration.php: -------------------------------------------------------------------------------- 1 | binlog_env_config = $binlog_env_config; 33 | $this->exception_handler = $exception_handler; 34 | $this->binlog_history_service = $binlog_history_service; 35 | $this->row_event_value_skipper = $row_event_value_skipper; 36 | $this->argv = $argv; 37 | } 38 | 39 | public static function newInstance( 40 | array $argv, 41 | BinlogEnvConfig $binlog_env_config, 42 | ExceptionHandlerInterface $exception_handler 43 | ): self { 44 | $self = new self( 45 | $argv, 46 | $binlog_env_config, 47 | $exception_handler, 48 | new BinlogHistoryService(), 49 | $binlog_env_config->row_event_value_skipper 50 | ); 51 | 52 | return $self; 53 | } 54 | 55 | public static function newInstanceForOnce( 56 | array $argv, 57 | BinlogEnvConfig $binlog_env_config, 58 | ExceptionHandlerInterface $exception_handler 59 | ): self { 60 | $self = new self( 61 | $argv, 62 | $binlog_env_config, 63 | $exception_handler, 64 | new OnceBinlogHistoryService(), 65 | $binlog_env_config->row_event_value_skipper 66 | ); 67 | 68 | return $self; 69 | } 70 | 71 | public function createPartitionerConfig(): BinlogPartitionerConfig 72 | { 73 | return BinlogPartitionerConfig::create( 74 | $this->binlog_env_config->binlog_connect_array, 75 | $this->binlog_env_config->binlog_config_array 76 | ); 77 | } 78 | 79 | public function createWorkerConfig(): BinlogWorkerConfig 80 | { 81 | return BinlogWorkerConfig::create( 82 | $this->binlog_env_config->binlog_connect_array, 83 | $this->binlog_env_config->binlog_config_array 84 | ); 85 | } 86 | 87 | public function extendWorkerConfig( 88 | array $replace_connect_array, 89 | array $replace_config_array 90 | ): BinlogWorkerConfig { 91 | $new_connect_array = array_merge($this->binlog_env_config->binlog_connect_array, $replace_connect_array); 92 | $new_config_array = array_merge($this->binlog_env_config->binlog_config_array, $replace_config_array); 93 | 94 | return BinlogWorkerConfig::create($new_connect_array, $new_config_array); 95 | } 96 | 97 | public function createConnectConfig(): Config 98 | { 99 | return BinlogConfigFactory::makeConfigFromArray($this->binlog_env_config->binlog_connect_array); 100 | } 101 | 102 | public static function createCustomConnectConfigWithReplace( 103 | array $binlog_connect_array, 104 | array $replace_connect_array 105 | ): Config { 106 | $new_connect_array = array_merge($binlog_connect_array, $replace_connect_array); 107 | 108 | return BinlogConfigFactory::makeConfigFromArray($new_connect_array); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Config/BinlogEnvConfig.php: -------------------------------------------------------------------------------- 1 | getenv('BINLOG_MYSQL_TARGET_HOST'), 34 | 'user' => getenv('BINLOG_MYSQL_TARGET_USER'), 35 | 'port' => getenv('BINLOG_MYSQL_TARGET_PORT'), 36 | 'password' => getenv('BINLOG_MYSQL_TARGET_PASSWORD'), 37 | 'dbname' => getenv('BINLOG_MYSQL_TARGET_DBNAME'), 38 | 'driver' => getenv('BINLOG_MYSQL_TARGET_DRIVER'), 39 | 'charset' => getenv('BINLOG_MYSQL_TARGET_CHARSET'), 40 | 'driverOptions' => [\PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8'], 41 | ]; 42 | } 43 | 44 | public static function getHistoryWriteDbParams(): array 45 | { 46 | return [ 47 | 'host' => getenv('BINLOG_MYSQL_HISTORY_WRITE_HOST'), 48 | 'user' => getenv('BINLOG_MYSQL_HISTORY_WRITE_USER'), 49 | 'port' => getenv('BINLOG_MYSQL_HISTORY_WRITE_PORT'), 50 | 'password' => getenv('BINLOG_MYSQL_HISTORY_WRITE_PASSWORD'), 51 | 'dbname' => getenv('BINLOG_MYSQL_HISTORY_WRITE_DBNAME'), 52 | 'driver' => getenv('BINLOG_MYSQL_HISTORY_WRITE_DRIVER'), 53 | 'charset' => getenv('BINLOG_MYSQL_HISTORY_WRITE_CHARSET'), 54 | 'driverOptions' => [\PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8'], 55 | ]; 56 | } 57 | 58 | public static function getConnectionParams(string $name): array 59 | { 60 | switch ($name) { 61 | case self::TARGET_DB: 62 | return self::getTargetBinlogDbParams(); 63 | case self::HISTORY_WRITE_DB: 64 | return self::getHistoryWriteDbParams(); 65 | } 66 | 67 | throw new MsgException("{$name} DB connection parameters are missing."); 68 | } 69 | 70 | public static function importDefaultConfig(RowEventValueSkipperInterface $row_event_value_skipper = null): self 71 | { 72 | return self::extendDefaultConfig([], $row_event_value_skipper); 73 | } 74 | 75 | public static function extendDefaultConfig( 76 | array $replace_binlog_config_array, 77 | RowEventValueSkipperInterface $row_event_value_skipper = null 78 | ): self { 79 | 80 | $default_binlog_connect_array = self::getDefaultBinlogConnectArray(); 81 | $default_binlog_config_array = self::getDefaultBinlogConfigArray(); 82 | 83 | $replace_binlog_connect_array = []; 84 | if ($row_event_value_skipper !== null) { 85 | $replace_binlog_connect_array['tablesOnly'] = $row_event_value_skipper->getTablesOnly(); 86 | $replace_binlog_connect_array['databasesOnly'] = $row_event_value_skipper->getDatabasesOnly(); 87 | 88 | } 89 | 90 | $self = new self(); 91 | $self->enable_sentry = (bool)getenv('BINLOG_ENABLE_SENTRY'); 92 | $self->sentry_key = getenv('BINLOG_SENTRY_KEY'); 93 | $self->binlog_connect_array = array_merge($default_binlog_connect_array, $replace_binlog_connect_array); 94 | $self->binlog_config_array = array_merge($default_binlog_config_array, $replace_binlog_config_array); 95 | $self->row_event_value_skipper = $row_event_value_skipper; 96 | 97 | return $self; 98 | } 99 | 100 | private static function getDefaultBinlogConnectArray(): array 101 | { 102 | return [ 103 | 'ip' => BinlogEnvConfig::getTargetBinlogDbParams()['host'], 104 | 'port' => BinlogEnvConfig::getTargetBinlogDbParams()['port'], 105 | 'user' => BinlogEnvConfig::getTargetBinlogDbParams()['user'], 106 | 'password' => BinlogEnvConfig::getTargetBinlogDbParams()['password'], 107 | 'charset' => BinlogEnvConfig::getTargetBinlogDbParams()['charset'], 108 | 'tablesOnly' => [], 109 | 'databasesOnly' => [], 110 | 'eventsIgnore' => [ConstEventType::FORMAT_DESCRIPTION_EVENT], 111 | ]; 112 | } 113 | 114 | private static function getDefaultBinlogConfigArray(): array 115 | { 116 | return [ 117 | 'gtid_partition_max_count' => 1000, 118 | 'jump_offset_for_next_partition' => 10000, 119 | 'is_all_print_event' => false, 120 | 'child_process_max_count' => 10, 121 | 'once_processed_max_event_count_in_gtid' => 100, 122 | 'gtid_count_for_persist_per_partition' => 500, 123 | ]; 124 | } 125 | 126 | public function validateTarget(): void 127 | { 128 | if (count($this->binlog_connect_array['tablesOnly']) === 0) { 129 | throw new MsgException('tablesOnly is empty'); 130 | } 131 | if (count($this->binlog_connect_array['databasesOnly']) === 0) { 132 | throw new MsgException('databasesOnly is empty'); 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Config/BinlogPartitionerConfig.php: -------------------------------------------------------------------------------- 1 | connect_config = $connect_config; 24 | $binlog_config->gtid_partition_max_count = intval($array['gtid_partition_max_count']); 25 | $binlog_config->jump_offset_for_next_partition = intval($array['jump_offset_for_next_partition']); 26 | $binlog_config->binlog_connect_array = $binlog_connect_array; 27 | $binlog_config->validate(); 28 | 29 | return $binlog_config; 30 | } 31 | 32 | public static function create(array $binlog_connect_array, array $binlog_config_array): self 33 | { 34 | $connect_config = BinlogConfigFactory::makeConfigFromArray($binlog_connect_array); 35 | 36 | return self::importFromInit($connect_config, $binlog_config_array, $binlog_connect_array); 37 | } 38 | 39 | public function validate(): void 40 | { 41 | if ($this->gtid_partition_max_count === 0) { 42 | throw new MsgException('gtid_partition_max_count is empty'); 43 | } 44 | if ($this->jump_offset_for_next_partition === 0) { 45 | throw new MsgException('jump_offset_for_next_partition is empty'); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Config/BinlogWorkerConfig.php: -------------------------------------------------------------------------------- 1 | connect_config = $connect_config; 27 | $binlog_config->child_index = intval($array['child_index']); 28 | $binlog_config->is_all_print_event = boolval($array['is_all_print_event']); 29 | $binlog_config->child_process_max_count = intval($array['child_process_max_count']); 30 | $binlog_config->once_processed_max_event_count_in_gtid 31 | = intval($array['once_processed_max_event_count_in_gtid']); 32 | $binlog_config->gtid_count_for_persist_per_partition = intval($array['gtid_count_for_persist_per_partition']); 33 | 34 | $binlog_config->validate(); 35 | 36 | return $binlog_config; 37 | } 38 | 39 | public static function create(array $binlog_connect_array, array $binlog_config_array): self 40 | { 41 | $connect_config = BinlogConfigFactory::makeConfigFromArray($binlog_connect_array); 42 | 43 | return self::importFromInit($connect_config, $binlog_config_array); 44 | } 45 | 46 | public function validate(): void 47 | { 48 | if ($this->child_process_max_count === 0) { 49 | throw new MsgException('child_process_max_count is empty'); 50 | } 51 | if ($this->once_processed_max_event_count_in_gtid === 0) { 52 | throw new MsgException('once_processed_max_event_count_in_gtid is empty'); 53 | } 54 | if ($this->gtid_count_for_persist_per_partition === 0) { 55 | throw new MsgException('gtid_count_for_persist_per_partition is empty'); 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Dto/BinlogHistoryDto.php: -------------------------------------------------------------------------------- 1 | gtid = $binlog_offset_dto->getBinlogKey(); 57 | $dto->binlog_offset_dto = $binlog_offset_dto; 58 | $dto->table_name = $table_name; 59 | $dto->pk_ids = $pk_ids; 60 | $dto->action = $action; 61 | $dto->data_dict = $data_dict; 62 | $dto->reg_date = $reg_date; 63 | $dto->event_index = $event_index; 64 | 65 | return $dto; 66 | } 67 | 68 | public function getGtid(): string 69 | { 70 | return 'bin|' . $this->gtid . '|' . $this->event_index; 71 | } 72 | 73 | public function exportBinlogInfoDatabaseVer3(): array 74 | { 75 | $dict = [ 76 | 'binlog_filename' => $this->binlog_offset_dto->file_name, 77 | 'gtid_end_pos' => $this->binlog_offset_dto->position, 78 | 'reg_date' => $this->reg_date, 79 | ]; 80 | if ($this->binlog_id !== null) { 81 | $dict['id'] = $this->binlog_id; 82 | } 83 | 84 | return $dict; 85 | } 86 | 87 | public function findBinlogIdKeyVer3(): string 88 | { 89 | return $this->binlog_offset_dto->file_name . $this->binlog_offset_dto->position . $this->reg_date; 90 | } 91 | 92 | public function exportRowInfoDatabaseVer3(): array 93 | { 94 | $dict = [ 95 | 'binlog_id' => $this->binlog_id, 96 | 'idx' => $this->event_index, 97 | 'table_name' => $this->table_name, 98 | 'pk_ids' => $this->pk_ids, 99 | 'action' => $this->action, 100 | 'reg_date' => $this->reg_date, 101 | ]; 102 | if ($this->row_id !== null) { 103 | $dict['id'] = $this->row_id; 104 | } 105 | 106 | return $dict; 107 | } 108 | 109 | public function findRowIdKeyVer3(): string 110 | { 111 | return $this->binlog_id . $this->event_index . $this->table_name . $this->pk_ids; 112 | } 113 | 114 | public function exportColumnInfoWithRowIdDatabaseVer3(): array 115 | { 116 | $dtos = []; 117 | if ($this->action === ConstEventsNames::WRITE) { 118 | foreach ($this->data_dict as $key => $value) { 119 | $dtos[] = [ 120 | 'row_id' => $this->row_id, 121 | 'column' => $key, 122 | 'data_before' => null, 123 | 'data_after' => $value, 124 | ]; 125 | } 126 | } elseif ($this->action === ConstEventsNames::UPDATE) { 127 | $before_dict = $this->data_dict['before']; 128 | $after_dict = $this->data_dict['after']; 129 | if (is_array($after_dict)) { 130 | foreach ($after_dict as $key => $value) { 131 | $dtos[] = [ 132 | 'row_id' => $this->row_id, 133 | 'column' => $key, 134 | 'data_before' => $before_dict[$key], 135 | 'data_after' => $value, 136 | ]; 137 | } 138 | } 139 | } elseif ($this->action === ConstEventsNames::DELETE) { 140 | foreach ($this->data_dict as $key => $value) { 141 | $dtos[] = [ 142 | 'row_id' => $this->row_id, 143 | 'column' => $key, 144 | 'data_before' => $value, 145 | 'data_after' => null, 146 | ]; 147 | } 148 | } 149 | 150 | return $dtos; 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Dto/BinlogOffsetDto.php: -------------------------------------------------------------------------------- 1 | mariadb_gtid = $mariadb_gtid; 14 | $dto->file_name = $file_name; 15 | $dto->position = $position; 16 | 17 | return $dto; 18 | } 19 | 20 | public function __toString(): string 21 | { 22 | return "[{$this->mariadb_gtid}/{$this->file_name}/{$this->position}]"; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Dto/GtidOffsetRangeDto.php: -------------------------------------------------------------------------------- 1 | convertToBinlogOffsetDto( 29 | $gtid_range_dto->current_dto->file_name, 30 | $gtid_range_dto->current_dto->position 31 | ); 32 | $end_gtid_offset_dto = $replication_query->convertToBinlogOffsetDto( 33 | $gtid_range_dto->end_dto->file_name, 34 | $gtid_range_dto->end_dto->position 35 | ); 36 | 37 | return self::importFromInit($child_index, $start_gtid_offset_dto, $end_gtid_offset_dto); 38 | } 39 | 40 | private static function importFromInit(int $child_index, BinLogOffsetDto $start_dto, BinLogOffsetDto $end_dto): self 41 | { 42 | $dto = new self(); 43 | $dto->child_index = $child_index; 44 | $dto->start_dto = $start_dto; 45 | $dto->end_dto = $end_dto; 46 | 47 | return $dto; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Dto/OnlyBinlogOffsetDto.php: -------------------------------------------------------------------------------- 1 | file_name = $file_name; 23 | $dto->position = $position; 24 | $dto->date = $date; 25 | 26 | return $dto; 27 | } 28 | 29 | public function __toString(): string 30 | { 31 | if ($this->date === null) { 32 | return "[{$this->file_name}/{$this->position}]"; 33 | } 34 | 35 | return "[{$this->date}/{$this->file_name}/{$this->position}]"; 36 | } 37 | 38 | public function getBinlogKey(): string 39 | { 40 | return $this->file_name . '|' . $this->position; 41 | } 42 | 43 | /** 44 | * If the OnlyBinlogOffsetDto is equal to the argument then 0 is returned. 45 | * If the OnlyBinlogOffsetDto is less than the argument then -1 is returned. 46 | * If the OnlyBinlogOffsetDto is greater than the argument then 1 is returned. 47 | * 48 | * @param OnlyBinlogOffsetDto $targetDto 49 | * 50 | * @return int 51 | */ 52 | public function compareTo(OnlyBinlogOffsetDto $targetDto): int 53 | { 54 | $current_seq = BinlogUtils::getSeqByBinlogFileName($this->file_name); 55 | $target_seq = BinlogUtils::getSeqByBinlogFileName($targetDto->file_name); 56 | 57 | if ($current_seq < $target_seq) { 58 | return -1; 59 | } 60 | if ($current_seq > $target_seq) { 61 | return 1; 62 | } 63 | 64 | if ($this->position < $targetDto->position) { 65 | return -1; 66 | } 67 | if ($this->position > $targetDto->position) { 68 | return 1; 69 | } 70 | 71 | return 0; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Dto/OnlyGtidOffsetRangeDto.php: -------------------------------------------------------------------------------- 1 | current_dto->file_name, 20 | $this->current_dto->position 21 | ); 22 | } 23 | 24 | public function exportToEndBinlogOffset(): OnlyBinlogOffsetDto 25 | { 26 | return OnlyBinlogOffsetDto::importOnlyBinlogOffset( 27 | $this->end_dto->file_name, 28 | $this->end_dto->position 29 | ); 30 | } 31 | 32 | public function __toString(): string 33 | { 34 | return "[{$this->child_index}. " . 35 | "({$this->current_date}/{$this->current_dto->file_name}/{$this->current_dto->position})~" . 36 | "({$this->end_dto->file_name}/{$this->end_dto->position})]"; 37 | } 38 | 39 | public function exportDatabase(): array 40 | { 41 | return [ 42 | 'current_bin_log_file_name' => $this->current_dto->file_name, 43 | 'current_bin_log_position' => $this->current_dto->position, 44 | 'end_bin_log_file_name' => $this->end_dto->file_name, 45 | 'end_bin_log_position' => $this->end_dto->position, 46 | 'current_bin_log_position_date' => $this->current_date, 47 | ]; 48 | } 49 | 50 | public static function importFromBinlogOffsets( 51 | OnlyBinlogOffsetDto $current_dto, 52 | OnlyBinlogOffsetDto $end_dto, 53 | string $current_date = null 54 | ): self { 55 | $dto = new self(); 56 | 57 | $dto->current_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset( 58 | $current_dto->file_name, 59 | $current_dto->position 60 | ); 61 | $dto->end_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset( 62 | $end_dto->file_name, 63 | $end_dto->position 64 | ); 65 | 66 | $dto->current_date = $current_date; 67 | 68 | return $dto; 69 | } 70 | 71 | public static function importFromDict(array $dict): self 72 | { 73 | $dto = new self(); 74 | if (isset($dict['child_index'])) { 75 | $dto->child_index = $dict['child_index']; 76 | } 77 | $dto->current_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset( 78 | $dict['current_bin_log_file_name'], 79 | $dict['current_bin_log_position'] 80 | ); 81 | $dto->end_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset( 82 | $dict['end_bin_log_file_name'], 83 | $dict['end_bin_log_position'] 84 | ); 85 | 86 | $dto->current_date = $dict['current_bin_log_position_date']; 87 | 88 | return $dto; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Exception/BinlogFinishedException.php: -------------------------------------------------------------------------------- 1 | target_tables = $target_tables; 22 | $this->target_databases = $target_databases; 23 | } 24 | 25 | /** 26 | * @return string[] 27 | */ 28 | public function getTablesOnly(): array 29 | { 30 | return $this->target_tables; 31 | } 32 | 33 | /** 34 | * @return string[] 35 | */ 36 | public function getDatabasesOnly(): array 37 | { 38 | return $this->target_databases; 39 | } 40 | 41 | abstract public function isTargetEventValue( 42 | int $binlog_event_timestamp, 43 | string $table, 44 | string $type, 45 | array $value 46 | ): bool; 47 | } 48 | -------------------------------------------------------------------------------- /src/Binlog/Collector/External/ExceptionHandlerInterface.php: -------------------------------------------------------------------------------- 1 | logger = self::createLogger($dir, $name); 30 | if ($binlog_env_config->enable_sentry) { 31 | $this->enableSentry($binlog_env_config->sentry_key); 32 | } 33 | 34 | set_error_handler( 35 | function ($severity, $message, $file, $line): void { 36 | if (!(error_reporting() & $severity)) { 37 | // This error code is not included in error_reporting 38 | return; 39 | } 40 | throw new \ErrorException($message, 0, $severity, $file, $line); 41 | } 42 | ); 43 | 44 | set_exception_handler( 45 | function (\Throwable $exception): void { 46 | $this->logger->info('file: ' . $exception->getFile() . "\n"); 47 | $this->logger->info('message: ' . $exception->getMessage() . "\n"); 48 | $this->logger->info('errorCode: ' . $exception->getCode() . "\n"); 49 | $this->logger->info('trace: ' . $exception->getTraceAsString() . "\n"); 50 | 51 | if ($exception instanceof \Exception) { 52 | self::triggerException($exception); 53 | } elseif ($exception instanceof \Error) { 54 | $error_exception = new \ErrorException( 55 | $exception->getMessage(), 56 | $exception->getCode(), 57 | 0, 58 | $exception->getFile(), 59 | $exception->getLine() 60 | ); 61 | self::triggerException($error_exception); 62 | } 63 | } 64 | ); 65 | } 66 | 67 | /** 68 | * @param string $dir 69 | * @param string $name 70 | * 71 | * @return Logger 72 | */ 73 | private static function createLogger(string $dir, string $name): Logger 74 | { 75 | $file_name = 'cron_' . $name . self::LOG_FILE_NAME_EXTENSION; 76 | 77 | $logger = new Logger($name); 78 | $logger->pushHandler(new StreamHandler("php://stdout")); 79 | $logger->pushHandler(new StreamHandler($dir . $file_name)); 80 | 81 | return $logger; 82 | } 83 | 84 | private function enableSentry(string $sentry_key): void 85 | { 86 | $raven_client = new BinlogRavenClient($sentry_key, ['processors' => []]); 87 | 88 | $GLOBALS[self::DEFAULT_RAVEN_CLIENT_NAME] = $raven_client; 89 | } 90 | 91 | public function triggerException(\Exception $e): bool 92 | { 93 | if (!$this->hasRavenClientInitialized()) { 94 | return false; 95 | } 96 | 97 | $client = $this->getRavenClient(); 98 | if (!($client instanceof Raven_Client)) { 99 | return false; 100 | } 101 | 102 | $client->captureException($e); 103 | 104 | return true; 105 | } 106 | 107 | public function triggerMessage(string $string, array $params = [], array $level_or_options = []): bool 108 | { 109 | if (!$this->hasRavenClientInitialized()) { 110 | return false; 111 | } 112 | 113 | $client = $this->getRavenClient(); 114 | if (!($client instanceof Raven_Client)) { 115 | return false; 116 | } 117 | 118 | $client->captureMessage($string, $params, $level_or_options, true); 119 | 120 | return true; 121 | } 122 | 123 | private function getRavenClient(): ?Raven_Client 124 | { 125 | return $this->hasRavenClientInitialized() ? $GLOBALS[self::DEFAULT_RAVEN_CLIENT_NAME] : null; 126 | } 127 | 128 | private function hasRavenClientInitialized(): bool 129 | { 130 | return isset($GLOBALS[self::DEFAULT_RAVEN_CLIENT_NAME]); 131 | } 132 | 133 | public function getLogger(): Logger 134 | { 135 | return $this->logger; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/Binlog/Collector/External/RowEventValueSkipperInterface.php: -------------------------------------------------------------------------------- 1 | getMessage() === MySQLReplicationException::DISCONNECTED_MESSAGE 33 | ) { 34 | return; 35 | } 36 | $data['level'] = self::ERROR; 37 | 38 | parent::captureException($exception, $data, $logger, $vars); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Interfaces/BinlogHistoryServiceInterface.php: -------------------------------------------------------------------------------- 1 | ping() === false) { 23 | $connection->close(); 24 | $connection->connect(); 25 | } 26 | 27 | return $connection; 28 | } 29 | 30 | protected static function createConnection(string $group_name): Connection 31 | { 32 | $connection = DriverManager::getConnection(BinlogEnvConfig::getConnectionParams($group_name)); 33 | $connection->setFetchMode(\PDO::FETCH_OBJ); 34 | 35 | return $connection; 36 | } 37 | 38 | protected static function closeAllConnections(): void 39 | { 40 | foreach (self::$connection_pool as $connection) { 41 | $connection->close(); 42 | } 43 | self::$connection_pool = []; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Library/DB/GnfConnectionProvider.php: -------------------------------------------------------------------------------- 1 | lock_file_name = $filtered_name . '.lock'; 19 | } 20 | 21 | /** 22 | * lock 얻고 실패하면 throw 23 | * @throws \RuntimeException 24 | */ 25 | public function lock(): void 26 | { 27 | $this->tryLock(); 28 | 29 | if (!$this->isHasLock()) { 30 | throw new \RuntimeException($this->lock_file_name . ' already exists, exiting'); 31 | } 32 | } 33 | 34 | /** 35 | * lock 얻기 시도하고 성공하면 true, 실패하면 false 36 | * @return bool 37 | */ 38 | public function tryLock(): bool 39 | { 40 | $this->lock_file = fopen(sys_get_temp_dir() . '/' . $this->lock_file_name, 'w+'); 41 | $this->is_has_lock = flock($this->lock_file, LOCK_EX | LOCK_NB); 42 | 43 | return $this->is_has_lock; 44 | } 45 | 46 | public function unlock(): void 47 | { 48 | if ($this->isHasLock()) { 49 | flock($this->lock_file, LOCK_UN); 50 | @fclose($this->lock_file); 51 | 52 | $this->is_has_lock = false; 53 | } 54 | } 55 | 56 | /** 57 | * @return bool 현재 lock을 가지고 있는지 여부 58 | */ 59 | public function isHasLock(): bool 60 | { 61 | return $this->is_has_lock; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Model/BinlogHistoryBaseModel.php: -------------------------------------------------------------------------------- 1 | db = $db; 19 | } 20 | 21 | /** 22 | * @param base $db 23 | * 24 | * @return static 25 | */ 26 | private static function create(base $db) 27 | { 28 | return new static($db); 29 | } 30 | 31 | /** 32 | * @return static 33 | */ 34 | public static function createBinlogHistoryWrite() 35 | { 36 | return self::create(GnfConnectionProvider::getGnfConnection(BinlogEnvConfig::HISTORY_WRITE_DB)); 37 | } 38 | 39 | public function transactional(callable $callable): bool 40 | { 41 | return $this->db->transactional($callable); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Model/BinlogHistoryChildOffsetModel.php: -------------------------------------------------------------------------------- 1 | db->sqlDicts( 16 | 'SELECT * from platform_universal_history_child_offset order by child_index' 17 | ); 18 | 19 | $dtos = []; 20 | foreach ($dicts as $dict) { 21 | $dtos[] = OnlyGtidOffsetRangeDto::importFromDict($dict); 22 | } 23 | 24 | return $dtos; 25 | } 26 | 27 | public function getChildGtidOffsetRangeCount(): int 28 | { 29 | return $this->db->sqlData('SELECT count(*) from platform_universal_history_child_offset'); 30 | } 31 | 32 | public function upsertChildGtidOffsetRange( 33 | int $child_index, 34 | OnlyBinlogOffsetDto $current_binlog_offset_dto, 35 | OnlyBinlogOffsetDto $end_gtid_offset_dto, 36 | string $current_binlog_offset_date 37 | ): int { 38 | $array = [ 39 | 'child_index' => $child_index, 40 | 'current_bin_log_file_name' => $current_binlog_offset_dto->file_name, 41 | 'current_bin_log_position' => $current_binlog_offset_dto->position, 42 | 'end_bin_log_file_name' => $end_gtid_offset_dto->file_name, 43 | 'end_bin_log_position' => $end_gtid_offset_dto->position, 44 | 'current_bin_log_position_date' => $current_binlog_offset_date, 45 | ]; 46 | 47 | return $this->db->sqlInsertOrUpdate('platform_universal_history_child_offset', $array); 48 | } 49 | 50 | public function insertChildGtidOffsetRange(OnlyGtidOffsetRangeDto $gtid_offset_range_dto): int 51 | { 52 | return $this->db->sqlInsert( 53 | 'platform_universal_history_child_offset', 54 | $gtid_offset_range_dto->exportDatabase() 55 | ); 56 | } 57 | 58 | public function deleteAllChildGtidOffsetRanges(): int 59 | { 60 | $where = [ 61 | 'child_index' => sqlNot(''), 62 | ]; 63 | 64 | return $this->db->sqlDelete('platform_universal_history_child_offset', $where); 65 | } 66 | 67 | public function deleteChildGtidOffsetRangeById(int $child_index): int 68 | { 69 | $where = [ 70 | 'child_index' => $child_index, 71 | ]; 72 | 73 | return $this->db->sqlDelete('platform_universal_history_child_offset', $where); 74 | } 75 | 76 | public function getMinCurrentBinlogPositionDate(): ?string 77 | { 78 | return $this->db->sqlData( 79 | 'SELECT MIN(current_bin_log_position_date) FROM platform_universal_history_child_offset' 80 | ); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Model/BinlogHistoryParentOffsetModel.php: -------------------------------------------------------------------------------- 1 | db->sqlDict( 14 | 'SELECT * FROM platform_universal_history_offset WHERE ?', 15 | sqlWhere(['offset_type' => self::CURRENT_OFFSET]) 16 | ); 17 | 18 | return OnlyBinlogOffsetDto::importOnlyBinlogOffset( 19 | $dict['end_bin_log_file_name'], 20 | $dict['end_bin_log_position'], 21 | $dict['end_bin_log_date'] 22 | ); 23 | } 24 | 25 | public function getParentBinlogDate(): ?string 26 | { 27 | return $this->db->sqlData( 28 | 'SELECT end_bin_log_date FROM platform_universal_history_offset WHERE ?', 29 | sqlWhere(['offset_type' => self::CURRENT_OFFSET]) 30 | ); 31 | } 32 | 33 | public function upsertParentBinlogOffset(OnlyBinlogOffsetDto $binlog_offset_dto, string $binlog_date = null): int 34 | { 35 | $datas = [ 36 | 'end_bin_log_file_name' => $binlog_offset_dto->file_name, 37 | 'end_bin_log_position' => $binlog_offset_dto->position, 38 | 'end_bin_log_date' => $binlog_date, 39 | 'offset_type' => self::CURRENT_OFFSET, 40 | ]; 41 | 42 | return $this->db->sqlInsertOrUpdate('platform_universal_history_offset', $datas); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Model/OnceBinlogHistoryChildOffsetModel.php: -------------------------------------------------------------------------------- 1 | db->sqlDicts( 16 | 'SELECT * from platform_once_history_child_offset order by child_index' 17 | ); 18 | 19 | $dtos = []; 20 | foreach ($dicts as $dict) { 21 | $dtos[] = OnlyGtidOffsetRangeDto::importFromDict($dict); 22 | } 23 | 24 | return $dtos; 25 | } 26 | 27 | public function getChildGtidOffsetRangeCount(): int 28 | { 29 | return $this->db->sqlData('SELECT count(*) from platform_once_history_child_offset'); 30 | } 31 | 32 | public function upsertChildGtidOffsetRange( 33 | int $child_index, 34 | OnlyBinlogOffsetDto $current_binlog_offset_dto, 35 | OnlyBinlogOffsetDto $end_gtid_offset_dto, 36 | string $current_binlog_offset_date 37 | ): int { 38 | $array = [ 39 | 'child_index' => $child_index, 40 | 'current_bin_log_file_name' => $current_binlog_offset_dto->file_name, 41 | 'current_bin_log_position' => $current_binlog_offset_dto->position, 42 | 'end_bin_log_file_name' => $end_gtid_offset_dto->file_name, 43 | 'end_bin_log_position' => $end_gtid_offset_dto->position, 44 | 'current_bin_log_position_date' => $current_binlog_offset_date, 45 | ]; 46 | 47 | return $this->db->sqlInsertOrUpdate('platform_once_history_child_offset', $array); 48 | } 49 | 50 | public function insertChildGtidOffsetRange(OnlyGtidOffsetRangeDto $gtid_offset_range_dto): int 51 | { 52 | return $this->db->sqlInsert( 53 | 'platform_once_history_child_offset', 54 | $gtid_offset_range_dto->exportDatabase() 55 | ); 56 | } 57 | 58 | public function deleteAllChildGtidOffsetRanges(): int 59 | { 60 | $where = [ 61 | 'child_index' => sqlNot(''), 62 | ]; 63 | 64 | return $this->db->sqlDelete('platform_once_history_child_offset', $where); 65 | } 66 | 67 | public function deleteChildGtidOffsetRangeById(int $child_index): int 68 | { 69 | $where = [ 70 | 'child_index' => $child_index, 71 | ]; 72 | 73 | return $this->db->sqlDelete('platform_once_history_child_offset', $where); 74 | } 75 | 76 | public function getMinCurrentBinlogPositionDate(): ?string 77 | { 78 | return $this->db->sqlData('SELECT MIN(current_bin_log_position_date) FROM platform_once_history_child_offset'); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Model/OnceBinlogHistoryParentOffsetModel.php: -------------------------------------------------------------------------------- 1 | db->sqlDict( 14 | 'SELECT * FROM platform_once_history_offset WHERE ?', 15 | sqlWhere(['offset_type' => self::CURRENT_OFFSET]) 16 | ); 17 | 18 | return OnlyBinlogOffsetDto::importOnlyBinlogOffset( 19 | $dict['end_bin_log_file_name'], 20 | $dict['end_bin_log_position'], 21 | $dict['end_bin_log_date'] 22 | ); 23 | } 24 | 25 | public function getParentBinlogDate(): ?string 26 | { 27 | return $this->db->sqlData( 28 | 'SELECT end_bin_log_date FROM platform_once_history_offset WHERE ?', 29 | sqlWhere(['offset_type' => self::CURRENT_OFFSET]) 30 | ); 31 | } 32 | 33 | public function upsertParentBinlogOffset(OnlyBinlogOffsetDto $binlog_offset_dto, string $binlog_date = null): int 34 | { 35 | $datas = [ 36 | 'end_bin_log_file_name' => $binlog_offset_dto->file_name, 37 | 'end_bin_log_position' => $binlog_offset_dto->position, 38 | 'end_bin_log_date' => $binlog_date, 39 | 'offset_type' => self::CURRENT_OFFSET, 40 | ]; 41 | 42 | return $this->db->sqlInsertOrUpdate('platform_once_history_offset', $datas); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Model/ReplicationDbModel.php: -------------------------------------------------------------------------------- 1 | connection = $this->initConnection($config); 17 | } 18 | 19 | private function initConnection(Config $config): Connection 20 | { 21 | $config->validate(); 22 | 23 | return DriverManager::getConnection( 24 | [ 25 | 'user' => $config->getUser(), 26 | 'password' => $config->getPassword(), 27 | 'host' => $config->getHost(), 28 | 'port' => empty($config->getPort()) ? 3306 : $config->getPort(), 29 | 'driver' => 'pdo_mysql', 30 | 'charset' => $config->getCharset(), 31 | ] 32 | ); 33 | } 34 | 35 | public function getConnection(): Connection 36 | { 37 | if (false === $this->connection->ping()) { 38 | $this->connection->close(); 39 | $this->connection->connect(); 40 | } 41 | 42 | return $this->connection; 43 | } 44 | 45 | public function close(): void 46 | { 47 | $this->connection->close(); 48 | } 49 | 50 | public function getBinlogGtidPos(string $binlog_filename, int $binlog_offset): string 51 | { 52 | $gtid = $this->getConnection()->fetchAssoc( 53 | "SELECT BINLOG_GTID_POS(\"{$binlog_filename}\", {$binlog_offset} ) as mariadb_gtid" 54 | )['mariadb_gtid']; 55 | 56 | return $gtid ?? ''; 57 | } 58 | 59 | public function showMasterStatus(): array 60 | { 61 | return $this->getConnection()->fetchAssoc("SHOW MASTER STATUS"); 62 | } 63 | 64 | public function showBinlogEvents(string $log_name, int $pos, int $offset = 0, int $row_count = 1000): array 65 | { 66 | try { 67 | return $this->getConnection()->fetchAll( 68 | "SHOW BINLOG EVENTS IN '{$log_name}' FROM {$pos} LIMIT {$offset}, {$row_count}" 69 | ); 70 | } catch (\Throwable $e) { 71 | return []; 72 | } 73 | } 74 | 75 | public function showBinlogEventsFromInit(string $log_name, int $offset = 0, int $row_count = 1000): array 76 | { 77 | try { 78 | return $this->getConnection()->fetchAll( 79 | "SHOW BINLOG EVENTS IN '{$log_name}' LIMIT {$offset}, {$row_count}" 80 | ); 81 | } catch (\Throwable $e) { 82 | return []; 83 | } 84 | } 85 | 86 | public function showBinlogEventsUsingThrowException(string $log_name): array 87 | { 88 | return $this->getConnection()->fetchAll("SHOW BINLOG EVENTS IN '{$log_name}' LIMIT 0, 1"); 89 | } 90 | 91 | public function getTableNames(array $table_schemas): array 92 | { 93 | $sql = "SELECT DISTINCT `TABLE_NAME` FROM `information_schema`.`COLUMNS`"; 94 | 95 | $where = ''; 96 | if (count($table_schemas) > 0) { 97 | $where = 'WHERE `TABLE_SCHEMA` in ( "' . implode('","', $table_schemas) . '")'; 98 | } 99 | 100 | return $this->getConnection()->fetchAll($sql . $where); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Monitor/Constants/TimeMonitorConst.php: -------------------------------------------------------------------------------- 1 | id = intval($dict['id']); 20 | $dto->type = $dict['type']; 21 | $dto->elapsed_time = intval($dict['elapsed_time']); 22 | $dto->reg_date = $dict['reg_date']; 23 | 24 | return $dto; 25 | } 26 | 27 | public static function importFromElapsedTime(string $type, int $elapsed_time): self 28 | { 29 | $dto = new self(); 30 | 31 | $dto->type = $type; 32 | $dto->elapsed_time = $elapsed_time; 33 | 34 | return $dto; 35 | } 36 | 37 | public function exportToDatabase(): array 38 | { 39 | return [ 40 | 'type' => $this->type, 41 | 'elapsed_time' => $this->elapsed_time 42 | ]; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Monitor/Model/BinlogTimeMonitorModel.php: -------------------------------------------------------------------------------- 1 | db->sqlInsert('platform_time_monitor', $monitor_dto->exportToDatabase()); 13 | 14 | return $this->db->insert_id(); 15 | } 16 | 17 | public function getTimeMonitor(int $id): ?TimeMonitorDto 18 | { 19 | $dict = $this->db->sqlDict('SELECT * FROM platform_time_monitor WHERE ?', sqlWhere(['id' => $id])); 20 | 21 | return ($dict !== null) ? TimeMonitorDto::importFromDatabase($dict) : null; 22 | } 23 | 24 | public function deleteTimeMonitor(int $id): int 25 | { 26 | return $this->db->sqlDelete('platform_time_monitor', ['id' => $id]); 27 | } 28 | 29 | /** 30 | * @param string $type {@link TimeMonitorConst} 31 | * 32 | * @return string|null 33 | */ 34 | public function getLastTimeMonitor(string $type): ?string 35 | { 36 | $where = ['type' => $type]; 37 | 38 | return $this->db->sqlData('SELECT MAX(reg_date) FROM platform_time_monitor WHERE ?', sqlWhere($where)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Monitor/TimeMonitor.php: -------------------------------------------------------------------------------- 1 | insertTimeMonitor($monitor_dto); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Monitor/TimeMonitorService.php: -------------------------------------------------------------------------------- 1 | insertTimeMonitor($monitor_dto); 15 | } 16 | 17 | public static function getTimeMonitor(int $monitor_id): ?TimeMonitorDto 18 | { 19 | return BinlogTimeMonitorModel::createBinlogHistoryWrite()->getTimeMonitor($monitor_id); 20 | } 21 | 22 | public static function deleteTimeMonitor(int $id): int 23 | { 24 | return BinlogTimeMonitorModel::createBinlogHistoryWrite()->deleteTimeMonitor($id); 25 | } 26 | 27 | /** 28 | * @param string $type {@link TimeMonitorConst} 29 | * 30 | * @return string|null 31 | */ 32 | public static function getLastTimeMonitor(string $type): ?string 33 | { 34 | return BinlogTimeMonitorModel::createBinlogHistoryWrite()->getLastTimeMonitor($type); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/Binlog/Collector/OnceBinlogHistoryService.php: -------------------------------------------------------------------------------- 1 | transactional($callable); 31 | } 32 | 33 | /** 34 | * @return OnlyGtidOffsetRangeDto[] 35 | */ 36 | public function getChildGtidOffsetRanges(): array 37 | { 38 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 39 | 40 | return $child_offset_model->getChildGtidOffsetRanges(); 41 | } 42 | 43 | public function getChildGtidOffsetRangeCount(): int 44 | { 45 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 46 | 47 | return $child_offset_model->getChildGtidOffsetRangeCount(); 48 | } 49 | 50 | public function upsertChildGtidOffsetRange( 51 | int $child_index, 52 | OnlyBinlogOffsetDto $current_binlog_offset_dto, 53 | OnlyBinlogOffsetDto $end_gtid_offset_dto, 54 | string $current_binlog_offset_date 55 | ): int { 56 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 57 | 58 | $affected_rows = $child_offset_model->upsertChildGtidOffsetRange( 59 | $child_index, 60 | $current_binlog_offset_dto, 61 | $end_gtid_offset_dto, 62 | $current_binlog_offset_date 63 | ); 64 | 65 | return $affected_rows; 66 | } 67 | 68 | public function insertChildGtidOffsetRange(OnlyGtidOffsetRangeDto $gtid_offset_range_dto): int 69 | { 70 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 71 | 72 | return $child_offset_model->insertChildGtidOffsetRange($gtid_offset_range_dto); 73 | } 74 | 75 | public function deleteAllChildGtidOffsetRanges(): int 76 | { 77 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 78 | 79 | return $child_offset_model->deleteAllChildGtidOffsetRanges(); 80 | } 81 | 82 | public function deleteChildGtidOffsetRangeById(int $child_index): int 83 | { 84 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 85 | 86 | return $child_offset_model->deleteChildGtidOffsetRangeById($child_index); 87 | } 88 | 89 | public function getMinCurrentBinlogPositionDate(): ?string 90 | { 91 | $child_offset_model = OnceBinlogHistoryChildOffsetModel::createBinlogHistoryWrite(); 92 | 93 | return $child_offset_model->getMinCurrentBinlogPositionDate(); 94 | } 95 | 96 | /** 97 | * insert Universal History Bulk 98 | * 99 | * @param BinlogHistoryDto[] $dtos 100 | * 101 | * @return int 102 | */ 103 | public function insertHistoryBulk(array $dtos): int 104 | { 105 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 106 | 107 | return $binlog_history_model->insertHistoryBulk($dtos); 108 | } 109 | 110 | public function getEmptyGtidBinlogCount(): int 111 | { 112 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 113 | 114 | return $binlog_history_model->getEmptyGtidBinlogCount(); 115 | } 116 | 117 | public function getRecentEmptyGtidBinlogId(): int 118 | { 119 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 120 | 121 | return $binlog_history_model->getRecentEmptyGtidBinlogId(); 122 | } 123 | 124 | public function getEmptyGtidBinlogDictsByLesserEqualId(int $id, int $limit): array 125 | { 126 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 127 | 128 | return $binlog_history_model->getEmptyGtidBinlogDictsByLesserEqualId($id, $limit); 129 | } 130 | 131 | public function getEmptyGtidBinlogDictsByLesserId(int $id, int $limit): array 132 | { 133 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 134 | 135 | return $binlog_history_model->getEmptyGtidBinlogDictsByLesserId($id, $limit); 136 | } 137 | 138 | public function getEmptyGtidBinlogIdByLesserIdAndOffset(int $id, int $offset): int 139 | { 140 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 141 | 142 | return $binlog_history_model->getEmptyGtidBinlogIdByLesserIdAndOffset($id, $offset); 143 | } 144 | 145 | public function updateBinlogGtid(int $id, string $gtid): void 146 | { 147 | $binlog_history_model = OnceBinlogHistoryModel::createBinlogHistoryWrite(); 148 | 149 | $binlog_history_model->updateBinlogGtid($id, $gtid); 150 | } 151 | 152 | public function getParentBinlogOffset(): OnlyBinlogOffsetDto 153 | { 154 | $parent_offset_model = OnceBinlogHistoryParentOffsetModel::createBinlogHistoryWrite(); 155 | 156 | return $parent_offset_model->getParentBinlogOffset(); 157 | } 158 | 159 | public function getParentBinlogDate(): ?string 160 | { 161 | $parent_offset_model = OnceBinlogHistoryParentOffsetModel::createBinlogHistoryWrite(); 162 | 163 | return $parent_offset_model->getParentBinlogDate(); 164 | } 165 | 166 | public function upsertParentBinlogOffset(OnlyBinlogOffsetDto $binlog_offset_dto, string $binlog_date = null): int 167 | { 168 | $parent_offset_model = OnceBinlogHistoryParentOffsetModel::createBinlogHistoryWrite(); 169 | 170 | return $parent_offset_model->upsertParentBinlogOffset($binlog_offset_dto, $binlog_date); 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/Binlog/Collector/ReplicationQuery.php: -------------------------------------------------------------------------------- 1 | replication_db_model = $replication_db_model; 19 | } 20 | 21 | /** 22 | * @param string $file_name 23 | * @param int $position 24 | * @param bool $use_strict_check 25 | * 26 | * @return OnlyBinlogOffsetDto 27 | * @throws MsgException 28 | */ 29 | public function convertToOnlyBinlogOffsetDto( 30 | string $file_name, 31 | int $position, 32 | bool $use_strict_check = false 33 | ): OnlyBinlogOffsetDto { 34 | if ($use_strict_check) { 35 | $mariadb_gtid = $this->getMariaDbGtid($file_name, $position); 36 | if (empty($mariadb_gtid)) { 37 | $file_name = BinlogUtils::calculateNextSeqFile($file_name); 38 | } 39 | } 40 | 41 | return OnlyBinlogOffsetDto::importOnlyBinlogOffset($file_name, $position); 42 | } 43 | 44 | /** 45 | * @param string $file_name 46 | * @param int $position 47 | * 48 | * @return BinlogOffsetDto 49 | * @throws MsgException 50 | */ 51 | public function convertToBinlogOffsetDto(string $file_name, int $position): BinlogOffsetDto 52 | { 53 | $mariadb_gtid = $this->getMariaDbGtid($file_name, $position); 54 | if (empty($mariadb_gtid)) { 55 | throw new MsgException("can't calculate mariaDb_gtid (using SELECT BINLOG_GTID_POS"); 56 | } 57 | 58 | return BinlogOffsetDto::importBinlogOffset($mariadb_gtid, $file_name, $position); 59 | } 60 | 61 | /** 62 | * @param string $file_name 63 | * @param int $position 64 | * 65 | * @return string 66 | * @throws MsgException 67 | */ 68 | private function getMariaDbGtid(string $file_name, int $position): string 69 | { 70 | if ($this->replication_db_model === null) { 71 | throw new MsgException('replication_db_model is null'); 72 | } 73 | 74 | return $this->replication_db_model->getBinlogGtidPos($file_name, $position); 75 | } 76 | 77 | /** 78 | * @return BinlogOffsetDto 79 | * @throws MsgException 80 | */ 81 | public function getMasterBinlogOffset(): BinlogOffsetDto 82 | { 83 | $result = $this->replication_db_model->showMasterStatus(); 84 | if (empty($result)) { 85 | throw new MsgException("can't execute 'show master status'"); 86 | } 87 | 88 | return $this->convertToBinlogOffsetDto($result['File'], $result['Position']); 89 | } 90 | 91 | public function showBinlogEvents(string $log_name, int $pos, int $offset = 0, int $row_count = 1000): array 92 | { 93 | return $this->replication_db_model->showBinlogEvents($log_name, $pos, $offset, $row_count); 94 | } 95 | 96 | public function showBinlogEventsFromInit(string $log_name, int $offset = 0, int $row_count = 1000): array 97 | { 98 | return $this->replication_db_model->showBinlogEventsFromInit($log_name, $offset, $row_count); 99 | } 100 | 101 | public function getBinlogGtidPos(string $binlog_filename, int $binlog_offset): string 102 | { 103 | return $this->replication_db_model->getBinlogGtidPos($binlog_filename, $binlog_offset); 104 | } 105 | 106 | /** 107 | * assertCheckAuth 108 | * @throws MsgException 109 | */ 110 | public function assertCheckAuth(): void 111 | { 112 | $result = $this->replication_db_model->showMasterStatus(); 113 | if (empty($result)) { 114 | throw new MsgException("can't execute 'show master status'"); 115 | } 116 | $log_name = $result['File']; 117 | $this->replication_db_model->showBinlogEventsUsingThrowException($log_name); 118 | } 119 | 120 | /** 121 | * @param string[] $target_table_schemas 122 | * @param string[] $target_table_names 123 | * 124 | * @throws MsgException 125 | */ 126 | public function assertSelectTables(array $target_table_schemas, array $target_table_names): void 127 | { 128 | $table_names = $this->replication_db_model->getTableNames($target_table_schemas); 129 | $tables_names = collect($table_names)->pluck('TABLE_NAME')->all(); 130 | 131 | $diff_tables = array_diff($target_table_names, $tables_names); 132 | 133 | if (count($diff_tables) > 0) { 134 | throw new MsgException("can't find: " . implode(',', $diff_tables) . ' in information_schema.COLUMNS'); 135 | } 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Subscriber/GetInitBinlogDateSubscriber.php: -------------------------------------------------------------------------------- 1 | getType() === ConstEventsNames::FORMAT_DESCRIPTION) { 21 | return; 22 | } 23 | $timestamp = $event->getEventInfo()->getTimestamp(); 24 | $this->current_binlog_date = (new \DateTime())->setTimestamp($timestamp)->format('Y-m-d H:i:s'); 25 | } 26 | 27 | public function getCurrentBinlogDate(): ?string 28 | { 29 | return $this->current_binlog_date; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/Binlog/Collector/Utils/BinlogUtils.php: -------------------------------------------------------------------------------- 1 | mariadb-bin.000016 11 | * mariadb-bin.999999 -> mariadb-bin.000000 12 | * 13 | * @param string $binlog_file_name 14 | * 15 | * @return string 16 | */ 17 | public static function calculateNextSeqFile(string $binlog_file_name): string 18 | { 19 | $pos = strrpos($binlog_file_name, '.'); 20 | if ($pos !== false) { 21 | $prefix_filename = substr($binlog_file_name, 0, $pos); 22 | $zero_pad_sequence = substr($binlog_file_name, $pos + 1); 23 | $next_sequence = intval($zero_pad_sequence) + 1; 24 | 25 | if (strlen(strval($next_sequence)) > strlen($zero_pad_sequence)) { 26 | $next_sequence = 0; 27 | } 28 | 29 | $new_suffix = str_pad($next_sequence, strlen($zero_pad_sequence), '0', STR_PAD_LEFT); 30 | 31 | return "{$prefix_filename}.{$new_suffix}"; 32 | } 33 | 34 | return ''; 35 | } 36 | 37 | public static function getSeqByBinlogFileName(string $binlog_file_name): int 38 | { 39 | $pos = strrpos($binlog_file_name, '.'); 40 | if ($pos !== false) { 41 | $zero_pad_sequence = substr($binlog_file_name, $pos + 1); 42 | 43 | return intval($zero_pad_sequence); 44 | } 45 | throw new MsgException("invalid binlog_file_name: {$binlog_file_name}"); 46 | } 47 | 48 | public static function calculatePreviousSeqFile(string $binlog_file_name): string 49 | { 50 | $pos = strrpos($binlog_file_name, '.'); 51 | if ($pos !== false) { 52 | $prefix_filename = substr($binlog_file_name, 0, $pos); 53 | $zero_pad_sequence = substr($binlog_file_name, $pos + 1); 54 | $current_sequence = intval($zero_pad_sequence); 55 | 56 | if ($current_sequence === 0) { 57 | $new_suffix = str_pad('', strlen($zero_pad_sequence), '9', STR_PAD_LEFT); 58 | } else { 59 | $new_suffix = str_pad($current_sequence - 1, strlen($zero_pad_sequence), '0', STR_PAD_LEFT); 60 | } 61 | 62 | return "{$prefix_filename}.{$new_suffix}"; 63 | } 64 | 65 | return ''; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /tests/.env.test: -------------------------------------------------------------------------------- 1 | BINLOG_ENABLE_SENTRY=0 2 | #BINLOG_SENTRY_KEY= 3 | 4 | BINLOG_MYSQL_TARGET_HOST=127.0.0.1 5 | BINLOG_MYSQL_TARGET_USER=repl 6 | BINLOG_MYSQL_TARGET_PORT=13306 7 | BINLOG_MYSQL_TARGET_PASSWORD=1234 8 | BINLOG_MYSQL_TARGET_DBNAME=php_mysql_replication 9 | BINLOG_MYSQL_TARGET_DRIVER=pdo_mysql 10 | BINLOG_MYSQL_TARGET_CHARSET=utf8 11 | 12 | BINLOG_MYSQL_HISTORY_WRITE_HOST=127.0.0.1 13 | BINLOG_MYSQL_HISTORY_WRITE_USER=testUser 14 | BINLOG_MYSQL_HISTORY_WRITE_PORT=13307 15 | BINLOG_MYSQL_HISTORY_WRITE_PASSWORD=testUser 16 | BINLOG_MYSQL_HISTORY_WRITE_DBNAME=platform 17 | BINLOG_MYSQL_HISTORY_WRITE_DRIVER=pdo_mysql 18 | BINLOG_MYSQL_HISTORY_WRITE_CHARSET=utf8 19 | -------------------------------------------------------------------------------- /tests/Binlog/BinlogCollectorTest.php: -------------------------------------------------------------------------------- 1 | getBinlogCount(['id' => sqlNot(sqlNull())]); 39 | $binlog_row_count = $binlog_history_model->getBinlogRowCount(['u_row.id' => sqlNot(sqlNull())]); 40 | $binlog_col_count = $binlog_history_model->getBinlogColumnCount(['u_column.id' => sqlNot(sqlNull())]); 41 | 42 | print("binlog_total_count: {$binlog_count}\n"); 43 | print("binlog_total_row_count: {$binlog_row_count}\n"); 44 | print("binlog_total_col_count: {$binlog_col_count}\n"); 45 | 46 | $this->assertEquals($binlog_count, self::BINLOG_TOTAL_COUNT); 47 | $this->assertEquals($binlog_row_count, self::BINLOG_ROW_COUNT); 48 | $this->assertEquals($binlog_col_count, self::BINLOG_COL_COUNT); 49 | 50 | $write_row_count = $binlog_history_model->getBinlogRowCount( 51 | ['u_row.table_name' => self::TARGET_TABLES, 'u_row.action' => 'write'] 52 | ); 53 | $update_row_count = $binlog_history_model->getBinlogRowCount( 54 | ['u_row.table_name' => self::TARGET_TABLES, 'u_row.action' => 'update'] 55 | ); 56 | $delete_row_count = $binlog_history_model->getBinlogRowCount( 57 | ['u_row.table_name' => self::TARGET_TABLES, 'u_row.action' => 'delete'] 58 | ); 59 | 60 | print("write_binlog_row_count: {$write_row_count}\n"); 61 | print("update_binlog_row_count: {$update_row_count}\n"); 62 | print("delete_binlog_row_count: {$delete_row_count}\n"); 63 | 64 | $this->assertEquals($write_row_count, self::WRITE_BINLOG_ROW_COUNT); 65 | $this->assertEquals($update_row_count, self::UPDATE_BINLOG_ROW_COUNT); 66 | $this->assertEquals($delete_row_count, self::DELETE_BINLOG_ROW_COUNT); 67 | 68 | $write_col_count = $binlog_history_model->getBinlogColumnCount( 69 | ['u_row.table_name' => self::TARGET_TABLES, 'u_row.action' => 'write'] 70 | ); 71 | $update_col_count = $binlog_history_model->getBinlogColumnCount( 72 | ['u_row.table_name' => self::TARGET_TABLES, 'u_row.action' => 'update'] 73 | ); 74 | $delete_col_count = $binlog_history_model->getBinlogColumnCount( 75 | ['u_row.table_name' => self::TARGET_TABLES, 'u_row.action' => 'delete'] 76 | ); 77 | 78 | print("write_binlog_col_count: {$write_col_count}\n"); 79 | print("update_binlog_col_count: {$update_col_count}\n"); 80 | print("delete_binlog_col_count: {$delete_col_count}\n"); 81 | 82 | $this->assertEquals($write_col_count, self::WRITE_BINLOG_COL_COUNT); 83 | $this->assertEquals($update_col_count, self::UPDATE_BINLOG_COL_COUNT); 84 | $this->assertEquals($delete_col_count, self::DELETE_BINLOG_COL_COUNT); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /tests/Binlog/BinlogHistoryGtidChildUpdaterTest.php: -------------------------------------------------------------------------------- 1 | makeBinlogDict(4, 'mariadb-bin.007350', 10000001, '', '2017-09-14 17:14:42'); 14 | $dicts[] = $this->makeBinlogDict(2, 'mariadb-bin.007351', 20000001, '', '2017-09-14 17:14:47'); 15 | $dicts[] = $this->makeBinlogDict(3, 'mariadb-bin.007350', 10000002, '', '2017-09-14 17:14:45'); 16 | $dicts[] = $this->makeBinlogDict(1, 'mariadb-bin.007351', 20000002, '', '2017-09-14 17:14:48'); 17 | 18 | $dicts = BinlogHistoryGtidChildUpdater::sortDescendingByBinlogFileNameAndGtidEndPos($dicts); 19 | $this->assertEquals(1, $dicts[0]['id']); 20 | $this->assertEquals(2, $dicts[1]['id']); 21 | $this->assertEquals(3, $dicts[2]['id']); 22 | $this->assertEquals(4, $dicts[3]['id']); 23 | } 24 | 25 | public function makeBinlogDict( 26 | int $id, 27 | string $binlog_filename, 28 | int $gtid_end_pos, 29 | string $gtid, 30 | string $reg_date 31 | ): array { 32 | return [ 33 | 'id' => $id, 34 | 'binlog_filename' => $binlog_filename, 35 | 'gtid_end_pos' => $gtid_end_pos, 36 | 'gtid' => $gtid, 37 | 'reg_date' => $reg_date, 38 | ]; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tests/Binlog/Utils/BinlogConfigurationTest.php: -------------------------------------------------------------------------------- 1 | extendWorkerConfig( 26 | [ 27 | 'slaveId' => '999', 28 | 'ip' => '127.0.0.2', 29 | ], 30 | [ 31 | 'child_index' => 1, 32 | ] 33 | ); 34 | $this->assertEquals('127.0.0.2', $new_binlog_worker_config->connect_config->getHost()); 35 | $this->assertEquals('999', $new_binlog_worker_config->connect_config->getSlaveId()); 36 | $this->assertEquals(1, $new_binlog_worker_config->child_index); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/Binlog/Utils/BinlogUtilsTest.php: -------------------------------------------------------------------------------- 1 | assertEquals('mariadb-bin.000001', $actual); 16 | 17 | $actual = BinlogUtils::calculateNextSeqFile('mariadb-bin.000015'); 18 | $this->assertEquals('mariadb-bin.000016', $actual); 19 | 20 | $actual = BinlogUtils::calculateNextSeqFile('mariadb-bin.999999'); 21 | $this->assertEquals('mariadb-bin.000000', $actual); 22 | } 23 | 24 | public function testCalculatePreviousSeqFile(): void 25 | { 26 | $actual = BinlogUtils::calculatePreviousSeqFile('mariadb-bin.000016'); 27 | $this->assertEquals('mariadb-bin.000015', $actual); 28 | 29 | $actual = BinlogUtils::calculatePreviousSeqFile('mariadb-bin.000001'); 30 | $this->assertEquals('mariadb-bin.000000', $actual); 31 | 32 | $actual = BinlogUtils::calculatePreviousSeqFile('mariadb-bin.000000'); 33 | $this->assertEquals('mariadb-bin.999999', $actual); 34 | } 35 | 36 | public function testConvertGtidBinLogInfoToGtid(): void 37 | { 38 | $gtid = explode(' ', trim(str_replace(['BEGIN', 'GTID'], '', 'GTID 0-43-14535494')))[0]; 39 | $this->assertEquals('0-43-14535494', $gtid); 40 | $gtid = explode(' ', trim(str_replace(['BEGIN', 'GTID'], '', 'BEGIN GTID 0-43-14532884 cid=97361316')))[0]; 41 | $this->assertEquals('0-43-14532884', $gtid); 42 | $gtid = explode(' ', trim(str_replace(['BEGIN', 'GTID'], '', 'BEGIN GTID 0-43-14535495')))[0]; 43 | $this->assertEquals('0-43-14535495', $gtid); 44 | } 45 | 46 | 47 | public function testRemoveSkipServerId(): void 48 | { 49 | $gtid = '0-146-1683252403,12-12-19056634,11-11-1887613118'; 50 | $this->assertEquals('0-146-1683252403,11-11-1887613118,12-12-19056634', BinlogPosFinder::sortGtidList($gtid)); 51 | 52 | $actual = BinlogPosFinder::removeSkipServerId($gtid, '146'); 53 | $this->assertEquals('11-11-1887613118,12-12-19056634', $actual); 54 | 55 | $actual = BinlogPosFinder::removeSkipServerId($gtid, '12'); 56 | $this->assertEquals('0-146-1683252403,11-11-1887613118', $actual); 57 | 58 | $actual = BinlogPosFinder::removeSkipServerId($gtid, '11'); 59 | $this->assertEquals('0-146-1683252403,12-12-19056634', $actual); 60 | } 61 | 62 | public function testGetSeqByBinlogFileName(): void 63 | { 64 | $this->assertEquals(16, BinLogUtils::getSeqByBinlogFileName('mariadb-bin.000016')); 65 | $this->assertEquals(0, BinLogUtils::getSeqByBinlogFileName('mariadb-bin.000000')); 66 | } 67 | 68 | public function testOnlyBinlogOffsetDtoCompareTo(): void 69 | { 70 | $only_binlog_offset_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset('mariadb-bin.000001', 11111); 71 | 72 | $this->assertEquals(0, $only_binlog_offset_dto->compareTo($only_binlog_offset_dto)); 73 | $target_binlog_offset_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset('mariadb-bin.000002', 11111); 74 | $this->assertEquals(-1, $only_binlog_offset_dto->compareTo($target_binlog_offset_dto)); 75 | $target_binlog_offset_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset('mariadb-bin.000000', 11111); 76 | $this->assertEquals(1, $only_binlog_offset_dto->compareTo($target_binlog_offset_dto)); 77 | 78 | 79 | $target_binlog_offset_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset('mariadb-bin.000001', 11112); 80 | $this->assertEquals(-1, $only_binlog_offset_dto->compareTo($target_binlog_offset_dto)); 81 | $target_binlog_offset_dto = OnlyBinlogOffsetDto::importOnlyBinlogOffset('mariadb-bin.000001', 11110); 82 | $this->assertEquals(1, $only_binlog_offset_dto->compareTo($target_binlog_offset_dto)); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /tests/bootstrap.php: -------------------------------------------------------------------------------- 1 | load(); 8 | } 9 | -------------------------------------------------------------------------------- /tests/phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | ./ 9 | 10 | 11 | 12 | 13 | src/ 14 | 15 | 16 | 17 | 18 | --------------------------------------------------------------------------------