├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── main.yml ├── .gitignore ├── .rspec ├── CODE_OF_CONDUCT.md ├── Gemfile ├── Gemfile.lock ├── LICENSE.txt ├── README.md ├── Rakefile ├── Steepfile ├── bin ├── console └── setup ├── lib ├── mysql_rewinder.rb └── mysql_rewinder │ ├── cleaner.rb │ ├── cleaner │ ├── adapter.rb │ ├── mysql2_adapter.rb │ └── trilogy_adapter.rb │ ├── ext │ ├── mysql2_client.rb │ └── trilogy.rb │ └── version.rb ├── mysql_rewinder.gemspec ├── sig ├── mysql_rewinder.rbs └── mysql_rewinder │ ├── cleaner.rbs │ ├── cleaner │ ├── adapter.rbs │ ├── mysql2_adapter.rbs │ └── trilogy_adapter.rbs │ ├── ext │ ├── mysql2_client.rbs │ └── trilogy.rbs │ ├── mysql2.rbs │ ├── trilogy.rbs │ └── version.rbs └── spec ├── mysql_rewinder_spec.rb └── spec_helper.rb /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Summary 2 | 3 | ### Other Information 4 | 5 | ### Checklist 6 | 7 | * [ ] By placing an "x" in the box, I hereby understand, accept and agree to be bound by the terms and conditions of the [Contribution License Agreement](https://dena.github.io/cla/). 8 | 9 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Ruby 2 | 3 | on: 4 | push: 5 | branches: 6 | - trunk 7 | 8 | pull_request: 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | name: Ruby ${{ matrix.ruby }} / MySQL ${{ matrix.mysql }} 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | ruby: 18 | - "3.0" 19 | - "3.1" 20 | - "3.2" 21 | - "3.3" 22 | - "head" 23 | mysql: 24 | - "5.7" 25 | - "8.0" 26 | services: 27 | db: 28 | image: mysql:${{ matrix.mysql }} 29 | ports: 30 | - 3306:3306 31 | env: 32 | MYSQL_ALLOW_EMPTY_PASSWORD: "true" 33 | options: >- 34 | --health-cmd "mysqladmin ping" 35 | --health-interval 10s 36 | --health-timeout 5s 37 | --health-retries 5 38 | steps: 39 | - uses: actions/checkout@v4 40 | - name: Set up Ruby 41 | uses: ruby/setup-ruby@v1 42 | with: 43 | ruby-version: ${{ matrix.ruby }} 44 | bundler-cache: true 45 | - name: Test and type check 46 | run: | 47 | bundle exec steep check 48 | bundle exec rspec 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | /.idea/ 10 | .DS_Store 11 | 12 | # rspec failure tracking 13 | .rspec_status 14 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities. 4 | 5 | We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality. 6 | 7 | Examples of unacceptable behavior by participants include: 8 | 9 | * The use of sexualized language or imagery 10 | * Personal attacks 11 | * Trolling or insulting/derogatory comments 12 | * Public or private harassment 13 | * Publishing other's private information, such as physical or electronic addresses, without explicit permission 14 | * Other unethical or unprofessional conduct 15 | 16 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. By adopting this Code of Conduct, project maintainers commit themselves to fairly and consistently applying these principles to every aspect of managing this project. Project maintainers who do not follow or enforce the Code of Conduct may be permanently removed from the project team. 17 | 18 | This code of conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. 19 | 20 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers. 21 | 22 | This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 1.2.0, available at https://www.contributor-covenant.org/version/1/2/0/code-of-conduct.html 23 | 24 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | # Specify your gem's dependencies in mysql_rewinder.gemspec 6 | gemspec 7 | 8 | gem "rake", "~> 13.0" 9 | gem "rspec", "~> 3.0" 10 | gem "steep" 11 | gem "trilogy" 12 | gem "mysql2" 13 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | mysql_rewinder (0.1.4) 5 | 6 | GEM 7 | remote: https://rubygems.org/ 8 | specs: 9 | activesupport (7.1.1) 10 | base64 11 | bigdecimal 12 | concurrent-ruby (~> 1.0, >= 1.0.2) 13 | connection_pool (>= 2.2.5) 14 | drb 15 | i18n (>= 1.6, < 2) 16 | minitest (>= 5.1) 17 | mutex_m 18 | tzinfo (~> 2.0) 19 | ast (2.4.2) 20 | base64 (0.2.0) 21 | bigdecimal (3.1.4) 22 | concurrent-ruby (1.2.2) 23 | connection_pool (2.4.1) 24 | csv (3.2.7) 25 | diff-lcs (1.5.0) 26 | drb (2.2.0) 27 | ruby2_keywords 28 | ffi (1.16.3) 29 | fileutils (1.7.2) 30 | i18n (1.14.1) 31 | concurrent-ruby (~> 1.0) 32 | json (2.6.3) 33 | language_server-protocol (3.17.0.3) 34 | listen (3.8.0) 35 | rb-fsevent (~> 0.10, >= 0.10.3) 36 | rb-inotify (~> 0.9, >= 0.9.10) 37 | logger (1.6.0) 38 | minitest (5.20.0) 39 | mutex_m (0.2.0) 40 | mysql2 (0.5.5) 41 | parser (3.2.2.4) 42 | ast (~> 2.4.1) 43 | racc 44 | racc (1.7.3) 45 | rainbow (3.1.1) 46 | rake (13.1.0) 47 | rb-fsevent (0.11.2) 48 | rb-inotify (0.10.1) 49 | ffi (~> 1.0) 50 | rbs (3.2.2) 51 | rspec (3.12.0) 52 | rspec-core (~> 3.12.0) 53 | rspec-expectations (~> 3.12.0) 54 | rspec-mocks (~> 3.12.0) 55 | rspec-core (3.12.2) 56 | rspec-support (~> 3.12.0) 57 | rspec-expectations (3.12.3) 58 | diff-lcs (>= 1.2.0, < 2.0) 59 | rspec-support (~> 3.12.0) 60 | rspec-mocks (3.12.6) 61 | diff-lcs (>= 1.2.0, < 2.0) 62 | rspec-support (~> 3.12.0) 63 | rspec-support (3.12.1) 64 | ruby2_keywords (0.0.5) 65 | securerandom (0.3.0) 66 | steep (1.5.3) 67 | activesupport (>= 5.1) 68 | concurrent-ruby (>= 1.1.10) 69 | csv (>= 3.0.9) 70 | fileutils (>= 1.1.0) 71 | json (>= 2.1.0) 72 | language_server-protocol (>= 3.15, < 4.0) 73 | listen (~> 3.0) 74 | logger (>= 1.3.0) 75 | parser (>= 3.1) 76 | rainbow (>= 2.2.2, < 4.0) 77 | rbs (>= 3.1.0) 78 | securerandom (>= 0.1) 79 | strscan (>= 1.0.0) 80 | terminal-table (>= 2, < 4) 81 | strscan (3.0.7) 82 | terminal-table (3.0.2) 83 | unicode-display_width (>= 1.1.1, < 3) 84 | trilogy (2.6.0) 85 | tzinfo (2.0.6) 86 | concurrent-ruby (~> 1.0) 87 | unicode-display_width (2.5.0) 88 | 89 | PLATFORMS 90 | arm64-darwin 91 | x86_64-linux 92 | 93 | DEPENDENCIES 94 | mysql2 95 | mysql_rewinder! 96 | rake (~> 13.0) 97 | rspec (~> 3.0) 98 | steep 99 | trilogy 100 | 101 | BUNDLED WITH 102 | 2.5.10 103 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2023 DeNA 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MysqlRewinder ![test](https://github.com/github/docs/actions/workflows/test.yml/badge.svg) [![Gem Version](https://badge.fury.io/rb/mysql_rewinder.svg)](https://badge.fury.io/rb/mysql_rewinder) 2 | 3 | MysqlRewinder is a simple, stable, and fast database cleaner for mysql. 4 | 5 | ## Features 6 | 7 | * Fast cleanup using `DELETE` query 8 | * Supports multi-database 9 | * Supports both `mysql2` and `trilogy` as a client library 10 | * Works without ActiveRecord 11 | * Works with `fork` 12 | 13 | ## How does it work? 14 | 15 | 1. Capture SQL statements during test execution and extract `INSERT`ed table names, and record them into temporary files 16 | 2. Aggregate tmp files and execute DELETE query for `INSERT`ed tables 17 | 18 | ## What does `stable` mean? 19 | 20 | MysqlRewinder is stable because it does not depend on ActiveRecord's internal implementation. 21 | It only depends on `Mysql2::Client#query` and `Trilogy#query`. 22 | 23 | ## Installation 24 | 25 | Add this line to your Gemfile's `:test` group: 26 | 27 | ```ruby 28 | gem 'trilogy' 29 | # gem 'mysql2' # described later 30 | gem 'mysql_rewinder' 31 | ``` 32 | 33 | And then execute: 34 | 35 | ```shell 36 | $ bundle 37 | ``` 38 | 39 | ## Usage 40 | 41 | ### Basic configuration 42 | 43 | ```ruby 44 | RSpec.configure do |config| 45 | config.before(:suite) do 46 | db_config = { 47 | host: '127.0.0.1', 48 | port: '3306', 49 | username: 'user1', 50 | password: 'my_secure_password', 51 | database: 'myapp-test' 52 | } 53 | MysqlRewinder.setup([db_config]) 54 | MysqlRewinder.clean_all 55 | end 56 | 57 | config.after(:each) do 58 | MysqlRewinder.clean 59 | end 60 | end 61 | ``` 62 | 63 | ### Multi-database 64 | 65 | Pass all configurations to `MysqlRewinder.setup`. 66 | 67 | ```ruby 68 | MysqlRewinder.setup( 69 | [ 70 | { host: '127.0.0.1', port: '3306', username: 'user1', password: 'my_secure_password', database: 'myapp-test-shard1' }, 71 | { host: '127.0.0.1', port: '3306', username: 'user1', password: 'my_secure_password', database: 'myapp-test-shard2' }, 72 | ] 73 | ) 74 | ``` 75 | 76 | ### mysql2 77 | 78 | If you want to use `mysql2` as a client library, do the following: 79 | 80 | * Write `gem 'mysql2'` in your `Gemfile` 81 | * Pass `adapter: :mysql2` to `MysqlRewinder.setup`. 82 | 83 | ```ruby 84 | MysqlRewinder.setup(db_configs, adapter: :mysql2) 85 | ``` 86 | 87 | ### ActiveRecord 88 | 89 | If you want to use MysqlRewinder with ActiveRecord, do the following: 90 | 91 | * Generate db_configs from `ActiveRecord::Base.configurations` 92 | * Pass `ActiveRecord::SchemaMigration.new(nil).table_name` and `ActiveRecord::Base.internal_metadata_table_name` to `MysqlRewinder.setup` as `except_tables` 93 | 94 | ```ruby 95 | db_configs = ActiveRecord::Base.configurations.configs_for(env_name: 'test').map(&:configuration_hash) 96 | except_tables = [ 97 | ActiveRecord::Base.internal_metadata_table_name, 98 | 99 | # for AR >= 7.1 100 | ActiveRecord::SchemaMigration.new(nil).table_name, 101 | # for AR < 7.1 102 | # ActiveRecord::SchemaMigration.table_name, 103 | ] 104 | 105 | MysqlRewinder.setup(db_configs, except_tables: except_tables) 106 | ``` 107 | 108 | ### Logging 109 | 110 | If you want to enable logging, specify `logger` for `MysqlRewinder.setup` 111 | 112 | ```ruby 113 | MysqlRewinder.setup(db_configs, logger: Logger.new(STDOUT)) 114 | ``` 115 | 116 | ## Contributing 117 | 118 | Bug reports and pull requests are welcome on GitHub at https://github.com/DeNA/mysql_rewinder. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/DeNA/mysql_rewinder/blob/trunk/CODE_OF_CONDUCT.md). 119 | 120 | ## Code of Conduct 121 | 122 | Everyone interacting in the MysqlRewinder project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/DeNA/mysql_rewinder/blob/trunk/CODE_OF_CONDUCT.md). 123 | 124 | ## License 125 | 126 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 127 | 128 | ## Special Thanks 129 | 130 | * Thank you [@aeroastro](https://github.com/aeroastro) for the idea of using temporary files 131 | * This gem is heavily inspired by [amatsuda/database_rewinder](https://github.com/amatsuda/database_rewinder). 132 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | require "rspec/core/rake_task" 5 | 6 | RSpec::Core::RakeTask.new(:spec) 7 | 8 | task default: %i[spec] 9 | -------------------------------------------------------------------------------- /Steepfile: -------------------------------------------------------------------------------- 1 | D = Steep::Diagnostic 2 | 3 | target :lib do 4 | signature "sig" 5 | 6 | check "lib" 7 | 8 | library "pathname" 9 | library "fileutils" 10 | library "tmpdir" 11 | library "forwardable" 12 | 13 | configure_code_diagnostics(D::Ruby.lenient) 14 | end 15 | 16 | target :test do 17 | signature "sig", "sig-private" 18 | 19 | check "test" 20 | end 21 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "bundler/setup" 5 | require "mysql_rewinder" 6 | 7 | # You can add fixtures and/or initialization code here to make experimenting 8 | # with your gem easier. You can also use a different console, if you like. 9 | 10 | # (If you use this, don't forget to add pry to your Gemfile!) 11 | # require "pry" 12 | # Pry.start 13 | 14 | require "irb" 15 | IRB.start(__FILE__) 16 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /lib/mysql_rewinder.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "mysql_rewinder/version" 4 | require_relative "mysql_rewinder/cleaner" 5 | require 'set' 6 | require 'tmpdir' 7 | require 'fileutils' 8 | require 'forwardable' 9 | require 'logger' 10 | 11 | class MysqlRewinder 12 | class << self 13 | extend Forwardable 14 | delegate %i[clean clean_all record_inserted_table] => :@instance 15 | 16 | def setup(db_configs, except_tables: [], adapter: :trilogy, logger: nil) 17 | @instance = new(db_configs: db_configs, except_tables: except_tables, adapter: adapter, logger: logger) 18 | end 19 | end 20 | 21 | def initialize(db_configs:, except_tables:, adapter:, logger:) 22 | @initialized_pid = Process.pid 23 | @inserted_table_record_dir = Pathname(Dir.tmpdir) 24 | @cleaners = db_configs.map do |db_config| 25 | Cleaner.new( 26 | db_config.transform_keys(&:to_sym), 27 | except_tables: except_tables, 28 | adapter: adapter, 29 | logger: logger 30 | ) 31 | end 32 | @logger = logger 33 | reset_inserted_tables 34 | end 35 | 36 | def record_inserted_table(sql) 37 | return unless @initialized_pid 38 | 39 | @inserted_tables ||= Set.new 40 | sql.split(';').each do |statement| 41 | match = statement.match(/\A\s*INSERT(?:\s+IGNORE)?(?:\s+INTO)?\s+(?:\.*[`"]?([^.\s`"(]+)[`"]?)*/i) 42 | next unless match 43 | 44 | table = match[1] 45 | @inserted_tables << table if table 46 | end 47 | File.write( 48 | @inserted_table_record_dir.join("#{@initialized_pid}.#{Process.pid}.inserted_tables").to_s, 49 | @inserted_tables.to_a.join(',') 50 | ) 51 | end 52 | 53 | def reset_inserted_tables 54 | unless @initialized_pid == Process.pid 55 | raise "MysqlRewinder is initialize in process #{@initialized_pid}, but reset_inserted_tables is called in process #{Process.pid}" 56 | end 57 | 58 | @inserted_tables = Set.new 59 | files = Dir.glob(@inserted_table_record_dir.join("#{@initialized_pid}.*.inserted_tables").to_s) 60 | 61 | FileUtils.rm(files) 62 | @logger&.debug { "[MysqlRewinder] removed files: #{files.join(', ')}" } if files.any? 63 | end 64 | 65 | def calculate_inserted_tables 66 | unless @initialized_pid == Process.pid 67 | raise "MysqlRewinder is initialize in process #{@initialized_pid}, but calculate_inserted_tables is called in process #{Process.pid}" 68 | end 69 | 70 | Dir.glob(@inserted_table_record_dir.join("#{@initialized_pid}.*.inserted_tables").to_s).flat_map do |fname| 71 | File.read(fname).strip.split(',') 72 | end.uniq 73 | end 74 | 75 | def clean_all 76 | @cleaners.each(&:clean_all) 77 | reset_inserted_tables 78 | end 79 | 80 | def clean 81 | aggregated_inserted_tables = calculate_inserted_tables 82 | @cleaners.each { |c| c.clean(tables: aggregated_inserted_tables) } 83 | reset_inserted_tables 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/mysql_rewinder/cleaner.rb: -------------------------------------------------------------------------------- 1 | require_relative 'cleaner/adapter' 2 | require_relative 'cleaner/mysql2_adapter' 3 | require_relative 'cleaner/trilogy_adapter' 4 | 5 | class MysqlRewinder 6 | class Cleaner 7 | attr_reader :db_config 8 | 9 | def initialize(db_config, except_tables:, adapter:, logger: nil) 10 | @db_config = db_config 11 | @client = Adapter.generate(adapter, db_config.transform_keys(&:to_sym)) 12 | @except_tables = except_tables 13 | @logger = logger 14 | end 15 | 16 | def clean_all 17 | clean(tables: all_tables) 18 | end 19 | 20 | def clean(tables:) 21 | target_tables = (tables - @except_tables) & all_tables 22 | 23 | if target_tables.empty? 24 | @logger&.debug { "[MysqlRewinder][#{@db_config[:database]}] Skip DELETE query because target_table is empty. tables: #{tables}, @except_tables: #{@except_tables}, all_tables: #{all_tables}." } 25 | return 26 | end 27 | 28 | log_and_execute("SET FOREIGN_KEY_CHECKS = 0;") 29 | log_and_execute(target_tables.map { |table| "DELETE FROM #{table}" }.join(';')) 30 | end 31 | 32 | def all_tables 33 | @all_tables ||= @client.query(<<~SQL).flatten 34 | SELECT TABLE_NAME 35 | FROM INFORMATION_SCHEMA.TABLES 36 | WHERE TABLE_SCHEMA = DATABASE() 37 | SQL 38 | end 39 | 40 | private 41 | 42 | def log_and_execute(sql) 43 | return @client.execute(sql) unless @logger&.debug? 44 | 45 | start_ts = Time.now 46 | res = @client.execute(sql) 47 | duration = (Time.now - start_ts) * 1000 48 | 49 | name = "[MysqlRewinder][#{@db_config[:database]}] Cleaner SQL (#{duration.round(1)}ms)" 50 | msg = "\e[1m\e[30m#{name}\e[0m \e[34m#{sql}\e[0m" 51 | @logger.debug msg 52 | res 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/mysql_rewinder/cleaner/adapter.rb: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | class Adapter 4 | def self.generate(adapter, config) 5 | case adapter 6 | when :trilogy 7 | require 'trilogy' 8 | require_relative '../ext/trilogy' 9 | 10 | TrilogyAdapter.new(config) 11 | when :mysql2 12 | require 'mysql2' 13 | require_relative '../ext/mysql2_client' 14 | 15 | Mysql2Adapter.new(config) 16 | else 17 | raise 'adapter must be either :trilogy or :mysql2' 18 | end 19 | end 20 | 21 | def initialize(_config); end 22 | 23 | def query(sql) 24 | raise NotImplementedError 25 | end 26 | 27 | def execute(sql) 28 | raise NotImplementedError 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/mysql_rewinder/cleaner/mysql2_adapter.rb: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | class Mysql2Adapter < Adapter 4 | def initialize(db_config) 5 | super 6 | @db_config = db_config 7 | connect 8 | end 9 | 10 | def query(sql) 11 | with_reconnect do 12 | @client.query(sql, as: :array).to_a 13 | end 14 | end 15 | 16 | def execute(sql) 17 | with_reconnect do 18 | @client.query(sql) 19 | @client.store_result while @client.next_result 20 | end 21 | end 22 | 23 | private 24 | def with_reconnect(&block) 25 | retry_count = 0 26 | begin 27 | block.call 28 | rescue Mysql2::Error => e 29 | raise e if retry_count > 3 30 | 31 | connect 32 | retry_count += 1 33 | 34 | retry 35 | end 36 | end 37 | 38 | def connect 39 | @client&.close 40 | @client = Mysql2::Client.new(@db_config.merge(connect_flags: Mysql2::Client::MULTI_STATEMENTS)) 41 | end 42 | end 43 | end 44 | end -------------------------------------------------------------------------------- /lib/mysql_rewinder/cleaner/trilogy_adapter.rb: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | class TrilogyAdapter < Adapter 4 | def initialize(db_config) 5 | super 6 | @db_config = db_config 7 | connect 8 | end 9 | 10 | def query(sql) 11 | with_reconnect do 12 | @client.query(sql).to_a 13 | end 14 | end 15 | 16 | def execute(sql) 17 | with_reconnect do 18 | @client.query(sql) 19 | @client.next_result while @client.more_results_exist? 20 | end 21 | end 22 | 23 | private 24 | def with_reconnect(&block) 25 | retry_count = 0 26 | begin 27 | block.call 28 | rescue Trilogy::Error => e 29 | raise e if retry_count > 3 30 | 31 | connect 32 | retry_count += 1 33 | 34 | retry 35 | end 36 | end 37 | 38 | def connect 39 | @client&.close 40 | @client = Trilogy.new(@db_config.merge(multi_statement: true)) 41 | end 42 | end 43 | end 44 | end -------------------------------------------------------------------------------- /lib/mysql_rewinder/ext/mysql2_client.rb: -------------------------------------------------------------------------------- 1 | require 'mysql2' 2 | 3 | class MysqlRewinder 4 | module Ext 5 | module Mysql2Client 6 | def query(sql, _options = {}) 7 | MysqlRewinder.record_inserted_table(sql) 8 | 9 | super 10 | end 11 | end 12 | end 13 | end 14 | ::Mysql2::Client.prepend ::MysqlRewinder::Ext::Mysql2Client 15 | -------------------------------------------------------------------------------- /lib/mysql_rewinder/ext/trilogy.rb: -------------------------------------------------------------------------------- 1 | require 'trilogy' 2 | 3 | class MysqlRewinder 4 | module Ext 5 | module Trilogy 6 | def query(sql) 7 | MysqlRewinder.record_inserted_table(sql) 8 | 9 | super 10 | end 11 | end 12 | end 13 | end 14 | ::Trilogy.prepend ::MysqlRewinder::Ext::Trilogy 15 | -------------------------------------------------------------------------------- /lib/mysql_rewinder/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MysqlRewinder 4 | VERSION = "0.1.4" 5 | end 6 | -------------------------------------------------------------------------------- /mysql_rewinder.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "lib/mysql_rewinder/version" 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "mysql_rewinder" 7 | spec.version = MysqlRewinder::VERSION 8 | spec.authors = ["Yusuke Sangenya"] 9 | spec.email = ["longinus.eva@gmail.com"] 10 | 11 | spec.summary = "Simple, stable, and fast database cleaner for mysql" 12 | spec.homepage = "https://github.com/genya0407/mysql_rewinder" 13 | spec.license = "MIT" 14 | spec.required_ruby_version = ">= 3.0.0" 15 | 16 | spec.files = Dir.chdir(__dir__) do 17 | `git ls-files -z`.split("\x0").reject do |f| 18 | (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)}) 19 | end 20 | end 21 | spec.require_paths = ["lib"] 22 | end 23 | -------------------------------------------------------------------------------- /sig/mysql_rewinder.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | self.@instance: MysqlRewinder 3 | @initialized_pid: Integer 4 | @inserted_table_record_dir: Object 5 | # @inserted_table_record_dir: Pathname 6 | @cleaners: Array[Cleaner] 7 | @inserted_tables: Set[String] 8 | @logger: untyped 9 | 10 | def self.setup: (Array[Hash[Symbol,String]] db_configs, ?except_tables: Array[String], ?adapter: ::Symbol, ?logger: untyped) -> void 11 | def self.clean_all: () -> void 12 | def self.clean: () -> void 13 | def self.record_inserted_table: (String sql) -> void 14 | 15 | def initialize: (db_configs: Array[Hash[Symbol,String]], except_tables: Array[String], adapter: ::Symbol, logger: untyped) -> untyped 16 | def record_inserted_table: (String sql) -> void 17 | def reset_inserted_tables: () -> void 18 | def calculate_inserted_tables: () -> void 19 | def clean_all: () -> void 20 | def clean: () -> void 21 | end 22 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/cleaner.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | @db_config: Hash[Symbol,String] 4 | @client: Adapter 5 | @except_tables: Array[String] 6 | @all_tables: Array[String] 7 | @logger: untyped 8 | 9 | attr_reader db_config: Hash[Symbol,String] 10 | 11 | def initialize: (Hash[Symbol,String] db_config, except_tables: Array[String], adapter: Symbol, ?logger: untyped) -> untyped 12 | def clean_all: () -> void 13 | def clean: (tables: untyped) -> void 14 | def all_tables: () -> void 15 | 16 | private 17 | 18 | def log_and_execute: (String sql) -> void 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/cleaner/adapter.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | class Adapter 4 | def self.generate: (Symbol adapter, Hash[Symbol,String] db_config) -> Adapter 5 | def initialize: (Hash[Symbol,String] db_config) -> untyped 6 | def query: (String sql) -> Array[Array[Object]] 7 | def execute: (String sql) -> void 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/cleaner/mysql2_adapter.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | class Mysql2Adapter < Adapter 4 | @db_config: Hash[Symbol,String] 5 | @client: Mysql2::Client 6 | 7 | private 8 | 9 | def with_reconnect: [T] () { () -> T } -> T 10 | def connect: () -> void 11 | end 12 | end 13 | end -------------------------------------------------------------------------------- /sig/mysql_rewinder/cleaner/trilogy_adapter.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | class Cleaner 3 | class TrilogyAdapter < Adapter 4 | @db_config: Hash[Symbol,String] 5 | @client: ::Trilogy 6 | 7 | private 8 | 9 | def with_reconnect: [T] () { () -> T } -> T 10 | def connect: () -> void 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/ext/mysql2_client.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | module Ext 3 | module Mysql2Client 4 | def query: (String sql, ?::Hash[Symbol, Object] _options) -> ::Mysql2::Result 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/ext/trilogy.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | module Ext 3 | module Trilogy 4 | def query: (String sql) -> ::Trilogy::Result 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/mysql2.rbs: -------------------------------------------------------------------------------- 1 | class Mysql2 2 | class Result < Enumerator[Array[Object], void] 3 | end 4 | class Error < StandardError 5 | end 6 | 7 | class Client 8 | MULTI_STATEMENTS: Integer 9 | 10 | def initialize: (Hash[Symbol, Object] config) -> untyped 11 | def close: () -> void 12 | def next_result: () -> bool 13 | def store_result: () -> Array[Object] 14 | def query: (String sql, ?Hash[Symbol, Object] _options) -> Array[Array[Object]] 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/trilogy.rbs: -------------------------------------------------------------------------------- 1 | class Trilogy 2 | class Result < Enumerator[Array[Object], void] 3 | end 4 | class Error < StandardError 5 | end 6 | 7 | def initialize: (Hash[Symbol,Object] config) -> untyped 8 | def close: () -> void 9 | def more_results_exist?: () -> bool 10 | def next_result: () -> Array[Object] 11 | def query: (String sql) -> Array[Array[Object]] 12 | end 13 | -------------------------------------------------------------------------------- /sig/mysql_rewinder/version.rbs: -------------------------------------------------------------------------------- 1 | class MysqlRewinder 2 | VERSION: String 3 | end 4 | -------------------------------------------------------------------------------- /spec/mysql_rewinder_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe MysqlRewinder do 2 | %i[mysql2 trilogy].each do |adapter| 3 | describe "adapter: #{adapter}" do 4 | let(:root_db_config) { { host: '127.0.0.1', port: '3306', username: 'root', database: 'mysql' } } 5 | let(:client_class) { { mysql2: Mysql2::Client, trilogy: Trilogy }[adapter] } 6 | 7 | def root_client 8 | client_class.new(root_db_config) 9 | end 10 | 11 | context 'when initialized with multiple configs' do 12 | let(:db_configs) do 13 | [ 14 | { host: '127.0.0.1', port: '3306', username: 'database_rewinder_test_user1', database: 'database_rewinder_test_a' }, 15 | { host: '127.0.0.1', port: '3306', username: 'database_rewinder_test_user2', database: 'database_rewinder_test_b' }, 16 | { host: '127.0.0.1', port: '3306', username: 'database_rewinder_test_user2', database: 'database_rewinder_test_c' } 17 | ] 18 | end 19 | 20 | before do 21 | %w[a b c].each do |suffix| 22 | <<~SQL.split(';').map(&:strip).reject(&:empty?).each { |sql| root_client.query(sql) } 23 | DROP DATABASE IF EXISTS database_rewinder_test_#{suffix}; 24 | CREATE DATABASE database_rewinder_test_#{suffix}; 25 | CREATE TABLE database_rewinder_test_#{suffix}.hoge ( 26 | id INT NOT NULL AUTO_INCREMENT, 27 | price INT NOT NULL, 28 | PRIMARY KEY (id) 29 | ); 30 | SQL 31 | end 32 | %w[1 2].each do |suffix| 33 | <<~SQL.split(';').map(&:strip).reject(&:empty?).each { |sql| root_client.query(sql) } 34 | DROP USER IF EXISTS database_rewinder_test_user#{suffix}; 35 | CREATE USER database_rewinder_test_user#{suffix}; 36 | SQL 37 | end 38 | <<~SQL.split(';').map(&:strip).reject(&:empty?).each { |sql| root_client.query(sql) } 39 | GRANT ALL ON database_rewinder_test_a.* TO database_rewinder_test_user1; 40 | GRANT ALL ON database_rewinder_test_b.* TO database_rewinder_test_user2; 41 | GRANT ALL ON database_rewinder_test_c.* TO database_rewinder_test_user2; 42 | SQL 43 | 44 | MysqlRewinder.setup(db_configs, adapter: adapter, except_tables: []) 45 | 46 | <<~SQL.split(';').map(&:strip).reject(&:empty?).each { |sql| root_client.query(sql) } 47 | INSERT INTO database_rewinder_test_a.hoge (price) VALUES (108); 48 | INSERT INTO database_rewinder_test_b.hoge (price) VALUES (108); 49 | INSERT INTO database_rewinder_test_c.hoge (price) VALUES (108); 50 | SQL 51 | end 52 | 53 | it 'removes records using appropriate config' do 54 | expect { MysqlRewinder.clean }.to change { 55 | [ 56 | root_client.query('SELECT * FROM database_rewinder_test_a.hoge').to_a.size, 57 | root_client.query('SELECT * FROM database_rewinder_test_b.hoge').to_a.size, 58 | root_client.query('SELECT * FROM database_rewinder_test_b.hoge').to_a.size, 59 | ] 60 | }.from([1,1,1]).to([0,0,0]) 61 | end 62 | end 63 | 64 | context 'when record inserted before and after initialized' do 65 | before do 66 | <<~SQL.split(';').map(&:strip).reject(&:empty?).each { |sql| root_client.query(sql) } 67 | DROP DATABASE IF EXISTS database_rewinder_test; 68 | CREATE DATABASE database_rewinder_test; 69 | CREATE TABLE database_rewinder_test.foo ( 70 | id INT NOT NULL AUTO_INCREMENT, 71 | name VARCHAR(128) NOT NULL, 72 | PRIMARY KEY (id) 73 | ); 74 | CREATE TABLE database_rewinder_test.bar ( 75 | id INT NOT NULL AUTO_INCREMENT, 76 | age INT NOT NULL, 77 | PRIMARY KEY (id) 78 | ); 79 | CREATE TABLE database_rewinder_test.piyo ( 80 | id INT NOT NULL AUTO_INCREMENT, 81 | name VARCHAR(128) NOT NULL, 82 | PRIMARY KEY (id) 83 | ) PARTITION BY HASH( id ) PARTITIONS 6; 84 | CREATE TABLE database_rewinder_test.hohho ( 85 | id INT NOT NULL AUTO_INCREMENT, 86 | name VARCHAR(128) NOT NULL, 87 | PRIMARY KEY (id) 88 | ); 89 | 90 | DROP DATABASE IF EXISTS database_rewinder_test_2; 91 | CREATE DATABASE database_rewinder_test_2; 92 | CREATE TABLE database_rewinder_test_2.foo_2 ( 93 | id INT NOT NULL AUTO_INCREMENT, 94 | name VARCHAR(128) NOT NULL, 95 | PRIMARY KEY (id) 96 | ); 97 | SQL 98 | 99 | root_client.query(<<~SQL) 100 | INSERT INTO database_rewinder_test.foo (name) VALUES ("hitori") 101 | SQL 102 | 103 | MysqlRewinder.setup([root_db_config.merge(database: 'database_rewinder_test')], adapter: adapter, except_tables: []) 104 | 105 | root_client.query(<<~SQL) 106 | INSERT INTO database_rewinder_test.bar (age) VALUES (15) 107 | SQL 108 | 109 | root_client.query(<<~SQL) 110 | INSERT INTO database_rewinder_test.piyo (name) VALUES ("nijika") 111 | SQL 112 | 113 | root_client.query(<<~SQL) 114 | INSERT INTO database_rewinder_test_2.foo_2 (name) VALUES ("nijika") 115 | SQL 116 | 117 | pid = fork do 118 | client_class.new(root_db_config).query(<<~SQL) 119 | INSERT INTO database_rewinder_test.hohho (name) VALUES ("ryo") 120 | SQL 121 | end 122 | Process.waitpid(pid) 123 | end 124 | 125 | it 'removes records inserted after init' do 126 | expect { MysqlRewinder.clean }.to change { 127 | root_client.query('SELECT * FROM database_rewinder_test.bar').to_a.size 128 | }.from(1).to(0) 129 | end 130 | 131 | it 'does not remove records inserted before init' do 132 | expect { MysqlRewinder.clean }.not_to change { 133 | root_client.query('SELECT * FROM database_rewinder_test.foo').to_a.size 134 | }.from(1) 135 | end 136 | 137 | it 'removes records in partitioned table' do 138 | expect { MysqlRewinder.clean }.to change { 139 | root_client.query('SELECT * FROM database_rewinder_test.piyo').to_a.size 140 | }.from(1).to(0) 141 | end 142 | 143 | 144 | it 'removes records inserted in child process' do 145 | expect { MysqlRewinder.clean }.to change { 146 | root_client.query('SELECT * FROM database_rewinder_test.hohho').to_a.size 147 | }.from(1).to(0) 148 | end 149 | 150 | it 'does not remove records in other database' do 151 | expect { MysqlRewinder.clean }.not_to change { 152 | root_client.query('SELECT * FROM database_rewinder_test_2.foo_2').to_a.size 153 | }.from(1) 154 | end 155 | end 156 | 157 | context 'when a table has foreign key constraints' do 158 | before do 159 | <<~SQL.split(';').map(&:strip).reject(&:empty?).each { |sql| root_client.query(sql) } 160 | DROP DATABASE IF EXISTS database_rewinder_test; 161 | CREATE DATABASE database_rewinder_test; 162 | CREATE TABLE database_rewinder_test.foo ( 163 | id INT NOT NULL AUTO_INCREMENT, 164 | name VARCHAR(128) NOT NULL, 165 | PRIMARY KEY (id) 166 | ); 167 | CREATE TABLE database_rewinder_test.bar ( 168 | id INT NOT NULL AUTO_INCREMENT, 169 | foo_id INT NOT NULL, 170 | name VARCHAR(128) NOT NULL, 171 | PRIMARY KEY (id), 172 | FOREIGN KEY (foo_id) 173 | REFERENCES foo(id) 174 | ); 175 | SQL 176 | 177 | MysqlRewinder.setup([root_db_config.merge(database: 'database_rewinder_test')], adapter: adapter, except_tables: []) 178 | 179 | root_client.query(<<~SQL) 180 | INSERT INTO database_rewinder_test.foo (name) VALUES ("seika"); 181 | SQL 182 | root_client.query(<<~SQL) 183 | INSERT INTO database_rewinder_test.bar (foo_id, name) VALUES (1, "nijika"); 184 | SQL 185 | end 186 | 187 | it 'removes records in parent table without foreign key constraints error' do 188 | expect { MysqlRewinder.clean }.to change { 189 | root_client.query('SELECT * FROM database_rewinder_test.foo').to_a.size 190 | }.from(1).to(0) 191 | end 192 | end 193 | end 194 | end 195 | end 196 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "mysql_rewinder" 4 | require "trilogy" 5 | require "mysql2" 6 | 7 | RSpec.configure do |config| 8 | # Enable flags like --only-failures and --next-failure 9 | config.example_status_persistence_file_path = ".rspec_status" 10 | 11 | # Disable RSpec exposing methods globally on `Module` and `main` 12 | config.disable_monkey_patching! 13 | 14 | config.expect_with :rspec do |c| 15 | c.syntax = :expect 16 | end 17 | end 18 | --------------------------------------------------------------------------------