├── init.rb ├── tasks └── table_migrator_tasks.rake ├── lib ├── table_migrator.rb ├── table_migrator │ ├── copy_strategy.rb │ ├── raw_sql_strategy.rb │ ├── base.rb │ ├── change_table_strategy.rb │ └── copy_engine.rb └── table_migration.rb ├── test ├── test_helper.rb ├── lib │ └── database.rb └── unit │ └── migration_strategy_test.rb ├── Rakefile ├── MIT-LICENSE └── README.md /init.rb: -------------------------------------------------------------------------------- 1 | # Include hook code here 2 | -------------------------------------------------------------------------------- /tasks/table_migrator_tasks.rake: -------------------------------------------------------------------------------- 1 | # desc "Explaining what the task does" 2 | # task :table_migrator do 3 | # # Task goes here 4 | # end 5 | -------------------------------------------------------------------------------- /lib/table_migrator.rb: -------------------------------------------------------------------------------- 1 | module TableMigrator 2 | extend self 3 | 4 | def new(*args, &block) 5 | Base.new(*args, &block) 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'shoulda' 3 | require 'active_record' 4 | 5 | TEST_ROOT = File.expand_path(File.dirname(__FILE__)) 6 | PLUGIN_ROOT = File.expand_path(TEST_ROOT, '..') 7 | 8 | $: << File.join(TEST_ROOT, 'lib') 9 | 10 | load 'database.rb' 11 | 12 | 13 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | require 'rake/testtask' 3 | require 'rake/rdoctask' 4 | 5 | desc 'Default: run unit tests.' 6 | task :default => :test 7 | 8 | desc 'Test the table_migrator plugin.' 9 | Rake::TestTask.new(:test) do |t| 10 | t.libs << 'lib' 11 | t.pattern = 'test/**/*_test.rb' 12 | t.verbose = true 13 | end 14 | 15 | desc 'Generate documentation for the table_migrator plugin.' 16 | Rake::RDocTask.new(:rdoc) do |rdoc| 17 | rdoc.rdoc_dir = 'rdoc' 18 | rdoc.title = 'TableMigrator' 19 | rdoc.options << '--line-numbers' << '--inline-source' 20 | rdoc.rdoc_files.include('README') 21 | rdoc.rdoc_files.include('lib/**/*.rb') 22 | end 23 | -------------------------------------------------------------------------------- /lib/table_migrator/copy_strategy.rb: -------------------------------------------------------------------------------- 1 | module TableMigrator 2 | class CopyStrategy 3 | 4 | attr_accessor :table, :config, :connection 5 | 6 | def initialize(table, config, connection) 7 | self.table = table 8 | self.config = config 9 | self.connection = connection 10 | end 11 | 12 | def new_table 13 | "new_#{table}" 14 | end 15 | 16 | def old_table 17 | if config[:migration_name] 18 | "#{table}_before_#{config[:migration_name]}" 19 | else 20 | "#{table}_old" 21 | end 22 | end 23 | 24 | def column_names 25 | connection.columns(table).map {|c| c.name } 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /test/lib/database.rb: -------------------------------------------------------------------------------- 1 | require 'active_record' 2 | 3 | ActiveRecord::Base.establish_connection({ 4 | :adapter => 'mysql', 5 | :host => 'localhost', 6 | :user => 'root', 7 | :database => 'table_migrator_test' 8 | }) 9 | 10 | def create_users 11 | ActiveRecord::Schema.define do 12 | create_table "users", :force => true do |t| 13 | t.string :name, :null => false 14 | t.string :email 15 | # t.string :short_bio 16 | t.timestamps 17 | end 18 | 19 | add_index :users, :updated_at 20 | # add_index :users, :name 21 | # add_index :users, :email 22 | end 23 | end 24 | 25 | def create_news_stories 26 | ActiveRecord::Schema.define do 27 | create_table "news_stories", :force => true do |t| 28 | t.integer :user_id 29 | t.text :story 30 | # t.string :actors 31 | t.datetime :create_at, :null => false 32 | end 33 | 34 | add_index :news_stories, :user_id 35 | add_index :news_stories, :created_at 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/unit/migration_strategy_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.join(File.dirname(__FILE__), '../test_helper.rb')) 2 | 3 | require 'active_record/connection_adapters/abstract/schema_definitions' 4 | require 'table_migrator' 5 | 6 | class MigrationStrategyTest < Test::Unit::TestCase 7 | MigrationStrategy = ::TableMigrator::MigrationStrategy 8 | Table = ::ActiveRecord::ConnectionAdapters::Table 9 | Connection = ActiveRecord::Base.connection 10 | 11 | context "An instance of MigrationStrategy" do 12 | setup do 13 | create_users 14 | @strategy = MigrationStrategy.new(:users) 15 | end 16 | 17 | should "implement all Table methods" do 18 | Table.instance_methods.each do |table_method| 19 | assert @strategy.respond_to?(table_method), "Does not respond to method '#{table_method}'" 20 | end 21 | end 22 | 23 | should "have correct existing columns list" do 24 | expected_columns = Connection.columns(:users).map {|c| c.name } 25 | assert_equal expected_columns.sort, @strategy.existing_columns 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/table_migrator/raw_sql_strategy.rb: -------------------------------------------------------------------------------- 1 | module TableMigrator 2 | class RawSqlStrategy < CopyStrategy 3 | attr_accessor :base_copy_query, :schema_changes 4 | 5 | def initialize(table, config, connection, base_copy_query, schema_changes) 6 | super(table, config, connection) 7 | 8 | self.base_copy_query = base_copy_query 9 | self.schema_changes = schema_changes 10 | end 11 | 12 | def apply_changes 13 | schema_changes.each do |sql| 14 | connection.execute sub_new_table(sql, new_table) 15 | end 16 | end 17 | 18 | # columns are the responsibility of the user 19 | def base_copy_query(insert_or_replace) 20 | copy = @base_copy_query.gsub(/\A\s*INSERT/i, insert_or_replace) 21 | copy = sub_new_table(copy, new_table) 22 | copy = sub_table(copy, table) 23 | copy 24 | end 25 | 26 | private 27 | 28 | def sub_table(sql, table) 29 | sql.to_s.gsub(":table_name", "`#{table}`") 30 | end 31 | 32 | def sub_new_table(sql, new_table) 33 | sql.to_s.gsub(":new_table_name", "`#{new_table}`") 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 Serious Business 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /lib/table_migration.rb: -------------------------------------------------------------------------------- 1 | class TableMigration < ActiveRecord::Migration 2 | class << self 3 | attr_reader :table_migrator 4 | delegate :schema_changes, :to => :table_migrator 5 | delegate :column_names, :to => :table_migrator 6 | delegate :quoted_column_names, :to => :table_migrator 7 | delegate :base_copy_query, :to => :table_migrator 8 | end 9 | 10 | def self.change_table(*args, &block) 11 | migrates(*args) if table_migrator.nil? 12 | 13 | table_migrator.change_table(&block) 14 | end 15 | 16 | def self.create_table_and_copy_info 17 | table_migrator.create_table_and_copy_info 18 | end 19 | 20 | def self.migrates(table_name, config = {}) 21 | default = {:migration_name => name.underscore} 22 | puts default.update(config).inspect 23 | @table_migrator = TableMigrator::Base.new(table_name, default.update(config)) 24 | end 25 | 26 | def self.up 27 | table_migrator.up! 28 | raise "Dry Run!" if table_migrator.dry_run? 29 | end 30 | 31 | def self.down 32 | table_migrator.down! 33 | raise "Dry Run!" if table_migrator.dry_run? 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /lib/table_migrator/base.rb: -------------------------------------------------------------------------------- 1 | module TableMigrator 2 | class Base 3 | 4 | attr_accessor :table, :config 5 | attr_accessor :schema_changes, :column_names, :quoted_column_names, :base_copy_query 6 | 7 | def initialize(table, config = {}) 8 | self.table = table 9 | 10 | defaults = { :dry_run => false, :create_temp_table => true, :delta_column => 'updated_at'} 11 | self.config = defaults.merge(config) 12 | end 13 | 14 | def up! 15 | engine.set_epoch(Time.parse(ENV['NEXT_EPOCH'])) if !ENV['NEXT_EPOCH'].blank? 16 | engine.up! 17 | end 18 | 19 | def down! 20 | engine.down! 21 | end 22 | 23 | # config methods 24 | 25 | def schema_changes 26 | @schema_changes ||= [] 27 | end 28 | 29 | def change_table(&block) 30 | @strategy = ChangeTableStrategy.new(table, config, connection, &block) 31 | end 32 | 33 | # helpers 34 | 35 | def column_names 36 | @column_names ||= connection.columns(table).map { |c| c.name } 37 | end 38 | 39 | def quoted_column_names 40 | @quoted_column_names ||= column_names.map { |n| "`#{n}`" } 41 | end 42 | 43 | def base_copy_query(columns = nil) 44 | unless columns.nil? 45 | @base_copy_query = nil 46 | columns = columns.map { |n| "`#{n}`" } 47 | else 48 | columns = quoted_column_names 49 | end 50 | 51 | @base_copy_query ||= %(INSERT INTO :new_table_name (#{columns.join(", ")}) SELECT #{columns.join(", ")} FROM :table_name) 52 | end 53 | 54 | def dry_run? 55 | config[:dry_run] == true 56 | end 57 | 58 | def create_table_and_copy_info 59 | engine.create_table_and_copy_info 60 | end 61 | 62 | private 63 | 64 | def strategy 65 | # if change_table hasn't been called, this will use RawSqlStrategy 66 | @strategy ||= RawSqlStrategy.new(table, config, connection, base_copy_query, schema_changes) 67 | end 68 | 69 | def connection 70 | ActiveRecord::Base.connection 71 | end 72 | 73 | def engine 74 | @engine ||= CopyEngine.new(strategy) 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /lib/table_migrator/change_table_strategy.rb: -------------------------------------------------------------------------------- 1 | module TableMigrator 2 | 3 | class ChangeTableStrategy < CopyStrategy 4 | attr_accessor :changes, :renames 5 | 6 | class TableNameMismatchError < Exception; end 7 | 8 | def initialize(table, config, connection) 9 | super(table, config, connection) 10 | 11 | @changes = [] 12 | @renames = Hash.new {|h,k| h[k.to_s] = k.to_s } 13 | 14 | yield ::ActiveRecord::ConnectionAdapters::Table.new(table, self) 15 | end 16 | 17 | # interface used by Base. 18 | 19 | def apply_changes 20 | changes.each do |method, args| 21 | connection.send(method, new_table, *args) 22 | end 23 | end 24 | 25 | def base_copy_query(insert_or_replace) 26 | copied = column_names.reject {|c| renames[c].nil? } 27 | renamed = copied.map {|c| renames[c] } 28 | renamed = renamed.map {|c| "`#{c}`"} 29 | copied = copied.map {|c| "`#{c}`"} 30 | 31 | "#{insert_or_replace} INTO `#{new_table}` (#{renamed.join(', ')}) 32 | SELECT #{copied.join(', ')} FROM `#{table}`" 33 | end 34 | 35 | 36 | # delegate methods used for table introspection to the native connection 37 | 38 | def type_to_sql(*args); connection.type_to_sql(*args); end 39 | def quote_column_name(*args); connection.quote_column_name(*args); end 40 | def add_column_options!(*args); connection.add_column_options!(*args); end 41 | def native_database_types(*args); connection.native_database_types(*args); end 42 | 43 | 44 | # change registration callbacks 45 | 46 | def method_missing(method, table_name, *args) 47 | if table_name != @table 48 | raise TableNameMismatchError, "Expected table `#{@table}`, got `#{table_name}`!" 49 | end 50 | 51 | # register the change if we need to do something special during the copy phase. 52 | send("register_#{method}", *args) if respond_to?("register_#{method}") 53 | 54 | # record for replay later 55 | changes << [method, args] 56 | end 57 | 58 | def register_rename_column(col, new_name) 59 | puts "rename called" 60 | renames[col.to_s] = new_name.to_s 61 | end 62 | 63 | def register_remove_column(*columns) 64 | columns.each do |col| 65 | renames[col.to_s] = nil 66 | end 67 | end 68 | 69 | def register_remove_timestamps 70 | register_remove_column :created_at, :updated_at 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/table_migrator/copy_engine.rb: -------------------------------------------------------------------------------- 1 | module TableMigrator 2 | class CopyEngine 3 | 4 | attr_accessor :strategy 5 | 6 | # magic numbers 7 | MAX_DELTA_PASSES = 5 8 | DELTA_CONTINUE_THRESHOLD = 5 9 | PAGE_SIZE = 50_000 10 | DELTA_PAGE_SIZE = 1000 11 | PAUSE_LENGTH = 5 12 | 13 | def initialize(strategy) 14 | self.strategy = strategy 15 | 16 | #updated_at sanity check 17 | unless dry_run? or strategy.column_names.include?(delta_column.to_s) 18 | raise "Cannot use #{self.class.name} on #{table_name.inspect} table without a delta column: #{delta_column.inspect}" 19 | end 20 | end 21 | 22 | def up! 23 | info 'Executing dry run...' if dry_run? 24 | 25 | self.create_new_table if create_temp_table? 26 | 27 | # is there any data to copy? 28 | if dry_run? or execute("SELECT * FROM `#{table_name}` LIMIT 1").fetch_row 29 | 30 | # copy bulk of table data 31 | self.paged_copy if create_temp_table? 32 | 33 | # multi-pass delta copy to reduce the size of the locked pass 34 | self.multi_pass_delta_copy if multi_pass? 35 | 36 | if create_temp_table? || multi_pass? 37 | # wait here... 38 | info "Waiting for #{PAUSE_LENGTH} seconds" 39 | PAUSE_LENGTH.times { info '.'; $stdout.flush; sleep 1 } 40 | info ' ' 41 | end 42 | 43 | # lock for write, copy final delta, and swap 44 | in_table_lock(table_name, new_table_name) do 45 | self.full_delta_copy 46 | execute("ALTER TABLE `#{table_name}` RENAME TO `#{old_table_name}`") 47 | execute("ALTER TABLE `#{new_table_name}` RENAME TO `#{table_name}`") 48 | end 49 | 50 | else 51 | # if there are no rows previously, lock and copy everything (probably still nothing). 52 | # this will not be the case in production. 53 | in_table_lock(table_name, new_table_name) do 54 | execute(self.base_copy_query) 55 | execute("ALTER TABLE `#{table_name}` RENAME TO `#{old_table_name}`") 56 | execute("ALTER TABLE `#{new_table_name}` RENAME TO `#{table_name}`") 57 | end 58 | end 59 | end 60 | 61 | def down! 62 | in_table_lock(table_name, old_table_name) do 63 | execute("ALTER TABLE `#{table_name}` RENAME TO `#{new_table_name}`") 64 | execute("ALTER TABLE `#{old_table_name}` RENAME TO `#{table_name}`") 65 | execute("DROP TABLE `#{new_table_name}`") 66 | end 67 | end 68 | 69 | # performs only the table creation and copy phases so that the actual migration 70 | # is as quick as possible. 71 | def create_table_and_copy_info 72 | create_new_table 73 | paged_copy 74 | multi_pass_delta_copy if multi_pass? 75 | end 76 | 77 | 78 | # migration steps 79 | def create_new_table 80 | execute("CREATE TABLE `#{new_table_name}` LIKE `#{table_name}`") 81 | 82 | # make schema changes 83 | info "Applying schema changes to new table" 84 | strategy.apply_changes unless dry_run? 85 | end 86 | 87 | def paged_copy 88 | info "Copying #{table_name} to #{new_table_name}" 89 | 90 | # record start of this epoch 91 | self.flop_epoch 92 | 93 | start = 0 94 | page = 0 95 | loop do 96 | info "page #{page += 1}..." 97 | execute(paged_copy_query(start, PAGE_SIZE)) 98 | 99 | new_start = if dry_run? 100 | 0 101 | else 102 | select_all("select max(id) from `#{new_table_name}`").first.values.first.to_i 103 | end 104 | 105 | break if start == new_start 106 | start = new_start 107 | end 108 | end 109 | 110 | def multi_pass_delta_copy 111 | info "Multi-pass non-locking delta copy from #{table_name} to #{new_table_name}" 112 | 113 | pass = 0 114 | loop do 115 | info "pass #{pass += 1}..." 116 | 117 | time_start = Time.now 118 | self.paged_delta_copy 119 | time_elapsed = Time.now.to_i - time_start.to_i 120 | 121 | break if time_elapsed <= DELTA_CONTINUE_THRESHOLD or pass == MAX_DELTA_PASSES 122 | end 123 | end 124 | 125 | def paged_delta_copy 126 | epoch = self.flop_epoch 127 | updated_ids = select_all(updated_ids_query(epoch)).map{|r| r['id'].to_i} 128 | 129 | updated_ids.in_groups_of(DELTA_PAGE_SIZE, false) do |ids| 130 | info "Executing: #{paged_delta_copy_query(['IDS'])}" 131 | execute(paged_delta_copy_query(ids), true) 132 | end 133 | end 134 | 135 | def full_delta_copy 136 | epoch = self.last_epoch 137 | info_with_time "Copying delta from #{table_name} to #{new_table_name}" do 138 | execute(full_delta_copy_query(epoch)) 139 | end 140 | end 141 | 142 | # Logging 143 | 144 | def info(str) 145 | ActiveRecord::Migration.say(str) 146 | end 147 | 148 | def info_with_time(str, &block) 149 | ActiveRecord::Migration.say_with_time(str, &block) 150 | end 151 | 152 | # Manage the Epoch 153 | 154 | def set_epoch(time) 155 | @next_epoch = time 156 | end 157 | 158 | def flop_epoch 159 | epoch = @next_epoch 160 | @next_epoch = self.next_epoch 161 | info "Current Epoch starts at: #{@next_epoch}" 162 | epoch 163 | end 164 | 165 | def last_epoch 166 | @next_epoch 167 | end 168 | 169 | def next_epoch 170 | return Time.at(0) if dry_run? 171 | 172 | epoch_query = "SELECT `#{delta_column}` FROM `#{table_name}` 173 | ORDER BY `#{delta_column}` DESC LIMIT 1" 174 | 175 | select_all(epoch_query).first[delta_column] 176 | end 177 | 178 | 179 | # Queries 180 | 181 | def base_copy_query 182 | strategy.base_copy_query('REPLACE') 183 | end 184 | 185 | def paged_copy_query(start, limit) 186 | "#{base_copy_query} WHERE `id` > #{start} LIMIT #{limit}" 187 | end 188 | 189 | def full_delta_copy_query(epoch) 190 | "#{base_copy_query} WHERE `#{delta_column}` >= '#{epoch}'" 191 | end 192 | 193 | def updated_ids_query(epoch) 194 | "SELECT `id` FROM #{table_name} WHERE `#{delta_column}` >= '#{epoch}'" 195 | end 196 | 197 | def paged_delta_copy_query(ids) 198 | "#{base_copy_query} WHERE `id` in (#{ids.join(', ')})" 199 | end 200 | 201 | 202 | # Config Helpers 203 | 204 | def delta_column 205 | strategy.config[:delta_column] 206 | end 207 | 208 | def table_name 209 | strategy.table 210 | end 211 | 212 | def new_table_name 213 | strategy.new_table 214 | end 215 | 216 | def old_table_name 217 | strategy.old_table 218 | end 219 | 220 | # behavior 221 | 222 | def dry_run? 223 | strategy.config[:dry_run] == true 224 | end 225 | 226 | def create_temp_table? 227 | strategy.config[:create_temp_table] == true 228 | end 229 | 230 | def multi_pass? 231 | strategy.config[:multi_pass] == true 232 | end 233 | 234 | 235 | # SQL Execution 236 | 237 | def execute(sql, quiet = false) 238 | execution = lambda do 239 | unless dry_run? 240 | strategy.connection.execute(sql) 241 | end 242 | end 243 | if quiet 244 | execution.call 245 | else 246 | info_with_time("Executing: #{sql}", &execution) 247 | end 248 | end 249 | 250 | def select_all(sql, quiet = false) 251 | execution = lambda do 252 | if dry_run? 253 | [] 254 | else 255 | strategy.connection.select_all(sql) 256 | end 257 | end 258 | if quiet 259 | execution.call 260 | else 261 | info_with_time("Finding: #{sql}", &execution) 262 | end 263 | end 264 | 265 | def in_table_lock(*tables) 266 | info_with_time "Acquiring write lock." do 267 | begin 268 | execute('SET autocommit=0') 269 | table_locks = tables.map {|t| "`#{t}` WRITE"}.join(', ') 270 | execute("LOCK TABLES #{table_locks}") 271 | 272 | yield 273 | 274 | execute('COMMIT') 275 | execute('UNLOCK TABLES') 276 | ensure 277 | execute('SET autocommit=1') 278 | end 279 | end 280 | end 281 | end 282 | end 283 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TableMigrator 2 | 3 | Zero-downtime migrations of large tables in MySQL. 4 | 5 | See the example for usage. Make sure you have an index on updated_at. 6 | 7 | Install as a rails plugin: `./script/plugin install git://github.com/freels/table_migrator.git` 8 | 9 | 10 | ### What this does 11 | 12 | TableMigrator is a method for altering large MySQL tables while incurring as little downtime as possible. There is nothing special about `ALTER TABLE`. All it does is create a new table with the new schema and copy over each row from the original table. Oh, and it does this in a big giant table lock, so you won't be using that big table for a while... 13 | 14 | TableMigrator does essentially the same thing, but more intelligently. First, we create a new table like the original one, and then apply one or more `ALTER TABLE` statements to the unused, empty table. Second we copy all rows from the original table into the new one. All this time, reads and writes are going to the original table, so the two tables are not consistent. Finally, we acquire a write lock on the original table before copying over all new/changed rows, and swapping in the new table. 15 | 16 | The solution to find updated or new rows is to use a column like `updated_at` (if a row is mutable) or `created_at` (if it is immutable) to determine which rows have been modified since the copy started. These rows are copied over to the new table using `REPLACE`. 17 | 18 | If we do a single sweep of changed rows (you set `:multi_pass` to false), then a write lock is acquired before the sweep, new/changed rows are copied to the new table, the tables are swapped, and then the lock is released. 19 | 20 | The default method (`:multi_pass => true`) copies over changed rows in a non-blocking manner multiple times until we can be reasonably sure that the final sweep will take very little time. The last sweep is done within the write lock, and then the tables are swapped. The length of time taken in the write lock is extremely short, hence the claim zero-downtime migration. 21 | 22 | The key to making these sweeps for changes fast is to have an index on the column used to find them. Having an index on the relevant column makes this process fast. Without that index, TableMigrator eventually has to do a table scan within the final synchronous sweep, and that means downtime will be unavoidable. 23 | 24 | ### A note about DELETE 25 | 26 | This method will not propagate normal `DELETES`s to the new table if they happen during/after the copy. In order to avoid this, use paranoid deletion, and update the column you are using to find changes appropriately. 27 | 28 | 29 | ## Examples 30 | 31 | ### Simple Migration 32 | 33 | TableMigrator supports two APIs for defining migrations. One uses ActiveRecord's `change_table` syntax, and the other uses manually defined SQL snippets. You can create your migration by inheriting from TableMigration and skip some of the setup normally required. 34 | 35 | # using change_table 36 | 37 | class AddStuffToMyBigTable < TableMigration 38 | 39 | migrates :users 40 | # migrates also can take an options hash: 41 | # :multi_pass - See explanation above. Defaults to true 42 | # :migration_name - the original table is not dropped after the migration. 43 | # It will instead have a name based on this option. 44 | # The default is based on the migration class. (The old table 45 | # will end up named 'users_before_add_stuff_to_my_big_table' 46 | # in this case) 47 | # :create_temp_table - Performs the migration in two steps if false. Read below 48 | # for details. Defaults to true. 49 | # :dry_run - If true, the migration will not actually run, just emit 50 | # fake progress to the log. Defaults to false. 51 | 52 | change_table do |t| 53 | t.integer :foo, :null => false, :default => 0 54 | t.string :bar 55 | t.index :foo, :name => 'index_for_foo' 56 | end 57 | end 58 | 59 | 60 | # using raw sql 61 | 62 | class AddStuffToMyBigTable < TableMigration 63 | migrates :users 64 | 65 | # push alter tables to schema_changes 66 | schema_changes.push <<-SQL 67 | ALTER TABLE :new_table_name 68 | ADD COLUMN `foo` int(11) unsigned NOT NULL DEFAULT 0 69 | SQL 70 | 71 | schema_changes.push <<-SQL 72 | ALTER TABLE :new_table_name 73 | ADD COLUMN `bar` varchar(255) 74 | SQL 75 | 76 | schema_changes.push <<-SQL 77 | ALTER TABLE :new_table_name 78 | ADD INDEX `index_foo` (`foo`) 79 | SQL 80 | 81 | # some helpers are provided: 82 | # table_migrator - access to the TableMigrator instance being configured 83 | # column_names - defaults to an array of the original table's columns. 84 | # quoted_column_names - the above, with each quoted in back-ticks. 85 | 86 | # you can also customize the base copy query by setting it to an INSERT statement 87 | # with no conditions. The default is based on column_names, above. 88 | # :table_name and :new_table_name are replaced with the original table and 89 | # new table names, respectively INSERT is substituted for REPLACE after the initial 90 | # bulk copy. 91 | table_migrator.base_copy_query = <<-SQL 92 | INSERT INTO :new_table_name (#{column_names.join(", ")}) 93 | SELECT #{column_names.join(", ")} FROM :table_name 94 | SQL 95 | end 96 | 97 | 98 | ### Advanced Migration 99 | 100 | You can use a normal ActiveRecord::Migration, you just have to set up a TableMigrator instance yourself. Otherwise, it works the same as above. 101 | 102 | class AddStuffToMyBigTable < ActiveRecord::Migration 103 | 104 | # just a helper method so we don't have to repeat this in self.up and self.down 105 | def self.setup 106 | 107 | # create a new TableMigrator instance for the table `users` 108 | # TableMigrator#initialize takes the same arguments as 'migrates' 109 | @tm = TableMigrator.new(:users, 110 | :migration_name => 'add_stuff', 111 | :multi_pass => true, 112 | :create_temp_table => true, # default 113 | :dry_run => false 114 | ) 115 | 116 | # push alter tables to schema_changes 117 | @tm.schema_changes.push <<-SQL 118 | ALTER TABLE :new_table_name 119 | ADD COLUMN `foo` int(11) unsigned NOT NULL DEFAULT 0 120 | SQL 121 | 122 | @tm.schema_changes.push <<-SQL 123 | ALTER TABLE :new_table_name 124 | ADD COLUMN `bar` varchar(255) 125 | SQL 126 | 127 | @tm.schema_changes.push <<-SQL 128 | ALTER TABLE :new_table_name 129 | ADD INDEX `index_foo` (`foo`) 130 | SQL 131 | 132 | # customizing @tm.column_names 133 | @tm.column_names = %w(id name session password_hash created_at updated_at) 134 | 135 | # for convenience 136 | column_list = @tm.quoted_column_names.join(', ') 137 | 138 | # the base INSERT query with no wheres. (We'll take care of that part.) 139 | @tm.base_copy_query = <<-SQL 140 | INSERT INTO :new_table_name (#{column_list}) SELECT #{column_list} FROM :table_name 141 | SQL 142 | end 143 | 144 | def self.up 145 | self.setup 146 | @tm.up! 147 | end 148 | 149 | def self.down 150 | self.setup 151 | @tm.down! 152 | end 153 | 154 | # see 'two-phase migration' below 155 | def self.create_table_and_copy_info 156 | self.setup 157 | @tm.create_table_and_copy_info 158 | end 159 | end 160 | 161 | 162 | ## Migrating in Two Phases 163 | 164 | You can run the migration in two phases if you set the `:create_temp_table` option to false. 165 | 166 | First, you deploy the code with the migration and manually run the `#create_table_and_copy_info` method: 167 | 168 | # if you use a TableMigration sublcass 169 | >> require 'db/migrate/13423423_my_migration.rb' 170 | >> now = Time.now 171 | >> MyMigration.create_table_and_copy_info 172 | >> puts %(NEXT_EPOCH="#{now}") 173 | NEXT_EPOCH="Tue Feb 16 13:22:14 -0800 2010" 174 | 175 | This creates the temporary table, copies the data over without locking anything. You can safely run this without halting your application. 176 | 177 | Finally, you put up whatever downtime notices you have and run your typical migration task. Since the table is already created, the script will only 178 | copy data (if multi_pass is enabled) and perform the actual table move. It assumes the temporary table has been created already. 179 | 180 | $ NEXT_EPOCH="Tue Feb 16 13:22:14 -0800 2010" RAILS_ENV=production rake db:migrate 181 | 182 | ## Contributors 183 | - Matt Freels 184 | - Rohith Ravi 185 | - Rick Olson 186 | 187 | Copyright (c) 2009 Serious Business, released under the MIT license. 188 | --------------------------------------------------------------------------------