├── .ruby-version ├── spec ├── internal │ ├── log │ │ └── .gitignore │ ├── app │ │ └── models │ │ │ ├── post.rb │ │ │ └── user.rb │ ├── db │ │ ├── migrate │ │ │ ├── 20161012223254_safe_add_index.rb │ │ │ ├── 20161012223256_safe_add_index_with_dsl.rb │ │ │ ├── 20161012223253_safe_add_column_with_default.rb │ │ │ ├── 20161012223257_add_index_concurrently.rb │ │ │ ├── 20161012223255_safe_add_index_with_env.rb │ │ │ ├── 20161012223252_rollup_migrations.rb │ │ │ └── 20161012223258_create_table_comments.rb │ │ └── schema.rb │ └── config │ │ └── database.yml ├── spec_helper.rb ├── zero_downtime_migrations_spec.rb └── zero_downtime_migrations │ ├── validation │ ├── add_index_spec.rb │ ├── find_each_spec.rb │ ├── add_column_spec.rb │ ├── ddl_migration_spec.rb │ └── mixed_migration_spec.rb │ └── validation_spec.rb ├── .gitignore ├── .rspec ├── Dockerfile ├── Gemfile ├── lib ├── zero_downtime_migrations │ ├── data.rb │ ├── relation.rb │ ├── error.rb │ ├── dsl.rb │ ├── validation │ │ ├── find_each.rb │ │ ├── ddl_migration.rb │ │ ├── mixed_migration.rb │ │ ├── add_index.rb │ │ └── add_column.rb │ ├── validation.rb │ └── migration.rb └── zero_downtime_migrations.rb ├── .buildkite └── pipeline.yml ├── docker-compose.yml ├── zero_downtime_migrations.gemspec ├── bin ├── rubocop └── test ├── LICENSE ├── Gemfile.lock └── README.md /.ruby-version: -------------------------------------------------------------------------------- 1 | 2.3.1 2 | -------------------------------------------------------------------------------- /spec/internal/log/.gitignore: -------------------------------------------------------------------------------- 1 | *.log -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | pkg 3 | rdoc -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format progress 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM lendinghome/rails 2 | MAINTAINER github@lendinghome.com 3 | -------------------------------------------------------------------------------- /spec/internal/app/models/post.rb: -------------------------------------------------------------------------------- 1 | class Post < ActiveRecord::Base 2 | end 3 | -------------------------------------------------------------------------------- /spec/internal/app/models/user.rb: -------------------------------------------------------------------------------- 1 | class User < ActiveRecord::Base 2 | end 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://www.rubygems.org" 2 | 3 | gemspec 4 | 5 | gem "codeclimate-test-reporter" 6 | gem "combustion" 7 | gem "pg" 8 | gem "rspec-rails" 9 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/data.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | module Data 3 | def initialize(*) 4 | Migration.data = true 5 | super 6 | end 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223254_safe_add_index.rb: -------------------------------------------------------------------------------- 1 | class SafeAddIndex < ActiveRecord::Migration[5.0] 2 | def change 3 | safety_assured { add_index :posts, :published } 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223256_safe_add_index_with_dsl.rb: -------------------------------------------------------------------------------- 1 | class SafeAddIndexWithDsl < ActiveRecord::Migration[5.0] 2 | safety_assured 3 | 4 | def change 5 | add_index :posts, :created_at 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/relation.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | module Relation 3 | prepend Data 4 | 5 | def each(*) 6 | Validation.validate!(:find_each) 7 | super 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/error.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Error < StandardError 3 | end 4 | 5 | class UndefinedValidationError < Error 6 | end 7 | 8 | class UnsafeMigrationError < Error 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: ":rubocop:" 3 | command: bin/rubocop 4 | timeout_in_minutes: 30 5 | 6 | - name: ":rspec:" 7 | command: bin/test 8 | timeout_in_minutes: 30 9 | artifact_paths: cache.tar.lz4 10 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223253_safe_add_column_with_default.rb: -------------------------------------------------------------------------------- 1 | class SafeAddColumnWithDefault < ActiveRecord::Migration[5.0] 2 | def change 3 | safety_assured { add_column :posts, :published, :boolean, default: false } 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223257_add_index_concurrently.rb: -------------------------------------------------------------------------------- 1 | class AddIndexConcurrently < ActiveRecord::Migration[5.0] 2 | disable_ddl_transaction! 3 | 4 | def change 5 | add_index :posts, :body, algorithm: :concurrently 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223255_safe_add_index_with_env.rb: -------------------------------------------------------------------------------- 1 | class SafeAddIndexWithEnv < ActiveRecord::Migration[5.0] 2 | def change 3 | ENV["SAFETY_ASSURED"] = "1" 4 | add_index :users, :created_at 5 | ENV.delete("SAFETY_ASSURED") 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/internal/config/database.yml: -------------------------------------------------------------------------------- 1 | test: 2 | adapter: postgresql 3 | <% if ENV["DATABASE_URL"] %> 4 | url: <%= ENV["DATABASE_URL"] %> 5 | <% else %> 6 | database: zero_downtime_migrations 7 | encoding: unicode 8 | pool: 5 9 | username: <%= ENV.fetch("USER") %> 10 | <% end %> 11 | -------------------------------------------------------------------------------- /spec/internal/db/schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define(version: 20161012223251) do 2 | create_table "users", force: :cascade do |t| 3 | t.string "email", null: false 4 | t.datetime "created_at", null: false 5 | t.datetime "updated_at", null: false 6 | end 7 | 8 | add_index "users", ["email"], name: "index_users_on_email", using: :btree 9 | end 10 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223252_rollup_migrations.rb: -------------------------------------------------------------------------------- 1 | class RollupMigrations < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :posts do |t| 4 | t.references :user, null: false 5 | t.string :title, null: false 6 | t.text :body, null: false 7 | t.timestamps null: false 8 | end 9 | 10 | add_index :posts, :title 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /spec/internal/db/migrate/20161012223258_create_table_comments.rb: -------------------------------------------------------------------------------- 1 | class CreateTableComments < ActiveRecord::Migration[5.0] 2 | def change 3 | create_table :comments do |t| 4 | t.references :user, null: false, index: true 5 | t.references :post, null: false, index: true 6 | t.text :body, null: false 7 | t.timestamps null: false 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/dsl.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | module DSL 3 | attr_accessor :current, :data, :ddl, :index, :safe 4 | 5 | def data? 6 | !!@data 7 | end 8 | 9 | def ddl? 10 | !!@ddl 11 | end 12 | 13 | def index? 14 | !!@index 15 | end 16 | 17 | def migrating? 18 | !!@current 19 | end 20 | 21 | def mixed? 22 | [data?, ddl?, index?].select(&:itself).size > 1 23 | end 24 | 25 | def safe? 26 | !!@safe || ENV["SAFETY_ASSURED"].presence 27 | end 28 | 29 | def safety_assured 30 | Migration.safe = true 31 | end 32 | 33 | def unsafe? 34 | !safe? 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | if ENV["CODECLIMATE_REPO_TOKEN"] 2 | require "codeclimate-test-reporter" 3 | CodeClimate::TestReporter.start 4 | else 5 | require "simplecov" 6 | SimpleCov.start { add_filter("/vendor/bundle/") } 7 | end 8 | 9 | require File.expand_path("../../lib/zero_downtime_migrations", __FILE__) 10 | ActiveRecord::Migration.verbose = false 11 | 12 | require "combustion" 13 | Combustion.initialize!(:active_record) 14 | 15 | require "rspec/rails" 16 | RSpec::Expectations.configuration.on_potential_false_positives = :nothing 17 | 18 | RSpec.configure do |config| 19 | config.filter_run :focus 20 | config.raise_errors_for_deprecations! 21 | config.run_all_when_everything_filtered = true 22 | config.use_transactional_fixtures = true 23 | end 24 | -------------------------------------------------------------------------------- /spec/zero_downtime_migrations_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations do 2 | describe "#initialize" do 3 | it "loads the db schema and migrations without errors" do 4 | expect(ActiveRecord::Base.connection.tables.size).to be > 1 5 | end 6 | end 7 | 8 | describe "#gemspec" do 9 | it "returns a gemspec object" do 10 | expect(described_class.gemspec).to be_a(Gem::Specification) 11 | end 12 | end 13 | 14 | describe "#root" do 15 | it "returns a pathname object" do 16 | expect(described_class.root).to be_a(Pathname) 17 | end 18 | end 19 | 20 | describe "#version" do 21 | it "returns a version string" do 22 | expect(described_class.version).to match(/^\d+\.\d+\.\d+$/) 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | services: 4 | app: 5 | build: 6 | context: "." 7 | image: $DOCKER_IMAGE 8 | depends_on: 9 | - postgres 10 | environment: 11 | - CI 12 | - CODECLIMATE_REPO_TOKEN=efff74f42f6e9dff9daa21e21e8d8d2d3cb33f37b84a9d06bb907edb48a2cf6f 13 | - DATABASE_URL=postgres://postgres@postgres/zero_downtime_migrations 14 | - DOCKER=true 15 | - RACK_ENV=test 16 | - RAILS_ENV=test 17 | - REBUILD_CACHE 18 | links: 19 | - postgres 20 | volumes: 21 | - .:/app 22 | - $BUILD_CACHE/bundler:/app/vendor/bundle 23 | - $BUILD_CACHE/tmp:/app/tmp/cache 24 | 25 | postgres: 26 | image: lendinghome/postgres:9.4.8 27 | 28 | rubocop: 29 | image: lendinghome/rubocop 30 | environment: 31 | - RUBOCOP_VERSION=2.3 32 | volumes: 33 | - .:/app 34 | -------------------------------------------------------------------------------- /zero_downtime_migrations.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.author = "LendingHome" 3 | s.email = "github@lendinghome.com" 4 | s.extra_rdoc_files = %w(LICENSE) 5 | s.files = `git ls-files`.split("\n") 6 | s.homepage = "https://github.com/lendinghome/zero_downtime_migrations" 7 | s.license = "MIT" 8 | s.name = "zero_downtime_migrations" 9 | s.rdoc_options = %w(--charset=UTF-8 --inline-source --line-numbers --main README.md) 10 | s.require_paths = %w(lib) 11 | s.required_ruby_version = ">= 2.0.0" 12 | s.summary = "Zero downtime migrations with ActiveRecord and PostgreSQL" 13 | s.test_files = `git ls-files -- spec/*`.split("\n") 14 | s.version = "0.0.7" 15 | 16 | s.add_dependency "activerecord" 17 | end 18 | -------------------------------------------------------------------------------- /spec/zero_downtime_migrations/validation/add_index_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations::Validation::AddIndex do 2 | let(:error) { ZeroDowntimeMigrations::UnsafeMigrationError } 3 | 4 | context "with ddl transaction enabled" do 5 | let(:migration) do 6 | Class.new(ActiveRecord::Migration[5.0]) do 7 | def change 8 | add_index :users, :updated_at 9 | end 10 | end 11 | end 12 | 13 | it "raises an unsafe migration error" do 14 | expect { migration.migrate(:up) }.to raise_error(error) 15 | end 16 | end 17 | 18 | context "without using the concurrently algorithm" do 19 | let(:migration) do 20 | Class.new(ActiveRecord::Migration[5.0]) do 21 | disable_ddl_transaction! 22 | 23 | def change 24 | add_index :users, :updated_at 25 | end 26 | end 27 | end 28 | 29 | it "raises an unsafe migration error" do 30 | expect { migration.migrate(:up) }.to raise_error(error) 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /bin/rubocop: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | echo -e "--- \033[33mWhat does this command do?\033[0m" 6 | echo 7 | echo " This script runs our LendingHome platform wide rubocop Docker image" 8 | echo " for the gem or service mounted at /app. It uses a configuration file" 9 | echo " that has been added directly into the rubocop Docker image itself. This" 10 | echo " configuration lives in the lendinghome-dockerfiles repository." 11 | echo 12 | echo " Check out the documentation on rubocop for more information!" 13 | echo 14 | echo " * https://github.com/bbatsov/rubocop" 15 | echo " * https://github.com/bbatsov/ruby-style-guide" 16 | echo 17 | 18 | if [ -v REBUILD_CACHE ]; then 19 | echo "--- :docker: Pulling latest docker image" 20 | docker-compose pull rubocop 21 | 22 | echo "--- :recycle: Updating build cache" 23 | echo "The build was triggered with REBUILD_CACHE" 24 | echo "Skipping the remaining build steps" 25 | else 26 | echo "--- :docker: Starting docker" 27 | docker-compose pull rubocop 28 | docker-compose run rubocop $@ 29 | fi -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/validation/find_each.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Validation 3 | class FindEach < Validation 4 | def validate! 5 | error!(message) 6 | end 7 | 8 | private 9 | 10 | def message 11 | <<-MESSAGE.strip_heredoc 12 | Using `ActiveRecord::Relation#each` is unsafe! 13 | 14 | Let's use the `find_each` method to fetch records in batches instead. 15 | 16 | Otherwise we may accidentally load tens or hundreds of thousands of 17 | records into memory all at the same time! 18 | 19 | If you're 100% positive that this migration is already safe, then wrap the 20 | call to `each` in a `safety_assured` block. 21 | 22 | class #{migration_name} < ActiveRecord::Migration 23 | def change 24 | safety_assured do 25 | # use ActiveRecord::Relation.each in this block 26 | end 27 | end 28 | end 29 | MESSAGE 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/validation.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Validation 3 | def self.validate!(type, *args) 4 | return unless Migration.migrating? && Migration.unsafe? 5 | 6 | begin 7 | validator = type.to_s.classify 8 | const_get(validator).new(Migration.current, *args).validate! 9 | rescue NameError 10 | raise UndefinedValidationError.new(validator) 11 | end 12 | end 13 | 14 | attr_reader :migration, :args 15 | 16 | def initialize(migration, *args) 17 | @migration = migration 18 | @args = args 19 | end 20 | 21 | def error!(message) 22 | error = UnsafeMigrationError 23 | debug = "#{error}: #{migration_name} is unsafe!" 24 | message = [message, debug, nil].join("\n") 25 | raise error.new(message) 26 | end 27 | 28 | def migration_name 29 | migration.class.name 30 | end 31 | 32 | def options 33 | args.last.is_a?(Hash) ? args.last : {} 34 | end 35 | 36 | def validate! 37 | raise NotImplementedError 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 LendingHome - github@lendinghome.com 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations.rb: -------------------------------------------------------------------------------- 1 | require "active_record" 2 | 3 | require_relative "zero_downtime_migrations/data" 4 | require_relative "zero_downtime_migrations/dsl" 5 | require_relative "zero_downtime_migrations/error" 6 | require_relative "zero_downtime_migrations/migration" 7 | require_relative "zero_downtime_migrations/relation" 8 | require_relative "zero_downtime_migrations/validation" 9 | require_relative "zero_downtime_migrations/validation/add_column" 10 | require_relative "zero_downtime_migrations/validation/add_index" 11 | require_relative "zero_downtime_migrations/validation/ddl_migration" 12 | require_relative "zero_downtime_migrations/validation/find_each" 13 | require_relative "zero_downtime_migrations/validation/mixed_migration" 14 | 15 | ActiveRecord::Migration.send(:prepend, ZeroDowntimeMigrations::Migration) 16 | ActiveRecord::Schema.send(:prepend, ZeroDowntimeMigrations::Migration) 17 | 18 | module ZeroDowntimeMigrations 19 | GEMSPEC = name.underscore.concat(".gemspec") 20 | 21 | class << self 22 | def gemspec 23 | @gemspec ||= Gem::Specification.load(root.join(GEMSPEC).to_s) 24 | end 25 | 26 | def root 27 | @root ||= Pathname.new(__FILE__).dirname.dirname 28 | end 29 | 30 | def version 31 | gemspec.version.to_s 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/validation/ddl_migration.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Validation 3 | class DdlMigration < Validation 4 | def validate! 5 | return unless migration.ddl_disabled? && !Migration.index? 6 | error!(message) 7 | end 8 | 9 | private 10 | 11 | def message 12 | <<-MESSAGE.strip_heredoc 13 | Disabling the DDL transaction is unsafe! 14 | 15 | The DDL transaction should only be disabled for migrations that add indexes. 16 | All other types of migrations should keep the DDL transaction enabled so 17 | that changes can be rolled back if any unexpected errors occur. 18 | 19 | Any other data or schema changes must live in their own migration files with 20 | the DDL transaction enabled just in case they need to be rolled back. 21 | 22 | If you're 100% positive that this migration is already safe, then simply 23 | add a call to `safety_assured` to your migration. 24 | 25 | class #{migration_name} < ActiveRecord::Migration 26 | disable_ddl_transaction! 27 | safety_assured 28 | 29 | def change 30 | # ... 31 | end 32 | end 33 | MESSAGE 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/validation/mixed_migration.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Validation 3 | class MixedMigration < Validation 4 | def validate! 5 | return unless Migration.mixed? 6 | error!(message) 7 | end 8 | 9 | private 10 | 11 | def message 12 | <<-MESSAGE.strip_heredoc 13 | Mixing data/index/schema changes in the same migration is unsafe! 14 | 15 | Instead, let's split apart these types of migrations into separate files. 16 | 17 | * Introduce schema changes with methods like `create_table` or `add_column` 18 | in one file. These should be run within a DDL transaction so that they 19 | can be rolled back if there are any issues. 20 | 21 | * Update data with methods like `update_all` or `save` in another file. 22 | Data migrations tend to be much more error prone than changing the 23 | schema or adding indexes. 24 | 25 | * Add indexes concurrently within their own file as well. Indexes should 26 | be created without the DDL transaction enabled to avoid table locking. 27 | 28 | If you're 100% positive that this migration is already safe, then simply 29 | add a call to `safety_assured` to your migration. 30 | 31 | class #{migration_name} < ActiveRecord::Migration 32 | safety_assured 33 | 34 | def change 35 | # ... 36 | end 37 | end 38 | MESSAGE 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /spec/zero_downtime_migrations/validation/find_each_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations::Validation::FindEach do 2 | let(:error) { ZeroDowntimeMigrations::UnsafeMigrationError } 3 | 4 | context "with data migrations using each" do 5 | let(:migration) do 6 | Class.new(ActiveRecord::Migration[5.0]) do 7 | def change 8 | User.all.each 9 | end 10 | end 11 | end 12 | 13 | it "raises an unsafe migration error" do 14 | expect { migration.migrate(:up) }.to raise_error(error) 15 | end 16 | end 17 | 18 | context "with data migrations using each within safety_assured" do 19 | let(:migration) do 20 | Class.new(ActiveRecord::Migration[5.0]) do 21 | def change 22 | safety_assured do 23 | User.all.each 24 | end 25 | end 26 | end 27 | end 28 | 29 | it "does not raise an unsafe migration error" do 30 | expect { migration.migrate(:up) }.not_to raise_error(error) 31 | end 32 | end 33 | 34 | context "with data migrations using find_each" do 35 | let(:migration) do 36 | Class.new(ActiveRecord::Migration[5.0]) do 37 | def change 38 | User.all.find_each 39 | end 40 | end 41 | end 42 | 43 | it "does not raise an unsafe migration error" do 44 | expect { migration.migrate(:up) }.not_to raise_error(error) 45 | end 46 | end 47 | 48 | context "outside of a migration" do 49 | it "does not raise an unsafe migration error" do 50 | expect { User.all.each }.not_to raise_error(error) 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /spec/zero_downtime_migrations/validation_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations::Validation do 2 | subject { described_class.new(migration, *args) } 3 | 4 | let(:migration) { double("migration") } 5 | let(:args) { [] } 6 | 7 | describe ".validate!" do 8 | let(:error) { ZeroDowntimeMigrations::UndefinedValidationError } 9 | 10 | it "raises UndefinedValidationError if one does not exist" do 11 | expect { described_class.validate!(:invalid?) }.to raise_error(error) 12 | end 13 | end 14 | 15 | describe "#args" do 16 | it "returns the initialized args" do 17 | expect(subject.args).to eq(args) 18 | end 19 | end 20 | 21 | describe "#error!" do 22 | let(:error) { ZeroDowntimeMigrations::UnsafeMigrationError } 23 | let(:message) { "test" } 24 | 25 | it "raises a new UnsafeMigrationError" do 26 | expect { subject.error!(message) }.to raise_error(error) 27 | end 28 | end 29 | 30 | describe "#migration" do 31 | it "returns the initialized migration" do 32 | expect(subject.migration).to eq(migration) 33 | end 34 | end 35 | 36 | describe "#options" do 37 | context "when the last arg is a hash" do 38 | let(:args) { [:one, :two, { three: :four }] } 39 | it "returns the last arg" do 40 | expect(subject.options).to eq(three: :four) 41 | end 42 | end 43 | 44 | context "when the last arg is not a hash" do 45 | let(:args) { [:one, :two] } 46 | 47 | it "returns the an empty hash" do 48 | expect(subject.options).to eq({}) 49 | end 50 | end 51 | end 52 | 53 | describe "#validate!" do 54 | it "raises NotImplementedError" do 55 | expect { subject.validate! }.to raise_error(NotImplementedError) 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/zero_downtime_migrations/validation/add_column_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations::Validation::AddColumn do 2 | let(:error) { ZeroDowntimeMigrations::UnsafeMigrationError } 3 | 4 | context "with a default" do 5 | let(:migration) do 6 | Class.new(ActiveRecord::Migration[5.0]) do 7 | def change 8 | add_column :users, :active, :boolean, default: true 9 | end 10 | end 11 | end 12 | 13 | it "raises an unsafe migration error" do 14 | expect { migration.migrate(:up) }.to raise_error(error) 15 | end 16 | end 17 | 18 | context "with a false default" do 19 | let(:migration) do 20 | Class.new(ActiveRecord::Migration[5.0]) do 21 | def change 22 | add_column :users, :active, :boolean, default: false 23 | end 24 | end 25 | end 26 | 27 | it "raises an unsafe migration error" do 28 | expect { migration.migrate(:up) }.to raise_error(error) 29 | end 30 | end 31 | 32 | context "with a null default" do 33 | let(:migration) do 34 | Class.new(ActiveRecord::Migration[5.0]) do 35 | def change 36 | add_column :users, :active, :boolean, default: nil 37 | end 38 | end 39 | end 40 | 41 | it "does not raise an unsafe migration error" do 42 | expect { migration.migrate(:up) }.not_to raise_error(error) 43 | end 44 | end 45 | 46 | context "without a default" do 47 | let(:migration) do 48 | Class.new(ActiveRecord::Migration[5.0]) do 49 | def change 50 | add_column :users, :active, :boolean 51 | end 52 | end 53 | end 54 | 55 | it "does not raise an unsafe migration error" do 56 | expect { migration.migrate(:up) }.not_to raise_error(error) 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/validation/add_index.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Validation 3 | class AddIndex < Validation 4 | def validate! 5 | return if concurrent? && migration.ddl_disabled? 6 | error!(message) 7 | end 8 | 9 | private 10 | 11 | def message 12 | <<-MESSAGE.strip_heredoc 13 | Adding a non-concurrent index is unsafe! 14 | 15 | This action can lock your database table while indexing existing data! 16 | 17 | Instead, let's add the index concurrently in its own migration with 18 | the DDL transaction disabled. 19 | 20 | This allows PostgreSQL to build the index without locking in a way 21 | that prevent concurrent inserts, updates, or deletes on the table. 22 | Standard indexes lock out writes (but not reads) on the table. 23 | 24 | class Index#{table_title}On#{column_title} < ActiveRecord::Migration 25 | disable_ddl_transaction! 26 | 27 | def change 28 | add_index :#{table}, #{column.inspect}, algorithm: :concurrently 29 | end 30 | end 31 | 32 | If you're 100% positive that this migration is already safe, then wrap the 33 | call to `add_index` in a `safety_assured` block. 34 | 35 | class Index#{table_title}On#{column_title} < ActiveRecord::Migration 36 | def change 37 | safety_assured { add_index :#{table}, #{column.inspect} } 38 | end 39 | end 40 | MESSAGE 41 | end 42 | 43 | def concurrent? 44 | options[:algorithm] == :concurrently 45 | end 46 | 47 | def column 48 | args[1] 49 | end 50 | 51 | def column_title 52 | Array(column).map(&:to_s).join("_and_").camelize 53 | end 54 | 55 | def table 56 | args[0] 57 | end 58 | 59 | def table_title 60 | table.to_s.camelize 61 | end 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /spec/zero_downtime_migrations/validation/ddl_migration_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations::Validation::DdlMigration do 2 | let(:error) { ZeroDowntimeMigrations::UnsafeMigrationError } 3 | 4 | context "with a migration that adds a column with ddl disabled" do 5 | let(:migration) do 6 | Class.new(ActiveRecord::Migration[5.0]) do 7 | disable_ddl_transaction! 8 | 9 | def change 10 | add_column :users, :active, :boolean 11 | end 12 | end 13 | end 14 | 15 | it "raises an unsafe migration error" do 16 | expect { migration.migrate(:up) }.to raise_error(error) 17 | end 18 | end 19 | 20 | context "with a migration that queries data with ddl disabled" do 21 | let(:migration) do 22 | Class.new(ActiveRecord::Migration[5.0]) do 23 | disable_ddl_transaction! 24 | 25 | def change 26 | User.find_in_batches 27 | end 28 | end 29 | end 30 | 31 | it "raises an unsafe migration error" do 32 | expect { migration.migrate(:up) }.to raise_error(error) 33 | end 34 | end 35 | 36 | context "with a migration that updates data with ddl disabled" do 37 | let(:migration) do 38 | Class.new(ActiveRecord::Migration[5.0]) do 39 | disable_ddl_transaction! 40 | 41 | def change 42 | add_column :users, :active, :boolean 43 | User.update_all(active: true) 44 | end 45 | end 46 | end 47 | 48 | it "raises an unsafe migration error" do 49 | expect { migration.migrate(:up) }.to raise_error(error) 50 | end 51 | end 52 | 53 | context "with a migration that creates data with ddl disabled" do 54 | let(:migration) do 55 | Class.new(ActiveRecord::Migration[5.0]) do 56 | disable_ddl_transaction! 57 | 58 | def change 59 | User.new(email: "test").save! 60 | end 61 | end 62 | end 63 | 64 | it "raises an unsafe migration error" do 65 | expect { migration.migrate(:up) }.to raise_error(error) 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /bin/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | if [[ -z "$DOCKER" ]]; then 6 | echo "--- :buildkite: Building zero_downtime_migrations" 7 | buildkite-agent --version 8 | 9 | echo "branch: $BUILDKITE_BRANCH" 10 | echo "commit: $BUILDKITE_COMMIT" 11 | echo "image: $DOCKER_IMAGE" 12 | 13 | if [ -v REBUILD_CACHE ]; then 14 | echo "--- :minidisc: Pulling images" 15 | docker-compose pull --ignore-pull-failures 16 | fi 17 | 18 | echo "--- :docker: Starting docker" 19 | docker --version 20 | docker-compose --version 21 | 22 | echo "Building $DOCKER_IMAGE" 23 | docker-compose run app /app/bin/test $@ 24 | 25 | if [ -v REBUILD_CACHE ]; then 26 | echo "--- :recycle: Updating build cache" 27 | echo "The build was triggered with REBUILD_CACHE" 28 | 29 | echo "Generating cache.tar.lz4" 30 | tar -c --use-compress-program=lz4 -f cache.tar.lz4 "$BUILD_CACHE" \ 31 | || echo "Ignoring permission denied failures" 32 | 33 | [ -v BUILDKITE ] && buildkite-agent meta-data set "rebuild-cache" "1" 34 | fi 35 | 36 | exit 0 37 | fi 38 | 39 | echo "--- :terminal: Loading environment" 40 | echo "bash: `bash --version | head -1`" 41 | echo "git: `git --version`" 42 | echo "imagemagick: `identify --version | head -1`" 43 | echo "java: `java -version 2>&1 | head -1`" 44 | echo "os: `cat /etc/issue | head -1`" 45 | echo "phantomjs: `phantomjs --version`" 46 | 47 | echo "--- :ruby: Installing ruby" 48 | rbenv install --skip-existing 49 | ruby --version 50 | rbenv --version 51 | 52 | echo "--- :rubygems: Installing ruby gems" 53 | echo "gem $(gem --version)" 54 | bundler --version 55 | cpus=$(nproc 2>/dev/null || echo 1) 56 | [[ $cpus -eq 1 ]] && jobs=1 || jobs=$((cpus - 1)) 57 | options="--path vendor/bundle --jobs $jobs --retry 3 --frozen --no-cache --no-prune" 58 | bundle check || bundle install $options 59 | 60 | echo "--- :postgres: Creating database" 61 | sleep 3 # give postgres container time to boot 62 | createdb -h postgres -U postgres -w zero_downtime_migrations || true 63 | 64 | if [[ -z "$REBUILD_CACHE" ]]; then 65 | echo "+++ :rspec: Running rspec tests" 66 | echo "rspec $(bundle exec rspec --version)" 67 | bundle exec rspec $@ 68 | fi -------------------------------------------------------------------------------- /spec/zero_downtime_migrations/validation/mixed_migration_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe ZeroDowntimeMigrations::Validation::MixedMigration do 2 | let(:error) { ZeroDowntimeMigrations::UnsafeMigrationError } 3 | 4 | context "with a migration that adds a column and index" do 5 | let(:migration) do 6 | Class.new(ActiveRecord::Migration[5.0]) do 7 | def change 8 | add_column :users, :active, :boolean 9 | add_index :users, :active 10 | end 11 | end 12 | end 13 | 14 | it "raises an unsafe migration error" do 15 | expect { migration.migrate(:up) }.to raise_error(error) 16 | end 17 | end 18 | 19 | context "with a migration that adds a column and queries data" do 20 | let(:migration) do 21 | Class.new(ActiveRecord::Migration[5.0]) do 22 | def change 23 | add_column :users, :active, :boolean 24 | User.find_in_batches 25 | end 26 | end 27 | end 28 | 29 | it "raises an unsafe migration error" do 30 | expect { migration.migrate(:up) }.to raise_error(error) 31 | end 32 | end 33 | 34 | context "with a migration that adds a column and updates data" do 35 | let(:migration) do 36 | Class.new(ActiveRecord::Migration[5.0]) do 37 | def change 38 | add_column :users, :active, :boolean 39 | User.update_all(active: true) 40 | end 41 | end 42 | end 43 | 44 | it "raises an unsafe migration error" do 45 | expect { migration.migrate(:up) }.to raise_error(error) 46 | end 47 | end 48 | 49 | context "with a migration that adds an index and updates data" do 50 | let(:migration) do 51 | Class.new(ActiveRecord::Migration[5.0]) do 52 | def change 53 | User.where(email: nil).delete_all 54 | add_index :users, :created_at 55 | end 56 | end 57 | end 58 | 59 | it "raises an unsafe migration error" do 60 | expect { migration.migrate(:up) }.to raise_error(error) 61 | end 62 | end 63 | 64 | context "with a migration that adds a column and creates data" do 65 | let(:migration) do 66 | Class.new(ActiveRecord::Migration[5.0]) do 67 | def change 68 | add_column :users, :active, :boolean 69 | User.new(email: "test").save! 70 | end 71 | end 72 | end 73 | 74 | it "raises an unsafe migration error" do 75 | expect { migration.migrate(:up) }.to raise_error(error) 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/migration.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | module Migration 3 | extend DSL 4 | 5 | def self.prepended(mod) 6 | mod.singleton_class.prepend(DSL) 7 | end 8 | 9 | def initialize(*) 10 | ActiveRecord::Base.send(:prepend, Data) 11 | ActiveRecord::Relation.send(:prepend, Relation) 12 | super 13 | end 14 | 15 | def ddl_disabled? 16 | !!disable_ddl_transaction 17 | end 18 | 19 | def define(*) 20 | Migration.current = self 21 | Migration.safe = true 22 | super.tap { Migration.current = nil } 23 | end 24 | 25 | def migrate(direction) 26 | @direction = direction 27 | 28 | Migration.current = self 29 | Migration.data = false 30 | Migration.ddl = false 31 | Migration.index = false 32 | Migration.safe ||= reverse_migration? || rollup_migration? 33 | 34 | super.tap do 35 | validate(:ddl_migration) 36 | validate(:mixed_migration) 37 | Migration.current = nil 38 | Migration.safe = false 39 | end 40 | end 41 | 42 | private 43 | 44 | def ddl_method?(method) 45 | %i( 46 | add_belongs_to 47 | add_column 48 | add_foreign_key 49 | add_reference 50 | add_timestamps 51 | change_column 52 | change_column_default 53 | change_column_null 54 | change_table 55 | create_join_table 56 | create_table 57 | drop_join_table 58 | drop_table 59 | remove_belongs_to 60 | remove_column 61 | remove_columns 62 | remove_foreign_key 63 | remove_index 64 | remove_index! 65 | remove_reference 66 | remove_timestamps 67 | rename_column 68 | rename_column_indexes 69 | rename_index 70 | rename_table 71 | rename_table_indexes 72 | ).include?(method) 73 | end 74 | 75 | def index_method?(method) 76 | %i(add_index).include?(method) 77 | end 78 | 79 | def method_missing(method, *args) 80 | Migration.ddl = true if ddl_method?(method) 81 | Migration.index = true if index_method?(method) 82 | validate(method, *args) 83 | super 84 | end 85 | 86 | def reverse_migration? 87 | @direction == :down 88 | end 89 | 90 | def rollup_migration? 91 | self.class.name == "RollupMigrations" 92 | end 93 | 94 | def safety_assured 95 | safe = Migration.safe 96 | Migration.safe = true 97 | yield 98 | ensure 99 | Migration.safe = safe 100 | end 101 | 102 | def validate(type, *args) 103 | Validation.validate!(type, *args) 104 | rescue UndefinedValidationError 105 | nil 106 | end 107 | end 108 | end 109 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | zero_downtime_migrations (0.0.7) 5 | activerecord 6 | 7 | GEM 8 | remote: https://www.rubygems.org/ 9 | specs: 10 | actionpack (5.0.0.1) 11 | actionview (= 5.0.0.1) 12 | activesupport (= 5.0.0.1) 13 | rack (~> 2.0) 14 | rack-test (~> 0.6.3) 15 | rails-dom-testing (~> 2.0) 16 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 17 | actionview (5.0.0.1) 18 | activesupport (= 5.0.0.1) 19 | builder (~> 3.1) 20 | erubis (~> 2.7.0) 21 | rails-dom-testing (~> 2.0) 22 | rails-html-sanitizer (~> 1.0, >= 1.0.2) 23 | activemodel (5.0.0.1) 24 | activesupport (= 5.0.0.1) 25 | activerecord (5.0.0.1) 26 | activemodel (= 5.0.0.1) 27 | activesupport (= 5.0.0.1) 28 | arel (~> 7.0) 29 | activesupport (5.0.0.1) 30 | concurrent-ruby (~> 1.0, >= 1.0.2) 31 | i18n (~> 0.7) 32 | minitest (~> 5.1) 33 | tzinfo (~> 1.1) 34 | arel (7.1.4) 35 | builder (3.2.2) 36 | codeclimate-test-reporter (0.6.0) 37 | simplecov (>= 0.7.1, < 1.0.0) 38 | combustion (0.5.5) 39 | activesupport (>= 3.0.0) 40 | railties (>= 3.0.0) 41 | thor (>= 0.14.6) 42 | concurrent-ruby (1.0.2) 43 | diff-lcs (1.2.5) 44 | docile (1.1.5) 45 | erubis (2.7.0) 46 | i18n (0.7.0) 47 | json (2.0.2) 48 | loofah (2.0.3) 49 | nokogiri (>= 1.5.9) 50 | method_source (0.8.2) 51 | mini_portile2 (2.1.0) 52 | minitest (5.9.1) 53 | nokogiri (1.6.8.1) 54 | mini_portile2 (~> 2.1.0) 55 | pg (0.19.0) 56 | rack (2.0.1) 57 | rack-test (0.6.3) 58 | rack (>= 1.0) 59 | rails-dom-testing (2.0.1) 60 | activesupport (>= 4.2.0, < 6.0) 61 | nokogiri (~> 1.6.0) 62 | rails-html-sanitizer (1.0.3) 63 | loofah (~> 2.0) 64 | railties (5.0.0.1) 65 | actionpack (= 5.0.0.1) 66 | activesupport (= 5.0.0.1) 67 | method_source 68 | rake (>= 0.8.7) 69 | thor (>= 0.18.1, < 2.0) 70 | rake (11.3.0) 71 | rspec-core (3.5.4) 72 | rspec-support (~> 3.5.0) 73 | rspec-expectations (3.5.0) 74 | diff-lcs (>= 1.2.0, < 2.0) 75 | rspec-support (~> 3.5.0) 76 | rspec-mocks (3.5.0) 77 | diff-lcs (>= 1.2.0, < 2.0) 78 | rspec-support (~> 3.5.0) 79 | rspec-rails (3.5.2) 80 | actionpack (>= 3.0) 81 | activesupport (>= 3.0) 82 | railties (>= 3.0) 83 | rspec-core (~> 3.5.0) 84 | rspec-expectations (~> 3.5.0) 85 | rspec-mocks (~> 3.5.0) 86 | rspec-support (~> 3.5.0) 87 | rspec-support (3.5.0) 88 | simplecov (0.12.0) 89 | docile (~> 1.1.0) 90 | json (>= 1.8, < 3) 91 | simplecov-html (~> 0.10.0) 92 | simplecov-html (0.10.0) 93 | thor (0.19.1) 94 | thread_safe (0.3.5) 95 | tzinfo (1.2.2) 96 | thread_safe (~> 0.1) 97 | 98 | PLATFORMS 99 | ruby 100 | 101 | DEPENDENCIES 102 | codeclimate-test-reporter 103 | combustion 104 | pg 105 | rspec-rails 106 | zero_downtime_migrations! 107 | 108 | BUNDLED WITH 109 | 1.12.5 110 | -------------------------------------------------------------------------------- /lib/zero_downtime_migrations/validation/add_column.rb: -------------------------------------------------------------------------------- 1 | module ZeroDowntimeMigrations 2 | class Validation 3 | class AddColumn < Validation 4 | def validate! 5 | return if options[:default].nil? # only nil is safe 6 | error!(message) 7 | end 8 | 9 | private 10 | 11 | def message 12 | <<-MESSAGE.strip_heredoc 13 | Adding a column with a default is unsafe! 14 | 15 | This can take a long time with significant database 16 | size or traffic and lock your table! 17 | 18 | First let’s add the column without a default. When we add 19 | a column with a default it has to lock the table while it 20 | performs an UPDATE for ALL rows to set this new default. 21 | 22 | class Add#{column_title}To#{table_title} < ActiveRecord::Migration 23 | def change 24 | add_column :#{table}, :#{column}, :#{column_type} 25 | end 26 | end 27 | 28 | Then we’ll set the new column default in a separate migration. 29 | Note that this does not update any existing data! This only 30 | sets the default for newly inserted rows going forward. 31 | 32 | class AddDefault#{column_title}To#{table_title} < ActiveRecord::Migration 33 | def change 34 | change_column_default :#{table}, :#{column}, #{column_default} 35 | end 36 | end 37 | 38 | Finally we’ll backport the default value for existing data in 39 | batches. This should be done in its own migration as well. 40 | Updating in batches allows us to lock 1000 rows at a time 41 | (or whatever batch size we prefer). 42 | 43 | class BackportDefault#{column_title}To#{table_title} < ActiveRecord::Migration 44 | def change 45 | #{table_model}.select(:id).find_in_batches.with_index do |records, index| 46 | puts "Processing batch \#{index + 1}\\r" 47 | #{table_model}.where(id: records).update_all(#{column}: #{column_default}) 48 | end 49 | end 50 | end 51 | 52 | Note that in some cases it may not even be necessary to backport a default value. 53 | 54 | class #{table_model} < ActiveRecord::Base 55 | def #{column} 56 | self["#{column}"] ||= #{column_default} 57 | end 58 | end 59 | 60 | If you're 100% positive that this migration is already safe, then wrap the 61 | call to `add_column` in a `safety_assured` block. 62 | 63 | class Add#{column_title}To#{table_title} < ActiveRecord::Migration 64 | def change 65 | safety_assured { add_column :#{table}, :#{column}, :#{column_type}, default: #{column_default} } 66 | end 67 | end 68 | MESSAGE 69 | end 70 | 71 | def column 72 | args[1] 73 | end 74 | 75 | def column_default 76 | options[:default].inspect 77 | end 78 | 79 | def column_title 80 | column.to_s.camelize 81 | end 82 | 83 | def column_type 84 | args[2] 85 | end 86 | 87 | def table 88 | args[0] 89 | end 90 | 91 | def table_model 92 | table_title.singularize 93 | end 94 | 95 | def table_title 96 | table.to_s.camelize 97 | end 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ![LendingHome](https://cloud.githubusercontent.com/assets/2419/19467866/7efa93a8-94c8-11e6-93e7-4375dbb8a7bc.png) zero_downtime_migrations 2 | [![Code Climate](https://codeclimate.com/github/LendingHome/zero_downtime_migrations/badges/gpa.svg)](https://codeclimate.com/github/LendingHome/zero_downtime_migrations) [![Coverage](https://codeclimate.com/github/LendingHome/zero_downtime_migrations/badges/coverage.svg)](https://codeclimate.com/github/LendingHome/zero_downtime_migrations) [![Gem Version](https://badge.fury.io/rb/zero_downtime_migrations.svg)](http://badge.fury.io/rb/zero_downtime_migrations) 3 | 4 | > Zero downtime migrations with ActiveRecord and PostgreSQL. 5 | 6 | Catch problematic migrations at development/test time! Heavily inspired by these similar projects: 7 | 8 | * https://github.com/ankane/strong_migrations 9 | * https://github.com/foobarfighter/safe-migrations 10 | 11 | ## Installation 12 | 13 | Simply add this gem to the project `Gemfile`. 14 | 15 | ```ruby 16 | gem "zero_downtime_migrations" 17 | ``` 18 | 19 | ## Usage 20 | 21 | This gem will automatically **raise exceptions when potential database locking migrations are detected**. 22 | 23 | It checks for common things like: 24 | 25 | * Adding a column with a default 26 | * Adding a non-concurrent index 27 | * Mixing data changes with index or schema migrations 28 | * Performing data or schema migrations with the DDL transaction disabled 29 | * Using `each` instead of `find_each` to loop thru `ActiveRecord` objects 30 | 31 | These exceptions display clear instructions of how to perform the same operation the "zero downtime way". 32 | 33 | ## Validations 34 | 35 | ### Adding a column with a default 36 | 37 | #### Bad 38 | 39 | This can take a long time with significant database size or traffic and lock your table! 40 | 41 | ```ruby 42 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 43 | def change 44 | add_column :posts, :published, :boolean, default: true 45 | end 46 | end 47 | ``` 48 | 49 | #### Good 50 | 51 | First let’s add the column without a default. When we add a column with a default it has to lock the table while it performs an UPDATE for ALL rows to set this new default. 52 | 53 | ```ruby 54 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 55 | def change 56 | add_column :posts, :published, :boolean 57 | end 58 | end 59 | ``` 60 | 61 | Then we’ll set the new column default in a separate migration. Note that this does not update any existing data! This only sets the default for newly inserted rows going forward. 62 | 63 | ```ruby 64 | class SetPublishedDefaultOnPosts < ActiveRecord::Migration[5.0] 65 | def change 66 | change_column_default :posts, :published, true 67 | end 68 | end 69 | ``` 70 | 71 | Finally we’ll backport the default value for existing data in batches. This should be done in its own migration as well. Updating in batches allows us to lock 1000 rows at a time (or whatever batch size we prefer). 72 | 73 | ```ruby 74 | class BackportPublishedDefaultOnPosts < ActiveRecord::Migration[5.0] 75 | def change 76 | Post.select(:id).find_in_batches.with_index do |batch, index| 77 | puts "Processing batch #{index}\r" 78 | Post.where(id: batch).update_all(published: true) 79 | end 80 | end 81 | end 82 | ``` 83 | 84 | ### Adding an index concurrently 85 | 86 | #### Bad 87 | 88 | This action can lock your database table while indexing existing data! 89 | 90 | ```ruby 91 | class IndexUsersOnEmail < ActiveRecord::Migration[5.0] 92 | def change 93 | add_index :users, :email 94 | end 95 | end 96 | ``` 97 | 98 | #### Good 99 | 100 | Instead, let's add the index concurrently in its own migration with the DDL transaction disabled. 101 | 102 | This allows PostgreSQL to build the index without locking in a way that prevent concurrent inserts, updates, or deletes on the table. Standard indexes lock out writes (but not reads) on the table. 103 | 104 | ```ruby 105 | class IndexUsersOnEmail < ActiveRecord::Migration[5.0] 106 | disable_ddl_transaction! 107 | 108 | def change 109 | add_index :users, :email, algorithm: :concurrently 110 | end 111 | end 112 | ``` 113 | 114 | ### Mixing data/index/schema migrations 115 | 116 | #### Bad 117 | 118 | Performing migrations that change the schema, update data, or add indexes within one big transaction is unsafe! 119 | 120 | ```ruby 121 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 122 | def change 123 | add_column :posts, :published, :boolean 124 | Post.update_all(published: true) 125 | add_index :posts, :published 126 | end 127 | end 128 | ``` 129 | 130 | #### Good 131 | 132 | Instead, let's split apart these types of migrations into separate files. 133 | 134 | * Introduce schema changes with methods like `create_table` or `add_column` in one file. These should be run within a DDL transaction so that they can be rolled back if there are any issues. 135 | * Update data with methods like `update_all` or `save` in another file. Data migrations tend to be much more error prone than changing the schema or adding indexes. 136 | * Add indexes concurrently within their own file as well. Indexes should be created without the DDL transaction enabled to avoid table locking. 137 | 138 | ```ruby 139 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 140 | def change 141 | add_column :posts, :published, :boolean 142 | end 143 | end 144 | ``` 145 | 146 | ```ruby 147 | class BackportPublishedOnPosts < ActiveRecord::Migration[5.0] 148 | def change 149 | Post.update_all(published: true) 150 | end 151 | end 152 | ``` 153 | 154 | ```ruby 155 | class IndexPublishedOnPosts < ActiveRecord::Migration[5.0] 156 | disable_ddl_transaction! 157 | 158 | def change 159 | add_index :posts, :published, algorithm: :concurrently 160 | end 161 | end 162 | ``` 163 | 164 | ### Disabling the DDL transaction 165 | 166 | #### Bad 167 | 168 | The DDL transaction should only be disabled for migrations that add indexes. All other types of migrations should keep the DDL transaction enabled so that changes can be rolled back if any unexpected errors occur. 169 | 170 | ```ruby 171 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 172 | disable_ddl_transaction! 173 | 174 | def change 175 | add_column :posts, :published, :boolean 176 | end 177 | end 178 | ``` 179 | 180 | ```ruby 181 | class UpdatePublishedOnPosts < ActiveRecord::Migration[5.0] 182 | disable_ddl_transaction! 183 | 184 | def change 185 | Post.update_all(published: true) 186 | end 187 | end 188 | ``` 189 | 190 | #### Good 191 | 192 | Any other data or schema changes must live in their own migration files with the DDL transaction enabled just in case they make changes that need to be rolled back. 193 | 194 | ```ruby 195 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 196 | def change 197 | add_column :posts, :published, :boolean 198 | end 199 | end 200 | ``` 201 | 202 | ```ruby 203 | class UpdatePublishedOnPosts < ActiveRecord::Migration[5.0] 204 | def change 205 | Post.update_all(published: true) 206 | end 207 | end 208 | ``` 209 | 210 | ### Looping thru `ActiveRecord::Base` objects 211 | 212 | #### Bad 213 | 214 | This might accidentally load tens or hundreds of thousands of records into memory all at the same time! 215 | 216 | ```ruby 217 | class BackportPublishedDefaultOnPosts < ActiveRecord::Migration[5.0] 218 | def change 219 | Post.all.each do |post| 220 | post.update_attribute(published: true) 221 | end 222 | end 223 | end 224 | ``` 225 | 226 | #### Good 227 | 228 | Let's use the `find_each` method to fetch records in batches instead. 229 | 230 | ```ruby 231 | class BackportPublishedDefaultOnPosts < ActiveRecord::Migration[5.0] 232 | def change 233 | Post.all.find_each do |post| 234 | post.update_attribute(published: true) 235 | end 236 | end 237 | end 238 | ``` 239 | 240 | ### TODO 241 | 242 | * Changing a column type 243 | * Removing a column 244 | * Renaming a column 245 | * Renaming a table 246 | 247 | ## Disabling "zero downtime migration" enforcements 248 | 249 | We can disable any of these "zero downtime migration" enforcements by wrapping them in a `safety_assured` block. 250 | 251 | ```ruby 252 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 253 | def change 254 | safety_assured do 255 | add_column :posts, :published, :boolean, default: true 256 | end 257 | end 258 | end 259 | ``` 260 | 261 | We can also mark an entire migration as safe by using the `safety_assured` helper method. 262 | 263 | ```ruby 264 | class AddPublishedToPosts < ActiveRecord::Migration[5.0] 265 | safety_assured 266 | 267 | def change 268 | add_column :posts, :published, :boolean 269 | Post.where("created_at >= ?", 1.day.ago).update_all(published: true) 270 | end 271 | end 272 | ``` 273 | 274 | Enforcements can be globally disabled by setting `ENV["SAFETY_ASSURED"]` when running migrations. 275 | 276 | ```bash 277 | SAFETY_ASSURED=1 bundle exec rake db:migrate --trace 278 | ``` 279 | 280 | These enforcements are **automatically disabled by default for the following scenarios**: 281 | 282 | * The database schema is being loaded with `rake db:schema:load` instead of `db:migrate` 283 | * The current migration is a reverse (down) migration 284 | * The current migration is named `RollupMigrations` 285 | 286 | ## Testing 287 | 288 | ```bash 289 | bundle exec rspec 290 | ``` 291 | 292 | ## Contributing 293 | 294 | * Fork the project. 295 | * Make your feature addition or bug fix. 296 | * Add tests for it. This is important so we don't break it in a future version unintentionally. 297 | * Commit, do not mess with the version or history. 298 | * Open a pull request. Bonus points for topic branches. 299 | 300 | ## Authors 301 | 302 | * [Sean Huber](https://github.com/shuber) 303 | 304 | ## License 305 | 306 | [MIT](https://github.com/lendinghome/zero_downtime_migrations/blob/master/LICENSE) - Copyright © 2016 LendingHome 307 | --------------------------------------------------------------------------------