├── lib ├── db_leftovers │ ├── version.rb │ ├── constraint.rb │ ├── definition.rb │ ├── table_dsl.rb │ ├── generic_database_interface.rb │ ├── index.rb │ ├── foreign_key.rb │ ├── mysql_database_interface.rb │ ├── postgres_database_interface.rb │ └── dsl.rb ├── db_leftovers.rb └── tasks │ └── leftovers.rake ├── .document ├── spec ├── support │ ├── sql_matcher.rb │ ├── mock_database_interface.rb │ └── shared_db_tests.rb ├── spec_helper.rb ├── mysql_spec.rb ├── config │ └── database.yml.sample ├── postgres_spec.rb └── db_leftovers_spec.rb ├── Changelog ├── TODO ├── Rakefile ├── Gemfile ├── db_leftovers.gemspec ├── .gitignore ├── LICENSE.txt ├── Gemfile.lock └── README.md /lib/db_leftovers/version.rb: -------------------------------------------------------------------------------- 1 | module DbLeftovers 2 | VERSION = '1.6.0' 3 | end 4 | -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | bin/* 3 | - 4 | features/**/*.feature 5 | LICENSE.txt 6 | -------------------------------------------------------------------------------- /spec/support/sql_matcher.rb: -------------------------------------------------------------------------------- 1 | 2 | RSpec::Matchers.define :have_seen_sql do |sql| 3 | match do |db| 4 | db.saw_sql(sql) 5 | end 6 | 7 | failure_message_for_should do |db| 8 | "Should have seen...\n#{sql}\n...but instead saw...\n#{db.sqls.join("\n.....\n")}" 9 | end 10 | end 11 | 12 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | 1.6.0 2 | ----- 3 | 4 | - Support foreign keys that are `DEFERRABLE INITIALLY DEFERRED` and 5 | `DEFERRABLE INITIALLY IMMEDIATE`. 6 | 7 | 1.5.0 8 | ----- 9 | 10 | - Permit custom names for foreign keys. 11 | 12 | 1.4.2 13 | ----- 14 | 15 | - Fixed problem with dropping and re-creating indexes every time. 16 | 17 | 1.4.1 18 | ----- 19 | 20 | - Improved feedback so we print the expression on functional indexes. 21 | 22 | 23 | 1.4.0 24 | ----- 25 | 26 | - Added support for functional indexes. 27 | 28 | 29 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - It'd be nice to have another Rake task that will read your database and generate a new `db_leftovers.rb` file, to help people migrating existing projects that already have lots of tables. 2 | 3 | - Support multi-column foreign keys. 4 | 5 | - Support custom names for foreign keys. (But then watch out excluding these from the list of indexes on MySQL!) 6 | 7 | - Support CREATE INDEX CONCURRENTLY on Postgres as an option to Rake or the `define` call. 8 | 9 | - Support NOT VALID for Postgres 9.1+ foreign keys and Postgres 9.2+ CHECK constraints. 10 | 11 | 12 | -------------------------------------------------------------------------------- /lib/db_leftovers.rb: -------------------------------------------------------------------------------- 1 | require 'db_leftovers/generic_database_interface.rb' 2 | require 'db_leftovers/postgres_database_interface.rb' 3 | require 'db_leftovers/mysql_database_interface.rb' 4 | require 'db_leftovers/index.rb' 5 | require 'db_leftovers/foreign_key.rb' 6 | require 'db_leftovers/constraint.rb' 7 | require 'db_leftovers/table_dsl.rb' 8 | require 'db_leftovers/dsl.rb' 9 | require 'db_leftovers/definition.rb' 10 | 11 | module DBLeftovers 12 | 13 | class RakeTie < Rails::Railtie 14 | rake_tasks do 15 | load 'tasks/leftovers.rake' 16 | end 17 | end 18 | 19 | end 20 | 21 | -------------------------------------------------------------------------------- /lib/db_leftovers/constraint.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class Constraint 4 | attr_accessor :constraint_name, :on_table, :check 5 | 6 | def initialize(constraint_name, on_table, check) 7 | @constraint_name = constraint_name.to_s 8 | @on_table = on_table.to_s 9 | @check = check 10 | end 11 | 12 | def equals(other) 13 | other.constraint_name == constraint_name and 14 | other.on_table == on_table and 15 | other.check == check 16 | end 17 | 18 | def to_s 19 | "<#{@constraint_name}: #{@on_table} CHECK (#{@check})>" 20 | end 21 | 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'rubygems' 4 | require 'bundler' 5 | begin 6 | Bundler.setup(:default, :development) 7 | rescue Bundler::BundlerError => e 8 | $stderr.puts e.message 9 | $stderr.puts "Run `bundle install` to install missing gems" 10 | exit e.status_code 11 | end 12 | 13 | 14 | require 'rspec/core' 15 | require 'rspec/core/rake_task' 16 | RSpec::Core::RakeTask.new(:spec) do |spec| 17 | spec.pattern = FileList['spec/**/*_spec.rb'] 18 | end 19 | desc 'Default: run specs' 20 | task :default => :spec 21 | 22 | 23 | Bundler::GemHelper.install_tasks 24 | 25 | 26 | task :readme => [] do |task| 27 | `markdown README.md >README.html` 28 | end 29 | -------------------------------------------------------------------------------- /lib/db_leftovers/definition.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class Definition 4 | def self.define(opts={}, &block) 5 | opts = { 6 | :do_indexes => true, 7 | :do_foreign_keys => true, 8 | :do_constraints => true, 9 | :db_interface => nil 10 | }.merge(opts) 11 | dsl = DSL.new( 12 | :verbose => ENV['DB_LEFTOVERS_VERBOSE'] || false, 13 | :db_interface => opts[:db_interface]) 14 | dsl.define(&block) 15 | dsl.record_indexes if opts[:do_indexes] 16 | dsl.record_foreign_keys if opts[:do_foreign_keys] 17 | dsl.record_constraints if opts[:do_constraints] 18 | end 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /lib/db_leftovers/table_dsl.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class TableDSL 4 | def initialize(dsl, table_name) 5 | @dsl = dsl 6 | @table_name = table_name 7 | end 8 | 9 | def define(&block) 10 | instance_eval(&block) 11 | end 12 | 13 | def index(column_names, opts={}) 14 | @dsl.index(@table_name, column_names, opts) 15 | end 16 | 17 | def foreign_key(from_column=nil, to_table=nil, to_column=nil, opts={}) 18 | @dsl.foreign_key(@table_name, from_column, to_table, to_column, opts) 19 | end 20 | 21 | def check(constraint_name, check_expression) 22 | @dsl.check(@table_name, constraint_name, check_expression) 23 | end 24 | end 25 | 26 | end 27 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | # Add dependencies required to use your gem here. 3 | # Example: 4 | # gem "activesupport", ">= 2.3.5" 5 | 6 | gem 'rails', '>= 3.0.0' 7 | # gem 'rake' 8 | # gem 'activemodel', '= 3.0.11' 9 | # gem 'activesupport', '= 3.0.11' 10 | 11 | # Add dependencies to develop your gem here. 12 | # Include everything needed to run rake, tests, features, etc. 13 | group :development do 14 | gem "rspec", "~> 2.4.0" 15 | gem "bundler" 16 | end 17 | 18 | group :test do 19 | gem 'activerecord-postgresql-adapter', :platforms => :ruby 20 | gem 'activerecord-mysql2-adapter', :platforms => :ruby 21 | gem 'activerecord-jdbcpostgresql-adapter', :platforms => :jruby 22 | gem 'activerecord-jdbcmysql-adapter', :platforms => :jruby 23 | end 24 | -------------------------------------------------------------------------------- /lib/tasks/leftovers.rake: -------------------------------------------------------------------------------- 1 | namespace :db do 2 | 3 | desc "Set up indexes, foreign keys, and constraints" 4 | task :leftovers, [] => [:environment] do 5 | ENV['DB_LEFTOVERS_VERBOSE'] = ENV['VERBOSE'] || ENV['DB_LEFTOVERS_VERBOSE'] 6 | load File.join(::Rails.root.to_s, 'config', 'db_leftovers.rb') 7 | end 8 | 9 | desc "Drop all the indexes" 10 | task :drop_indexes, [] => [:environment] do 11 | DBLeftovers::Definition.define(:do_indexes => true, :do_foreign_keys => false, :do_constraints => false) do 12 | end 13 | end 14 | 15 | desc "Drop all the foreign keys" 16 | task :drop_foreign_keys, [] => [:environment] do 17 | DBLeftovers::Definition.define(:do_indexes => false, :do_foreign_keys => true, :do_constraints => false) do 18 | end 19 | end 20 | 21 | desc "Drop all the constraints" 22 | task :drop_constraints, [] => [:environment] do 23 | DBLeftovers::Definition.define(:do_indexes => false, :do_foreign_keys => false, :do_constraints => true) do 24 | end 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /db_leftovers.gemspec: -------------------------------------------------------------------------------- 1 | $:.push File.dirname(__FILE__) + '/lib' 2 | require 'db_leftovers/version' 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "db_leftovers" 6 | s.version = DbLeftovers::VERSION 7 | s.date = "2013-01-15" 8 | 9 | s.summary = "Used to define indexes and foreign keys for your Rails app" 10 | s.description = " Define indexes and foreign keys for your Rails app\n in one place using an easy-to-read DSL,\n then run a rake task to bring your database up-to-date.\n" 11 | 12 | s.authors = ["Paul A. Jungwirth"] 13 | s.homepage = "http://github.com/pjungwir/db_leftovers" 14 | s.email = "pj@illuminatedcomputing.com" 15 | 16 | s.licenses = ["MIT"] 17 | 18 | s.require_paths = ["lib"] 19 | s.executables = [] 20 | s.files = `git ls-files`.split("\n") 21 | s.test_files = `git ls-files -- {test,spec,fixtures}/*`.split("\n") 22 | 23 | s.add_runtime_dependency 'rails', '>= 3.0.0' 24 | s.add_development_dependency 'rspec', '~> 2.4.0' 25 | s.add_development_dependency 'bundler', '>= 0' 26 | 27 | end 28 | 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # rcov generated 2 | coverage 3 | 4 | # rdoc generated 5 | rdoc 6 | 7 | # yard generated 8 | doc 9 | .yardoc 10 | 11 | # bundler 12 | .bundle 13 | 14 | # jeweler generated 15 | pkg 16 | 17 | # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 18 | # 19 | # * Create a file at ~/.gitignore 20 | # * Include files you want ignored 21 | # * Run: git config --global core.excludesfile ~/.gitignore 22 | # 23 | # After doing this, these files will be ignored in all your git projects, 24 | # saving you from having to 'pollute' every project you touch with them 25 | # 26 | # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line) 27 | # 28 | # For MacOS: 29 | # 30 | #.DS_Store 31 | 32 | # For TextMate 33 | #*.tmproj 34 | #tmtags 35 | 36 | # For emacs: 37 | #*~ 38 | #\#* 39 | #.\#* 40 | 41 | # For vim: 42 | .*.swp 43 | .*.swo 44 | 45 | # For redcar: 46 | #.redcar 47 | 48 | # For rubinius: 49 | #*.rbc 50 | 51 | tmp 52 | README.html 53 | spec/config/database.yml 54 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 2 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 3 | require 'rspec' 4 | require 'rspec/matchers' 5 | require 'db_leftovers' 6 | 7 | # Requires supporting files with custom matchers and macros, etc., 8 | # in ./support/ and its subdirectories. 9 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f} 10 | 11 | RSpec.configure do |config| 12 | 13 | end 14 | 15 | 16 | def test_db_connection(adapter, conf) 17 | # DBI.connect(dbi_uri(adapter, conf), conf['username'], conf['password']) 18 | # RDBI.connect(adapter, conf) 19 | ActiveRecord::Base.establish_connection(conf) 20 | ActiveRecord::Base.connection 21 | end 22 | 23 | # def dbi_uri(adapter, conf) 24 | # "DBI:#{adapter}:#{conf.select{|k,v| k != 'username' and k != 'password'}.map{|k,v| "#{k}=#{v}"}.join(";")}" 25 | # end 26 | 27 | def test_database_yml(database) 28 | y = YAML.load(File.open(File.join(File.expand_path(File.dirname(__FILE__)), 'config', 'database.yml'))) 29 | y[database] 30 | rescue Errno::ENOENT 31 | return nil 32 | end 33 | 34 | 35 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Paul A. Jungwirth 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /spec/mysql_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails' 2 | require 'active_record' 3 | require File.expand_path(File.dirname(__FILE__) + '/spec_helper') 4 | 5 | def drop_all_mysql_tables(conn, database_name) 6 | table_names = conn.select_values("SELECT table_name FROM information_schema.tables WHERE table_schema = '#{database_name}'") 7 | # puts "MySQL drop_all_tables #{table_names.join(',')}" 8 | conn.execute("SET FOREIGN_KEY_CHECKS = 0") 9 | table_names.each do |t| 10 | conn.execute("DROP TABLE IF EXISTS #{t}") # In MySQL, CASCADE does nothing here. 11 | end 12 | conn.execute("SET FOREIGN_KEY_CHECKS = 1") 13 | end 14 | 15 | def mysql_config 16 | test_database_yml(RUBY_PLATFORM == 'java' ? 'jdbcmysql' : 'mysql') 17 | end 18 | 19 | describe DBLeftovers::MysqlDatabaseInterface do 20 | 21 | if not mysql_config 22 | it "WARN: Skipping MySQL tests because no database found. Use spec/config/database.yml to configure one." 23 | else 24 | before do 25 | y = mysql_config 26 | @conn = test_db_connection(nil, y) 27 | @db = DBLeftovers::MysqlDatabaseInterface.new(@conn, y['database']) 28 | drop_all_mysql_tables(@conn, y['database']) 29 | fresh_tables(@conn, y['database']) 30 | end 31 | 32 | it_behaves_like "DatabaseInterface" 33 | 34 | end 35 | end 36 | 37 | -------------------------------------------------------------------------------- /spec/config/database.yml.sample: -------------------------------------------------------------------------------- 1 | # Set this up by saying: 2 | # postgres=# create user db_leftovers_test with password 'testdb'; 3 | # postgres=# create database db_leftovers_test owner db_leftovers_test; 4 | # postgres=# grant all privileges on database db_leftovers_test to db_leftovers_test; 5 | 6 | postgres: 7 | adapter: postgres 8 | host: localhost 9 | database: db_leftovers_test 10 | username: db_leftovers_test 11 | password: testdb 12 | encoding: utf8 13 | template: template0 # Required for UTF-8 encoding 14 | 15 | 16 | # Set this up by saying: 17 | # mysql> create database db_leftovers_test; 18 | # mysql> grant all privileges on db_leftovers_test.* to db_leftovers@localhost identified by 'testdb'; 19 | 20 | mysql: 21 | adapter: mysql2 22 | host: localhost 23 | database: db_leftovers_test 24 | username: db_leftovers 25 | password: testdb 26 | encoding: utf8 27 | 28 | # These are used instead of the above if you're running under JRuby: 29 | 30 | jdbcpostgres: 31 | adapter: jdbcpostgresql 32 | host: localhost 33 | database: db_leftovers_test 34 | username: db_leftovers_test 35 | password: testdb 36 | encoding: utf8 37 | template: template0 # Required for UTF-8 encoding 38 | 39 | jdbcmysql: 40 | adapter: jdbcmysql 41 | host: localhost 42 | database: db_leftovers_test 43 | username: db_leftovers 44 | password: testdb 45 | encoding: utf8 46 | 47 | 48 | -------------------------------------------------------------------------------- /spec/support/mock_database_interface.rb: -------------------------------------------------------------------------------- 1 | class DBLeftovers::MockDatabaseInterface < DBLeftovers::GenericDatabaseInterface 2 | 3 | def initialize 4 | @sqls = [] 5 | end 6 | 7 | def sqls 8 | @sqls 9 | end 10 | 11 | def execute_sql(sql) 12 | @sqls << normal_whitespace(sql) 13 | end 14 | 15 | alias :old_execute_add_index :execute_add_index 16 | def execute_add_index(idx) 17 | old_execute_add_index(idx) 18 | @indexes[idx.index_name] = idx 19 | end 20 | 21 | alias :old_execute_add_constraint :execute_add_constraint 22 | def execute_add_constraint(chk) 23 | old_execute_add_constraint(chk) 24 | @constraints[chk.constraint_name] = chk 25 | end 26 | 27 | def saw_sql(sql) 28 | # puts sqls.join("\n\n\n") 29 | # Don't fail if only the whitespace is different: 30 | sqls.include?(normal_whitespace(sql)) 31 | end 32 | 33 | def starts_with(indexes=[], foreign_keys=[], constraints=[]) 34 | # Convert symbols to strings: 35 | @indexes = Hash[indexes.map{|idx| [idx.index_name, idx]}] 36 | @foreign_keys = Hash[foreign_keys.map{|fk| [fk.constraint_name, fk]}] 37 | @constraints = Hash[constraints.map{|chk| [chk.constraint_name, chk]}] 38 | end 39 | 40 | def lookup_all_indexes 41 | @indexes 42 | end 43 | 44 | def lookup_all_foreign_keys 45 | @foreign_keys 46 | end 47 | 48 | def lookup_all_constraints 49 | @constraints 50 | end 51 | 52 | private 53 | 54 | def normal_whitespace(sql) 55 | sql.gsub(/\s/m, ' ').gsub(/ +/, ' ').strip 56 | end 57 | 58 | end 59 | 60 | -------------------------------------------------------------------------------- /lib/db_leftovers/generic_database_interface.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class GenericDatabaseInterface 4 | 5 | def lookup_all_indexes 6 | raise "Should be overriden by a database-specific interface" 7 | end 8 | 9 | def lookup_all_foreign_keys 10 | raise "Should be overriden by a database-specific interface" 11 | end 12 | 13 | def lookup_all_constraints 14 | raise "Should be overriden by a database-specific interface" 15 | end 16 | 17 | def execute_add_index(idx) 18 | unique = idx.unique? ? 'UNIQUE' : '' 19 | where = idx.where_clause.present? ? "WHERE #{idx.where_clause}" : '' 20 | using = idx.using_clause.present? ? "USING #{idx.using_clause}" : '' 21 | 22 | sql = <<-EOQ 23 | CREATE #{unique} INDEX #{idx.index_name} 24 | ON #{idx.table_name} 25 | #{using} 26 | (#{idx.index_function || idx.column_names.join(', ')}) 27 | #{where} 28 | EOQ 29 | execute_sql(sql) 30 | end 31 | 32 | def execute_drop_index(table_name, index_name) 33 | sql = <<-EOQ 34 | DROP INDEX #{index_name} 35 | EOQ 36 | execute_sql(sql) 37 | end 38 | 39 | def execute_add_foreign_key(fk) 40 | on_delete = "ON DELETE CASCADE" if fk.cascade 41 | on_delete = "ON DELETE SET NULL" if fk.set_null 42 | deferrable = "DEFERRABLE INITIALLY DEFERRED" if fk.deferrable_initially_deferred 43 | deferrable = "DEFERRABLE INITIALLY IMMEDIATE" if fk.deferrable_initially_immediate 44 | execute_sql %{ALTER TABLE #{fk.from_table} 45 | ADD CONSTRAINT #{fk.constraint_name} 46 | FOREIGN KEY (#{fk.from_column}) 47 | REFERENCES #{fk.to_table} (#{fk.to_column}) 48 | #{on_delete} 49 | #{deferrable}} 50 | end 51 | 52 | def execute_drop_foreign_key(constraint_name, from_table, from_column) 53 | execute_sql %{ALTER TABLE #{from_table} DROP CONSTRAINT #{constraint_name}} 54 | end 55 | 56 | def execute_add_constraint(chk) 57 | sql = <<-EOQ 58 | ALTER TABLE #{chk.on_table} ADD CONSTRAINT #{chk.constraint_name} CHECK (#{chk.check}) 59 | EOQ 60 | execute_sql sql 61 | end 62 | 63 | def execute_drop_constraint(constraint_name, on_table) 64 | execute_sql %{ALTER TABLE #{on_table} DROP CONSTRAINT #{constraint_name}} 65 | end 66 | 67 | def execute_sql(sql) 68 | @conn.execute(sql) 69 | end 70 | 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/db_leftovers/index.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | # Just a struct to hold all the info for one index: 4 | class Index 5 | attr_accessor :table_name, :column_names, :index_name, 6 | :where_clause, :using_clause, :unique, :index_function 7 | 8 | def initialize(table_name, column_names, opts={}) 9 | opts = { 10 | :where => nil, 11 | :function => nil, 12 | :unique => false, 13 | :using => nil 14 | }.merge(opts) 15 | opts.keys.each do |k| 16 | raise "Unknown option: #{k}" unless [:where, :function, :unique, :using, :name].include?(k) 17 | end 18 | if column_names.is_a?(String) and opts[:function].nil? 19 | opts[:function] = column_names 20 | column_names = [] 21 | end 22 | @table_name = table_name.to_s 23 | @column_names = [column_names].flatten.map{|x| x.to_s} 24 | @where_clause = opts[:where] 25 | @index_function = opts[:function] 26 | @using_clause = opts[:using] 27 | @unique = !!opts[:unique] 28 | @index_name = (opts[:name] || choose_name(@table_name, @column_names, @index_function)).to_s 29 | 30 | raise "Indexes need a table!" unless @table_name 31 | raise "Indexes need at least column or an expression!" unless (@column_names.any? or @index_function) 32 | raise "Can't have both columns and an expression!" if (@column_names.size > 0 and @index_function) 33 | end 34 | 35 | def unique? 36 | !!@unique 37 | end 38 | 39 | def equals(other) 40 | other.table_name == table_name and 41 | other.column_names == column_names and 42 | other.index_name == index_name and 43 | other.where_clause == where_clause and 44 | other.index_function == index_function and 45 | other.using_clause == using_clause and 46 | other.unique == unique 47 | end 48 | 49 | def to_s 50 | "<#{@index_name}: #{@table_name}.[#{column_names.join(",")}] unique=#{@unique}, where=#{@where_clause}, function=#{@index_function}, using=#{@using_clause}>" 51 | end 52 | 53 | private 54 | 55 | def choose_name(table_name, column_names, index_function) 56 | topic = if column_names.any? 57 | column_names.join("_and_") 58 | else 59 | index_function 60 | end 61 | ret = "index_#{table_name}_on_#{topic}" 62 | ret = ret.gsub(/[^a-zA-Z0-9]/, '_'). 63 | gsub(/__+/, '_'). 64 | gsub(/^_/, ''). 65 | gsub(/_$/, '') 66 | # Max length in Postgres is 63; in MySQL 64: 67 | ret[0,63] 68 | end 69 | 70 | end 71 | 72 | end 73 | -------------------------------------------------------------------------------- /lib/db_leftovers/foreign_key.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class ForeignKey 4 | attr_accessor :constraint_name, :from_table, :from_column, :to_table, :to_column, :set_null, :cascade, :deferrable_initially_immediate, :deferrable_initially_deferred 5 | 6 | def initialize(from_table, from_column, to_table, to_column, opts={}) 7 | opts = { 8 | :deferrable => nil, 9 | :on_delete => nil, 10 | :name => name_constraint(from_table, from_column) 11 | }.merge(opts) 12 | opts.keys.each do |k| 13 | raise "`:set_null => true` should now be `:on_delete => :set_null`" if k.to_s == 'set_null' 14 | raise "`:cascade => true` should now be `:on_delete => :cascade`" if k.to_s == 'cascade' 15 | raise "Unknown option: #{k}" unless [:on_delete, :name, :deferrable].include?(k) 16 | end 17 | raise "Unknown on_delete option: #{opts[:on_delete]}" unless [nil, :set_null, :cascade].include?(opts[:on_delete]) 18 | raise "Unknown deferrable option: #{opts[:deferrable]}" unless [nil, :immediate, :deferred].include?(opts[:deferrable]) 19 | @constraint_name = opts[:name].to_s 20 | @from_table = from_table.to_s 21 | @from_column = from_column.to_s 22 | @to_table = to_table.to_s 23 | @to_column = to_column.to_s 24 | 25 | @set_null = opts[:on_delete] == :set_null 26 | @cascade = opts[:on_delete] == :cascade 27 | @deferrable_initially_immediate = opts[:deferrable] == :immediate 28 | @deferrable_initially_deferred = opts[:deferrable] == :deferred 29 | 30 | raise "ON DELETE can't be both set_null and cascade" if @set_null and @cascade 31 | raise "DEFERRABLE can't be both immediate and deferred" if @deferrable_initially_immediate and @deferrable_initially_deferred 32 | end 33 | 34 | def equals(other) 35 | other.constraint_name == constraint_name and 36 | other.from_table == from_table and 37 | other.from_column == from_column and 38 | other.to_table == to_table and 39 | other.to_column == to_column and 40 | other.set_null == set_null and 41 | other.cascade == cascade and 42 | other.deferrable_initially_immediate == deferrable_initially_immediate and 43 | other.deferrable_initially_deferred == deferrable_initially_deferred 44 | end 45 | 46 | def to_s 47 | [ 48 | "<#{@constraint_name}: from #{@from_table}.#{@from_column} to #{@to_table}.#{@to_column}", 49 | if @set_null; "ON DELETE SET NULL" 50 | elsif @cascade; "ON DELETE CASCADE" 51 | else; nil 52 | end, 53 | if @deferrable_initially_immediate; "DEFERRABLE INITIALLY IMMEDIATE" 54 | elsif @deferrable_initially_deferred; "DEFERRABLE INITIALLY DEFERRED" 55 | else; nil 56 | end, 57 | ">" 58 | ].compact.join(" ") 59 | end 60 | 61 | def name_constraint(from_table, from_column) 62 | "fk_#{from_table}_#{from_column}" 63 | end 64 | 65 | end 66 | 67 | end 68 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | actionmailer (3.1.3) 5 | actionpack (= 3.1.3) 6 | mail (~> 2.3.0) 7 | actionpack (3.1.3) 8 | activemodel (= 3.1.3) 9 | activesupport (= 3.1.3) 10 | builder (~> 3.0.0) 11 | erubis (~> 2.7.0) 12 | i18n (~> 0.6) 13 | rack (~> 1.3.5) 14 | rack-cache (~> 1.1) 15 | rack-mount (~> 0.8.2) 16 | rack-test (~> 0.6.1) 17 | sprockets (~> 2.0.3) 18 | activemodel (3.1.3) 19 | activesupport (= 3.1.3) 20 | builder (~> 3.0.0) 21 | i18n (~> 0.6) 22 | activerecord (3.1.3) 23 | activemodel (= 3.1.3) 24 | activesupport (= 3.1.3) 25 | arel (~> 2.2.1) 26 | tzinfo (~> 0.3.29) 27 | activerecord-jdbc-adapter (1.2.2) 28 | activerecord-jdbcmysql-adapter (1.2.2) 29 | activerecord-jdbc-adapter (~> 1.2.2) 30 | jdbc-mysql (~> 5.1.0) 31 | activerecord-jdbcpostgresql-adapter (1.2.2) 32 | activerecord-jdbc-adapter (~> 1.2.2) 33 | jdbc-postgres (>= 9.0, < 9.2) 34 | activerecord-mysql2-adapter (0.0.3) 35 | mysql2 36 | activerecord-postgresql-adapter (0.0.1) 37 | pg 38 | activeresource (3.1.3) 39 | activemodel (= 3.1.3) 40 | activesupport (= 3.1.3) 41 | activesupport (3.1.3) 42 | multi_json (~> 1.0) 43 | arel (2.2.1) 44 | builder (3.0.0) 45 | diff-lcs (1.1.3) 46 | erubis (2.7.0) 47 | hike (1.2.1) 48 | i18n (0.6.0) 49 | jdbc-mysql (5.1.13) 50 | jdbc-postgres (9.1.901) 51 | json (1.6.5) 52 | json (1.6.5-java) 53 | mail (2.3.0) 54 | i18n (>= 0.4.0) 55 | mime-types (~> 1.16) 56 | treetop (~> 1.4.8) 57 | mime-types (1.17.2) 58 | multi_json (1.0.4) 59 | mysql2 (0.3.11) 60 | pg (0.14.1) 61 | polyglot (0.3.3) 62 | rack (1.3.6) 63 | rack-cache (1.1) 64 | rack (>= 0.4) 65 | rack-mount (0.8.3) 66 | rack (>= 1.0.0) 67 | rack-ssl (1.3.2) 68 | rack 69 | rack-test (0.6.1) 70 | rack (>= 1.0) 71 | rails (3.1.3) 72 | actionmailer (= 3.1.3) 73 | actionpack (= 3.1.3) 74 | activerecord (= 3.1.3) 75 | activeresource (= 3.1.3) 76 | activesupport (= 3.1.3) 77 | bundler (~> 1.0) 78 | railties (= 3.1.3) 79 | railties (3.1.3) 80 | actionpack (= 3.1.3) 81 | activesupport (= 3.1.3) 82 | rack-ssl (~> 1.3.2) 83 | rake (>= 0.8.7) 84 | rdoc (~> 3.4) 85 | thor (~> 0.14.6) 86 | rake (0.9.2.2) 87 | rdoc (3.12) 88 | json (~> 1.4) 89 | rspec (2.4.0) 90 | rspec-core (~> 2.4.0) 91 | rspec-expectations (~> 2.4.0) 92 | rspec-mocks (~> 2.4.0) 93 | rspec-core (2.4.0) 94 | rspec-expectations (2.4.0) 95 | diff-lcs (~> 1.1.2) 96 | rspec-mocks (2.4.0) 97 | sprockets (2.0.3) 98 | hike (~> 1.2) 99 | rack (~> 1.0) 100 | tilt (~> 1.1, != 1.3.0) 101 | thor (0.14.6) 102 | tilt (1.3.3) 103 | treetop (1.4.10) 104 | polyglot 105 | polyglot (>= 0.3.1) 106 | tzinfo (0.3.31) 107 | 108 | PLATFORMS 109 | java 110 | ruby 111 | 112 | DEPENDENCIES 113 | activerecord-jdbcmysql-adapter 114 | activerecord-jdbcpostgresql-adapter 115 | activerecord-mysql2-adapter 116 | activerecord-postgresql-adapter 117 | bundler 118 | rails (>= 3.0.0) 119 | rspec (~> 2.4.0) 120 | -------------------------------------------------------------------------------- /lib/db_leftovers/mysql_database_interface.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class MysqlDatabaseInterface < GenericDatabaseInterface 4 | 5 | def initialize(conn=nil, database_name=nil) 6 | @conn = conn || ActiveRecord::Base.connection 7 | @db_name = database_name || ActiveRecord::Base.configurations[Rails.env]['database'] 8 | end 9 | 10 | def lookup_all_indexes 11 | ret = {} 12 | @conn.select_values("SHOW TABLES").each do |table_name| 13 | indexes = {} 14 | # Careful, MySQL automatically creates indexes whenever you define a foreign key. 15 | # Use our foreign key naming convention to ignore these: 16 | @conn.select_rows("SHOW INDEXES FROM #{table_name} WHERE key_name NOT LIKE 'fk_%'").each do |_, non_unique, key_name, seq_in_index, column_name, collation, cardinality, sub_part, packed, has_nulls, index_type, comment| 17 | unless key_name == 'PRIMARY' 18 | # Combine rows for multi-column indexes 19 | h = (indexes[key_name] ||= { unique: non_unique == 0, name: key_name, columns: {} }) 20 | h[:columns][seq_in_index.to_i] = column_name 21 | end 22 | end 23 | 24 | indexes.each do |index_name, h| 25 | ret[index_name] = Index.new( 26 | table_name, 27 | h[:columns].sort.map{|k, v| v}, 28 | unique: h[:unique], 29 | name: h[:name] 30 | ) 31 | end 32 | end 33 | 34 | return ret 35 | end 36 | 37 | def lookup_all_foreign_keys 38 | # TODO: Support multi-column foreign keys: 39 | ret = {} 40 | sql = <<-EOQ 41 | SELECT c.constraint_name, 42 | c.table_name, 43 | k.column_name, 44 | c.referenced_table_name, 45 | k.referenced_column_name, 46 | c.delete_rule 47 | FROM information_schema.referential_constraints c, 48 | information_schema.key_column_usage k 49 | WHERE c.constraint_schema = k.constraint_schema 50 | AND c.constraint_name = k.constraint_name 51 | AND c.constraint_schema IN (#{target_databases_quoted}) 52 | EOQ 53 | @conn.select_rows(sql).each do |constr_name, from_table, from_column, to_table, to_column, del_type| 54 | del_type = case del_type 55 | when 'RESTRICT'; nil 56 | when 'CASCADE'; :cascade 57 | when 'SET NULL'; :set_null 58 | else; raise "Unknown del type: #{del_type}" 59 | end 60 | ret[constr_name] = ForeignKey.new(from_table, from_column, to_table, to_column, :name => constr_name, :on_delete => del_type) 61 | end 62 | return ret 63 | end 64 | 65 | def lookup_all_constraints 66 | # TODO: Constrain it to the database for the current Rails project: 67 | # MySQL doesn't support CHECK constraints: 68 | return [] 69 | end 70 | 71 | def execute_drop_index(table_name, index_name) 72 | sql = <<-EOQ 73 | DROP INDEX #{index_name} ON #{table_name} 74 | EOQ 75 | execute_sql(sql) 76 | end 77 | 78 | def execute_drop_foreign_key(constraint_name, from_table, from_column) 79 | execute_sql %{ALTER TABLE #{from_table} DROP FOREIGN KEY #{constraint_name}} 80 | end 81 | 82 | private 83 | 84 | def target_databases_quoted 85 | case @db_name 86 | when Array 87 | @db_name.map{|x| "'#{x}'"}.join(", ") 88 | else 89 | "'#{@db_name}'" 90 | end 91 | end 92 | 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /spec/postgres_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails' 2 | require 'active_record' 3 | require File.expand_path(File.dirname(__FILE__) + '/spec_helper') 4 | 5 | def drop_all_postgres_tables(conn, database_name) 6 | table_names = conn.select_values("SELECT table_name FROM information_schema.tables WHERE table_catalog = '#{database_name}' AND table_schema NOT IN ('pg_catalog', 'information_schema')") 7 | # puts "Postgres drop_all_tables #{table_names.join(',')}" 8 | if table_names.size > 0 9 | conn.execute("DROP TABLE #{table_names.join(",")} CASCADE") 10 | end 11 | end 12 | 13 | def postgres_config 14 | test_database_yml(RUBY_PLATFORM == 'java' ? 'jdbcpostgres' : 'postgres') 15 | end 16 | 17 | describe DBLeftovers::PostgresDatabaseInterface do 18 | 19 | if not postgres_config 20 | it "WARN: Skipping Postgres tests because no database found. Use spec/config/database.yml to configure one." 21 | else 22 | before do 23 | y = postgres_config 24 | @conn = test_db_connection(nil, y) 25 | @db = DBLeftovers::PostgresDatabaseInterface.new(@conn) 26 | drop_all_postgres_tables(@conn, y['database']) 27 | fresh_tables(@conn, y['database']) 28 | end 29 | 30 | it_behaves_like "DatabaseInterface" 31 | 32 | it "should create indexes with a WHERE clause" do 33 | DBLeftovers::Definition.define :db_interface => @db do 34 | index :books, :publisher_id, :where => 'publication_year IS NOT NULL' 35 | end 36 | @db.lookup_all_indexes.size.should == 1 37 | @db.lookup_all_indexes.keys.sort.should == ['index_books_on_publisher_id'] 38 | end 39 | 40 | it "should not create indexes with a WHERE clause when they already exist" do 41 | starts_with(@db, [ 42 | DBLeftovers::Index.new(:books, :shelf_id), 43 | DBLeftovers::Index.new(:books, :publisher_id, :where => 'publication_year IS NOT NULL') 44 | ]) 45 | DBLeftovers::Definition.define :db_interface => @db do 46 | index :books, :shelf_id 47 | index :books, :publisher_id, :where => 'publication_year IS NOT NULL' 48 | end 49 | @db.lookup_all_indexes.size.should == 2 50 | end 51 | 52 | 53 | 54 | it "should redefine indexes when they have a new WHERE clause" do 55 | starts_with(@db, [ 56 | DBLeftovers::Index.new(:books, :shelf_id), 57 | DBLeftovers::Index.new(:books, :publisher_id, :where => 'publication_year IS NOT NULL'), 58 | DBLeftovers::Index.new(:books, :isbn) 59 | ]) 60 | DBLeftovers::Definition.define :db_interface => @db do 61 | index :books, :shelf_id, :where => 'name IS NOT NULL' 62 | index :books, :publisher_id, :where => 'publication_year > 1900' 63 | index :books, :isbn 64 | end 65 | @db.lookup_all_indexes.size.should == 3 66 | @db.lookup_all_indexes['index_books_on_shelf_id'].where_clause.should == 'name IS NOT NULL' 67 | @db.lookup_all_indexes['index_books_on_publisher_id'].where_clause.should == 'publication_year > 1900' 68 | end 69 | 70 | 71 | 72 | it "should create CHECK constraints on an empty database" do 73 | starts_with(@db, [], [], []) 74 | DBLeftovers::Definition.define :db_interface => @db do 75 | check :books, :books_have_positive_pages, 'pages_count > 0' 76 | end 77 | @db.lookup_all_constraints.size.should == 1 78 | @db.lookup_all_constraints['books_have_positive_pages'].check.should == 'pages_count > 0' 79 | end 80 | 81 | 82 | 83 | it "should remove obsolete CHECK constraints" do 84 | starts_with(@db, [], [], [ 85 | DBLeftovers::Constraint.new(:books_have_positive_pages, :books, 'pages_count > 0') 86 | ]) 87 | DBLeftovers::Definition.define :db_interface => @db do 88 | end 89 | @db.lookup_all_constraints.size.should == 0 90 | end 91 | 92 | 93 | 94 | it "should drop and re-create changed CHECK constraints" do 95 | starts_with(@db, [], [], [ 96 | DBLeftovers::Constraint.new(:books_have_positive_pages, :books, 'pages_count > 0') 97 | ]) 98 | DBLeftovers::Definition.define :db_interface => @db do 99 | check :books, :books_have_positive_pages, 'pages_count > 12' 100 | end 101 | @db.lookup_all_constraints.size.should == 1 102 | @db.lookup_all_constraints['books_have_positive_pages'].check.should == 'pages_count > 12' 103 | end 104 | 105 | it "should allow functional indexes specified as a string" do 106 | DBLeftovers::Definition.define :db_interface => @db do 107 | index :authors, 'lower(name)' 108 | end 109 | @db.lookup_all_indexes.size.should == 1 110 | @db.lookup_all_indexes.keys.sort.should == ['index_authors_on_lower_name'] 111 | end 112 | 113 | it "should allow functional indexes specified with an option", focus: true do 114 | DBLeftovers::Definition.define :db_interface => @db do 115 | index :authors, [], function: 'lower(name)' 116 | end 117 | @db.lookup_all_indexes.size.should == 1 118 | @db.lookup_all_indexes.keys.sort.should == ['index_authors_on_lower_name'] 119 | end 120 | 121 | end 122 | end 123 | 124 | -------------------------------------------------------------------------------- /lib/db_leftovers/postgres_database_interface.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class PostgresDatabaseInterface < GenericDatabaseInterface 4 | 5 | def initialize(conn=nil) 6 | @conn = conn || ActiveRecord::Base.connection 7 | end 8 | 9 | def lookup_all_indexes 10 | ret = {} 11 | sql = <<-EOQ 12 | SELECT ix.indexrelid, 13 | ix.indrelid, 14 | t.relname AS table_name, 15 | i.relname AS index_name, 16 | ix.indisunique AS is_unique, 17 | array_to_string(ix.indkey, ',') AS column_numbers, 18 | am.amname AS index_type, 19 | pg_get_expr(ix.indpred, ix.indrelid) AS where_clause, 20 | pg_get_expr(ix.indexprs, ix.indrelid) AS index_function 21 | FROM pg_class t, 22 | pg_class i, 23 | pg_index ix, 24 | pg_namespace n, 25 | pg_am am 26 | WHERE t.oid = ix.indrelid 27 | AND n.oid = t.relnamespace 28 | AND i.oid = ix.indexrelid 29 | AND t.relkind = 'r' 30 | AND n.nspname NOT IN ('pg_catalog', 'pg_toast') 31 | AND pg_catalog.pg_table_is_visible(t.oid) 32 | AND NOT ix.indisprimary 33 | AND i.relam = am.oid 34 | GROUP BY t.relname, 35 | i.relname, 36 | ix.indisunique, 37 | ix.indexrelid, 38 | ix.indrelid, 39 | ix.indkey, 40 | am.amname, 41 | ix.indpred, 42 | ix.indexprs 43 | ORDER BY t.relname, i.relname 44 | EOQ 45 | @conn.select_rows(sql).each do |indexrelid, indrelid, table_name, index_name, is_unique, column_numbers, index_method, where_clause, index_function| 46 | where_clause = remove_outer_parens(where_clause) if where_clause 47 | index_method = nil if index_method == 'btree' 48 | ret[index_name] = Index.new( 49 | table_name, 50 | column_names_for_index(indrelid, column_numbers.split(",")), 51 | unique: is_unique == 't', 52 | where: where_clause, 53 | function: index_function, 54 | using: index_method, 55 | name: index_name 56 | ) 57 | end 58 | return ret 59 | end 60 | 61 | 62 | 63 | def lookup_all_foreign_keys 64 | # confdeltype: a=nil, c=cascade, n=null 65 | ret = {} 66 | # TODO: Support multi-column foreign keys: 67 | sql = <<-EOQ 68 | SELECT c.conname, 69 | t1.relname AS from_table, 70 | a1.attname AS from_column, 71 | t2.relname AS to_table, 72 | a2.attname AS to_column, 73 | c.confdeltype, 74 | c.condeferrable AS deferrable, 75 | c.condeferred AS deferred 76 | FROM pg_catalog.pg_constraint c, 77 | pg_catalog.pg_class t1, 78 | pg_catalog.pg_class t2, 79 | pg_catalog.pg_attribute a1, 80 | pg_catalog.pg_attribute a2, 81 | pg_catalog.pg_namespace n1, 82 | pg_catalog.pg_namespace n2 83 | WHERE c.conrelid = t1.oid 84 | AND c.confrelid = t2.oid 85 | AND c.contype = 'f' 86 | AND a1.attrelid = t1.oid 87 | AND a1.attnum = ANY(c.conkey) 88 | AND a2.attrelid = t2.oid 89 | AND a2.attnum = ANY(c.confkey) 90 | AND t1.relkind = 'r' 91 | AND t2.relkind = 'r' 92 | AND n1.oid = t1.relnamespace 93 | AND n2.oid = t2.relnamespace 94 | AND n1.nspname NOT IN ('pg_catalog', 'pg_toast') 95 | AND n2.nspname NOT IN ('pg_catalog', 'pg_toast') 96 | AND pg_catalog.pg_table_is_visible(t1.oid) 97 | AND pg_catalog.pg_table_is_visible(t2.oid) 98 | EOQ 99 | @conn.select_rows(sql).each do |constr_name, from_table, from_column, to_table, to_column, del_type, deferrable, deferred| 100 | del_type = case del_type 101 | when 'a'; nil 102 | when 'c'; :cascade 103 | when 'n'; :set_null 104 | else; raise "Unknown del type: #{del_type}" 105 | end 106 | deferrable = deferrable == 't' 107 | deferred = deferred == 't' 108 | defer_type = if deferrable and deferred; :deferred 109 | elsif deferrable; :immediate 110 | else; nil 111 | end 112 | ret[constr_name] = ForeignKey.new(from_table, from_column, to_table, to_column, :name => constr_name, :on_delete => del_type, :deferrable => defer_type) 113 | end 114 | return ret 115 | end 116 | 117 | def lookup_all_constraints 118 | ret = {} 119 | sql = <<-EOQ 120 | SELECT c.conname, 121 | t.relname, 122 | pg_get_expr(c.conbin, c.conrelid) 123 | FROM pg_catalog.pg_constraint c, 124 | pg_catalog.pg_class t, 125 | pg_catalog.pg_namespace n 126 | WHERE c.contype = 'c' 127 | AND c.conrelid = t.oid 128 | AND t.relkind = 'r' 129 | AND n.oid = t.relnamespace 130 | AND n.nspname NOT IN ('pg_catalog', 'pg_toast') 131 | AND pg_catalog.pg_table_is_visible(t.oid) 132 | EOQ 133 | @conn.select_rows(sql).each do |constr_name, on_table, check_expr| 134 | ret[constr_name] = Constraint.new(constr_name, on_table, remove_outer_parens(check_expr)) 135 | end 136 | return ret 137 | end 138 | 139 | private 140 | 141 | def column_names_for_index(table_id, column_numbers) 142 | return [] if column_numbers == ['0'] 143 | column_numbers.map do |c| 144 | sql = <<-EOQ 145 | SELECT attname 146 | FROM pg_attribute 147 | WHERE attrelid = #{table_id} 148 | AND attnum = #{c} 149 | EOQ 150 | @conn.select_value(sql) 151 | end 152 | end 153 | 154 | def remove_outer_parens(str) 155 | str ? str.gsub(/^\((.*)\)$/, '\1') : nil 156 | end 157 | 158 | end 159 | end 160 | -------------------------------------------------------------------------------- /spec/support/shared_db_tests.rb: -------------------------------------------------------------------------------- 1 | 2 | =begin 3 | def drop_all_tables(conn, database_name) 4 | table_names = conn.select_values("SELECT table_name FROM information_schema.tables WHERE table_catalog = '#{database_name}' AND table_schema NOT IN ('pg_catalog', 'information_schema')") 5 | table_names.each do |t| 6 | conn.execute("DROP TABLE IF EXISTS #{t} CASCADE") 7 | end 8 | end 9 | =end 10 | 11 | def fresh_tables(conn, database_name) 12 | conn.execute <<-EOQ 13 | CREATE TABLE publishers ( 14 | id integer PRIMARY KEY, 15 | name varchar(255) 16 | ) 17 | EOQ 18 | conn.execute <<-EOQ 19 | CREATE TABLE authors ( 20 | id integer PRIMARY KEY, 21 | name varchar(255) 22 | ) 23 | EOQ 24 | conn.execute <<-EOQ 25 | CREATE TABLE shelves ( 26 | id integer PRIMARY KEY, 27 | name varchar(255) 28 | ) 29 | EOQ 30 | conn.execute <<-EOQ 31 | CREATE TABLE books ( 32 | id integer PRIMARY KEY, 33 | name varchar(255), 34 | author_id integer, 35 | coauthor_id integer, 36 | publisher_id integer, 37 | isbn varchar(255), 38 | publication_year integer, 39 | shelf_id integer, 40 | pages_count integer 41 | ) 42 | EOQ 43 | end 44 | 45 | def starts_with(db, indexes=[], foreign_keys=[], constraints=[]) 46 | indexes.each { |idx| db.execute_add_index(idx) } 47 | foreign_keys.each { |fk| db.execute_add_foreign_key(fk) } 48 | constraints.each { |chk| db.execute_add_constraint(chk) } 49 | end 50 | 51 | shared_examples_for "DatabaseInterface" do 52 | 53 | it "should create indexes on a fresh database" do 54 | DBLeftovers::Definition.define :db_interface => @db do 55 | index :books, :shelf_id 56 | index :books, :isbn, :unique => true 57 | # Not supported by MYSQL: 58 | # index :books, :publisher_id, :where => 'publication_year IS NOT NULL' 59 | end 60 | @db.lookup_all_indexes.size.should == 2 61 | @db.lookup_all_indexes.keys.sort.should == ['index_books_on_isbn', 'index_books_on_shelf_id'] 62 | end 63 | 64 | it "should create foreign keys on a fresh database" do 65 | DBLeftovers::Definition.define :db_interface => @db do 66 | foreign_key :books, :shelf_id, :shelves 67 | foreign_key :books, :publisher_id, :publishers, :id, :on_delete => :set_null 68 | foreign_key :books, :author_id, :authors, :id, :on_delete => :cascade 69 | end 70 | @db.lookup_all_foreign_keys.size.should == 3 71 | @db.lookup_all_foreign_keys.keys.sort.should == ['fk_books_author_id', 'fk_books_publisher_id', 'fk_books_shelf_id'] 72 | end 73 | 74 | it "should not create indexes when they already exist" do 75 | starts_with(@db, [ 76 | DBLeftovers::Index.new(:books, :shelf_id), 77 | DBLeftovers::Index.new(:books, :publisher_id, :unique => true) 78 | ]) 79 | DBLeftovers::Definition.define :db_interface => @db do 80 | index :books, :shelf_id 81 | index :books, :publisher_id, :unique => true 82 | end 83 | @db.lookup_all_indexes.size.should == 2 84 | end 85 | 86 | 87 | 88 | it "should create indexes when they have been redefined" do 89 | starts_with(@db, [ 90 | DBLeftovers::Index.new(:books, :shelf_id), 91 | # DBLeftovers::Index.new(:books, :publisher_id, :where => 'published'), 92 | DBLeftovers::Index.new(:books, :isbn, :unique => true) 93 | ]) 94 | DBLeftovers::Definition.define :db_interface => @db do 95 | index :books, :shelf_id, :unique => true 96 | # index :books, :publisher_id 97 | index :books, :isbn 98 | end 99 | @db.lookup_all_indexes.size.should == 2 100 | @db.lookup_all_indexes['index_books_on_shelf_id'].unique.should == true 101 | @db.lookup_all_indexes['index_books_on_isbn'].unique.should == false 102 | end 103 | 104 | 105 | it "should drop indexes when they are removed from the definition" do 106 | starts_with(@db, [ 107 | DBLeftovers::Index.new(:books, :shelf_id), 108 | DBLeftovers::Index.new(:books, :isbn, :unique => true) 109 | ]) 110 | DBLeftovers::Definition.define :db_interface => @db do 111 | index :books, :shelf_id 112 | end 113 | @db.lookup_all_indexes.size.should == 1 114 | end 115 | 116 | 117 | 118 | it "should drop foreign keys when they are removed from the definition" do 119 | starts_with(@db, [], [ 120 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id'), 121 | DBLeftovers::ForeignKey.new('books', 'author_id', 'authors', 'id') 122 | ]) 123 | DBLeftovers::Definition.define :db_interface => @db do 124 | foreign_key :books, :shelf_id, :shelves 125 | end 126 | @db.lookup_all_foreign_keys.size.should == 1 127 | end 128 | 129 | 130 | it "should support creating multi-column indexes" do 131 | starts_with(@db) 132 | DBLeftovers::Definition.define :db_interface => @db do 133 | index :books, [:publication_year, :name] 134 | end 135 | @db.lookup_all_indexes.size.should == 1 136 | puts @db.lookup_all_indexes.keys 137 | @db.lookup_all_indexes.should have_key('index_books_on_publication_year_and_name') 138 | end 139 | 140 | 141 | 142 | it "should support dropping multi-column indexes" do 143 | starts_with(@db, [ 144 | DBLeftovers::Index.new(:books, [:publication_year, :name]) 145 | ]) 146 | DBLeftovers::Definition.define :db_interface => @db do 147 | end 148 | @db.lookup_all_indexes.size.should == 0 149 | end 150 | 151 | 152 | it "should create foreign keys with a custom name" do 153 | DBLeftovers::Definition.define :db_interface => @db do 154 | foreign_key :books, :shelf_id, :shelves, :name => "fk_where_it_is" 155 | foreign_key :books, :publisher_id, :publishers, :id, :on_delete => :set_null, :name => "fk_who_published_it" 156 | foreign_key :books, :author_id, :authors, :id, :on_delete => :cascade, :name => "fk_who_wrote_it" 157 | end 158 | @db.lookup_all_foreign_keys.size.should == 3 159 | @db.lookup_all_foreign_keys.keys.sort.should == ['fk_where_it_is', 'fk_who_published_it', 'fk_who_wrote_it'] 160 | end 161 | 162 | 163 | it "should drop foreign keys when they are removed from the definition" do 164 | starts_with(@db, [], [ 165 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id', :name => "fk_where_it_is"), 166 | DBLeftovers::ForeignKey.new('books', 'author_id', 'authors', 'id', :name => "fk_who_wrote_it") 167 | ]) 168 | DBLeftovers::Definition.define :db_interface => @db do 169 | foreign_key :books, :shelf_id, :shelves, :name => "fk_where_it_is" 170 | end 171 | @db.lookup_all_foreign_keys.size.should == 1 172 | end 173 | 174 | 175 | end 176 | -------------------------------------------------------------------------------- /lib/db_leftovers/dsl.rb: -------------------------------------------------------------------------------- 1 | module DBLeftovers 2 | 3 | class DSL 4 | 5 | STATUS_EXISTS = 'exists' 6 | STATUS_CHANGED = 'changed' 7 | STATUS_NEW = 'new' 8 | 9 | def initialize(opts={}) 10 | @verbose = !!opts[:verbose] 11 | @db = opts[:db_interface] || get_database_interface 12 | 13 | @ignored_tables = Set.new(['delayed_jobs', 'schema_migrations'].map{|x| [x.to_s, x.to_sym]}.flatten) 14 | 15 | @indexes_by_table = {} # Set from the DSL 16 | @old_indexes = @db.lookup_all_indexes 17 | @new_indexes = {} 18 | 19 | @foreign_keys_by_table = {} # Set from the DSL 20 | @old_foreign_keys = @db.lookup_all_foreign_keys 21 | @new_foreign_keys = {} 22 | 23 | @constraints_by_table = {} # Set from the DSL 24 | @old_constraints = @db.lookup_all_constraints 25 | @new_constraints = {} 26 | end 27 | 28 | def define(&block) 29 | instance_eval(&block) 30 | end 31 | 32 | def ignore(*table_names) 33 | table_names = [table_names] unless table_names.is_a? Array 34 | table_names = table_names.map{|x| [x.to_s, x.to_sym]}.flatten 35 | @ignored_tables = Set.new(table_names) 36 | end 37 | 38 | def table(table_name, &block) 39 | table_dsl = TableDSL.new(self, table_name) 40 | table_dsl.define(&block) 41 | end 42 | 43 | def index(table_name, column_names, opts={}) 44 | add_index(Index.new(table_name, column_names, opts)) 45 | end 46 | 47 | # foreign_key(from_table, [from_column], to_table, [to_column], [opts]): 48 | # foreign_key(:books, :publishers) -> foreign_key(:books, nil, :publishers, nil) 49 | # foreign_key(:books, :co_author_id, :authors) -> foreign_key(:books, :co_author_id, :authors, nil) 50 | # foreign_key(:books, :publishers, opts) -> foreign_key(:books, nil, :publishers, nil, opts) 51 | # foreign_key(:books, :co_author_id, :authors, opts) -> foreign_key(:books, :co_author_id, :authors, nil, opts) 52 | def foreign_key(from_table, from_column=nil, to_table=nil, to_column=nil, opts={}) 53 | # First get the options hash into the right place: 54 | if to_column.class == Hash 55 | opts = to_column 56 | to_column = nil 57 | elsif to_table.class == Hash 58 | opts = to_table 59 | to_table = to_column = nil 60 | end 61 | 62 | # Sort out implicit arguments: 63 | if from_column and not to_table and not to_column 64 | to_table = from_column 65 | from_column = "#{to_table.to_s.singularize}_id" 66 | to_column = :id 67 | elsif from_column and to_table and not to_column 68 | to_column = :id 69 | end 70 | 71 | add_foreign_key(ForeignKey.new(from_table, from_column, to_table, to_column, opts)) 72 | end 73 | 74 | def check(table_name, constraint_name, check_expression) 75 | add_constraint(Constraint.new(constraint_name, table_name, check_expression)) 76 | end 77 | 78 | def record_indexes 79 | # First create any new indexes: 80 | @indexes_by_table.each do |table_name, indexes| 81 | indexes.each do |idx| 82 | next if ignore_index?(idx) 83 | # puts "#{idx.table_name}.[#{idx.column_names.join(',')}]" 84 | case index_status(idx) 85 | when STATUS_EXISTS 86 | puts "Index already exists: #{idx.index_name} on #{idx.table_name}" if @verbose 87 | when STATUS_CHANGED 88 | @db.execute_drop_index(idx.table_name, idx.index_name) 89 | @db.execute_add_index(idx) 90 | log_new_index(idx, true) 91 | when STATUS_NEW 92 | @db.execute_add_index(idx) 93 | log_new_index(idx, false) 94 | end 95 | @new_indexes[idx.index_name] = table_name 96 | end 97 | end 98 | 99 | # Now drop any old indexes that are no longer in the definition file: 100 | @old_indexes.each do |index_name, idx| 101 | next if ignore_index?(idx) 102 | if not @new_indexes[index_name] 103 | # puts "#{index_name} #{table_name}" 104 | @db.execute_drop_index(idx.table_name, index_name) 105 | puts "Dropped index: #{index_name} on #{idx.table_name}" 106 | end 107 | end 108 | end 109 | 110 | def record_foreign_keys 111 | # First create any new foreign keys: 112 | @foreign_keys_by_table.each do |table_name, fks| 113 | fks.each do |fk| 114 | next if ignore_foreign_key?(fk) 115 | case foreign_key_status(fk) 116 | when STATUS_EXISTS 117 | puts "Foreign Key already exists: #{fk.constraint_name} on #{fk.from_table}" if @verbose 118 | when STATUS_CHANGED 119 | @db.execute_drop_foreign_key(fk.constraint_name, fk.from_table, fk.from_column) 120 | @db.execute_add_foreign_key(fk) 121 | puts "Dropped & re-created foreign key: #{fk.constraint_name} on #{fk.from_table}" 122 | when STATUS_NEW 123 | @db.execute_add_foreign_key(fk) 124 | puts "Created foreign key: #{fk.constraint_name} on #{fk.from_table}" 125 | end 126 | @new_foreign_keys[fk.constraint_name] = fk 127 | end 128 | end 129 | 130 | # Now drop any old foreign keys that are no longer in the definition file: 131 | @old_foreign_keys.each do |constraint_name, fk| 132 | next if ignore_foreign_key?(fk) 133 | if not @new_foreign_keys[constraint_name] 134 | @db.execute_drop_foreign_key(constraint_name, fk.from_table, fk.from_column) 135 | puts "Dropped foreign key: #{constraint_name} on #{fk.from_table}" 136 | end 137 | end 138 | end 139 | 140 | def record_constraints 141 | # First create any new constraints: 142 | @constraints_by_table.each do |table_name, chks| 143 | chks.each do |chk| 144 | next if ignore_constraint?(chk) 145 | case constraint_status(chk) 146 | when STATUS_EXISTS 147 | puts "Constraint already exists: #{chk.constraint_name} on #{chk.on_table}" if @verbose 148 | when STATUS_CHANGED 149 | @db.execute_drop_constraint(chk.constraint_name, chk.on_table) 150 | @db.execute_add_constraint(chk) 151 | log_new_constraint(chk, true) 152 | when STATUS_NEW 153 | @db.execute_add_constraint(chk) 154 | log_new_constraint(chk, false) 155 | end 156 | @new_constraints[chk.constraint_name] = chk 157 | end 158 | end 159 | 160 | # Now drop any old constraints that are no longer in the definition file: 161 | @old_constraints.each do |constraint_name, chk| 162 | next if ignore_constraint?(chk) 163 | if not @new_constraints[constraint_name] 164 | @db.execute_drop_constraint(constraint_name, chk.on_table) 165 | puts "Dropped CHECK constraint: #{constraint_name} on #{chk.on_table}" 166 | end 167 | end 168 | end 169 | 170 | private 171 | 172 | def log_new_index(idx, altered=false) 173 | did_what = altered ? "Dropped & re-created" : "Created" 174 | 175 | msg = "#{did_what} index: #{idx.index_name} on #{idx.table_name}" 176 | if idx.index_function 177 | # NB: This is O(n*m) where n is your indexes and m is your indexes with WHERE clauses. 178 | # But it's hard to believe it matters: 179 | new_idx = @db.lookup_all_indexes[idx.index_name] 180 | msg = "#{msg}: #{new_idx.index_function}" 181 | end 182 | 183 | if idx.where_clause 184 | new_idx ||= @db.lookup_all_indexes[idx.index_name] 185 | msg = "#{msg} WHERE #{new_idx.where_clause}" 186 | end 187 | 188 | puts msg 189 | end 190 | 191 | def log_new_constraint(chk, altered=false) 192 | # NB: This is O(n^2) where n is your check constraints. 193 | # But it's hard to believe it matters: 194 | new_chk = @db.lookup_all_constraints[chk.constraint_name] 195 | puts "#{altered ? "Dropped & re-created" : "Created"} CHECK constraint: #{chk.constraint_name} on #{chk.on_table} as #{new_chk.check}" 196 | end 197 | 198 | def add_index(idx) 199 | t = (@indexes_by_table[idx.table_name] ||= []) 200 | t << idx 201 | end 202 | 203 | def add_foreign_key(fk) 204 | t = (@foreign_keys_by_table[fk.from_table] ||= []) 205 | t << fk 206 | end 207 | 208 | def add_constraint(chk) 209 | t = (@constraints_by_table[chk.on_table] ||= []) 210 | t << chk 211 | end 212 | 213 | def index_status(idx) 214 | old = @old_indexes[idx.index_name] 215 | if old 216 | return old.equals(idx) ? STATUS_EXISTS : STATUS_CHANGED 217 | else 218 | return STATUS_NEW 219 | end 220 | end 221 | 222 | def foreign_key_status(fk) 223 | old = @old_foreign_keys[fk.constraint_name] 224 | if old 225 | return old.equals(fk) ? STATUS_EXISTS : STATUS_CHANGED 226 | else 227 | return STATUS_NEW 228 | end 229 | end 230 | 231 | def constraint_status(chk) 232 | old = @old_constraints[chk.constraint_name] 233 | if old 234 | return old.equals(chk) ? STATUS_EXISTS : STATUS_CHANGED 235 | else 236 | return STATUS_NEW 237 | end 238 | end 239 | 240 | def get_database_interface 241 | db = ActiveRecord::Base.configurations[Rails.env]['adapter'] 242 | case db 243 | when 'postgresql', 'jdbcpostgresql', 'postgis' 244 | DBLeftovers::PostgresDatabaseInterface.new 245 | when 'mysql2' 246 | DBLeftovers::MySQLInterface.new 247 | else 248 | raise "Unsupported database: #{db}" 249 | end 250 | end 251 | 252 | def ignore_index?(idx) 253 | @ignored_tables.include?(idx.table_name) 254 | end 255 | 256 | def ignore_foreign_key?(fk) 257 | @ignored_tables.include?(fk.from_table) 258 | end 259 | 260 | def ignore_constraint?(chk) 261 | @ignored_tables.include?(chk.on_table) 262 | end 263 | 264 | end 265 | 266 | end 267 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | db\_leftovers 2 | ============= 3 | 4 | Db\_leftovers lets you define indexes, foreign keys, and CHECK constraints for your Rails app 5 | in one place using an easy-to-read DSL, 6 | then run a rake task to bring your database up-to-date. 7 | Whenever you edit the DSL, you can re-run the rake task and db\_leftovers will alter your database accordingly. 8 | This is useful because of the following limitations in vanilla Rails (note that very recently Rails has started to add some of these, e.g. `add_foreign_key`): 9 | 10 | * There are no built-in migration methods to create foreign keys or CHECK constraints. 11 | * Even if created, foreign keys and CHECK constraints won't appear in your schema.rb. 12 | * The built-in `add_index` method doesn't support WHERE clauses on your indexes. 13 | * If you're using Heroku, `db:push` and `db:pull` won't transfer your foreign keys and CHECK constraints. 14 | * Creating indexes in your migrations makes it hard to manage them. 15 | 16 | That last point deserves some elaboration. First, using `add_index` in your migrations is bug-prone because without rare developer discipline, you wind up missing indexes in some environments. (My rule is "never change a migration after a `git push`," but I haven't seen this followed elsewhere, and there is nothing that automatically enforces it.) Also, since missing indexes don't cause errors, it's easy to be missing one and not notice until users start complaining about performance. 17 | 18 | Second, scattering `add_index` methods throughout migrations doesn't match the workflow of optimizing database queries. Hopefully you create appropriate indexes when you set up your tables originally, but in practice you often need to go back later and add/remove indexes according to your database usage patterns. Or you just forget the indexes, because you're thinking about modeling the data, not optimizing the queries. 19 | It's easier to vet and analyze database indexes if you can see them all in one place, 20 | and db\_leftovers lets you do that easily. 21 | And since you can rest assured that each environment conforms to the same definition, you don't need to second-guess yourself about indexes that are present in development but missing in production. 22 | Db\_leftovers gives you confidence that your database matches a definition that is easy to read and checked into version control. 23 | 24 | At present db\_leftovers supports PostgreSQL and MySQL, although since MySQL doesn't support index WHERE clauses or CHECK constraints, using that functionality will raise errors. (If you need to share the same definitions across Postgres and MySQL, you can run arbitrary Ruby code inside the DSL to avoid defining unsupported objects when run against MySQL.) 25 | 26 | Configuration File 27 | ------------------ 28 | 29 | db\_leftovers reads a file named `config/db_leftovers.rb` to find out which indexes and constraints you want in your database. This file is a DSL implemented in Ruby, sort of like `config/routes.rb`. It should look something like this: 30 | 31 | DBLeftovers::Definition.define do 32 | 33 | table :users do 34 | index :email, :unique => true 35 | check :registered_at_unless_guest, "role_name = 'guest' OR registered_at IS NOT NULL" 36 | end 37 | 38 | foreign_key :orders, :users 39 | 40 | # . . 41 | end 42 | 43 | Within the DSL file, the following methods are supported: 44 | 45 | ### index(table\_name, columns, [opts]) 46 | 47 | This ensures that you have an index on the given table and column(s). The `columns` parameter can be either a symbol or a list of strings/symbols. (If you pass a single string for the `columns` parameter, it will be treated as the expression for a functional index rather than a column name.) Opts is a hash with the following possible keys: 48 | 49 | * `:name` The name of the index. Defaults to `index_`*table\_name*`_on_`*column\_names*, like the `add_index` method from Rails migrations. 50 | 51 | * `:unique` Set this to `true` if you'd like a unique index. 52 | 53 | * `:where` Accepts SQL to include in the `WHERE` part of the `CREATE INDEX` command, in case you want to limit the index to a subset of the table's rows. 54 | 55 | * `:using` Lets you specify what kind of index to create. Default is `btree`, but if you're on Postgres you might also want `gist`, `gin`, or `hash`. 56 | 57 | * `:function` Lets you specify an expression rather than a list of columns. If you give this option, you should pass an empty list of column names. Alternately, you can pass a string as the column name (rather than a symbol), and db\_leftovers will interpret it as a function. 58 | 59 | #### Examples 60 | 61 | index :books, :author_id 62 | index :books, [:publisher_id, :published_at] 63 | index :books, :isbn, :unique => true 64 | index :authors, [], function: 'lower(name)' 65 | index :authors, 'lower(name)' 66 | 67 | ### foreign\_key(from\_table, [from\_column], to\_table, [to\_column], [opts]) 68 | 69 | This ensures that you have a foreign key relating the given tables and columns. 70 | All parameters are strings/symbols except `opts`, which is a hash. 71 | If you omit the column names, db\_leftovers will infer them based on Rails conventions. (See examples below.) 72 | Opts is a hash with the following possible keys: 73 | 74 | * `:name` The name of the foreign key. Defaults to `fk_`*from\_table*`_`*from\_column*`. 75 | 76 | * `:on_delete` Sets the behavior when a row is deleted and other rows reference it. It may have any of these values: 77 | 78 | * `nil` Indicates that attempting to delete the referenced row should fail (the default). 79 | * `:set_null` Indicates that the foreign key should be set to null if the referenced row is deleted. 80 | * `:cascade` Indicates that the referencing row should be deleted if the referenced row is deleted. 81 | 82 | * `:deferrable` Marks the constraint as deferrable. Accepts these values: 83 | 84 | * `nil` Indicates the constraint is not deferrable (the default). 85 | * `:immediate` Indicates the constraint is usually enforced immediately but can be deferred. 86 | * `:deferred` Indicates the constraint is always enforced deferred. 87 | 88 | #### Examples 89 | 90 | foreign_key :books, :author_id, :authors, :id 91 | foreign_key :pages, :book_id, :books, :id, :on_delete => :cascade 92 | 93 | With implicit column names: 94 | 95 | foreign_key :books, :authors 96 | foreign_key :books, :authors, :on_delete => :cascade 97 | foreign_key :books, :co_author_id, :authors 98 | foreign_key :books, :co_author_id, :authors, :on_delete => :cascade, :deferred => :immediate 99 | 100 | ### check(on\_table, constraint\_name, expression) 101 | 102 | This ensures that you have a CHECK constraint on the given table with the given name and expression. 103 | All parameters are strings or symbols. 104 | 105 | #### Examples 106 | 107 | check :books, :books_have_positive_pages, 'page_count > 0' 108 | 109 | ### table(table\_name, &block) 110 | 111 | The `table` call is just a convenience so you can group all a table's indexes et cetera together and not keep repeating the table name. You use it like this: 112 | 113 | table :books do 114 | index :author_id 115 | foreign_key :publisher_id, :publishers 116 | check :books_have_positive_pages, 'page_count > 0' 117 | end 118 | 119 | You can repeat `table` calls for the same table several times if you like. This lets you put your indexes in one place and your foreign keys in another, for example. 120 | 121 | ### ignore(table\_name, [table\_name, ...]) 122 | 123 | Lets you specify one or more tables you'd like db\_leftovers to ignore completely. No objects will be added/removed to these tables. This is useful if you have tables that shouldn't be managed under db\_leftovers. 124 | 125 | If you don't call `ignore`, the list of ignored tables defaults to `schema_migrations` and `delayed_jobs`. If you do call `ignore`, you should probably include those in your list also. If you want db\_leftovers to manage those tables after all, just say `ignore []`. 126 | 127 | 128 | 129 | Running db\_leftovers 130 | --------------------- 131 | 132 | db\_leftovers comes with a Rake task named `db:leftovers`. So you can update your database to match your config file by saying this: 133 | 134 | rake db:leftovers 135 | 136 | db\_leftovers will notice whenever an index, foreign key, or CHECK constraint needs to be added, dropped, or changed. 137 | It will even catch cases where the name of the managed object is the same, but its attributes have changed. 138 | For instance, if you previously required books to have at least 1 page, but now you are introducing a "pamphlet" 139 | and want books to have at least 100 pages, you can change your config file to say: 140 | 141 | check :books, :books_have_positive_pages, 'page_count >= 100' 142 | 143 | and db\_leftovers will notice the changed expression. It will drop and re-add the constraint. 144 | 145 | One caveat, however: we pull the current expression from the database, and sometimes Postgres does things like 146 | add type conversions and extra parentheses. For instance, suppose you said `check :users, :email_length, 'LENGTH(email) > 2'`. 147 | The second time you run db\_leftovers, it will read the expression from Postgres and get `length((email)::text) > 2`, 148 | and so it will drop and re-create the constraint. 149 | It will drop and re-create it every time you run the rake task. 150 | To get around this, make sure your config file uses the same expression as printed by db\_leftovers in the rake output. 151 | This can also happen for index WHERE clauses and functional indexes, fixable by a similar workaround. 152 | MySQL doesn't have this problem because it doesn't support CHECK constraints or index WHERE clauses. 153 | 154 | To print messages even about indexes/foreign keys/constraints that haven't changed, you can say: 155 | 156 | rake db:leftovers VERBOSE=true 157 | 158 | or 159 | 160 | rake db:leftovers DB_LEFTOVERS_VERBOSE=true 161 | 162 | 163 | 164 | 165 | Capistrano Integration 166 | ---------------------- 167 | 168 | I recommend running `rake db:migrate` any time you deploy, and then running `rake db:leftovers` after that. Here is what you need in your `config/deploy.rb` to make that happen: 169 | 170 | set :rails_env, "production" 171 | 172 | namespace :db do 173 | desc "Set up constraints and indexes" 174 | task :leftovers do 175 | run("cd #{deploy_to}/current && bundle exec rake db:leftovers RAILS_ENV=#{rails_env}") 176 | end 177 | end 178 | 179 | after :deploy, 'deploy:migrate' 180 | after 'deploy:migrate', 'db:leftovers' 181 | 182 | You could also change this code to *not* run migrations after each deploy, if you like. But in that case I'd recommend not running db:leftovers until after the latest migrations (if any), since new entries in the DSL are likely to reference newly-created tables/columns. 183 | 184 | 185 | 186 | Known Issues 187 | ------------ 188 | 189 | * Multi-column foreign keys are not supported. This shouldn't be a problem for a Rails project, unless you are using a legacy database. If you need this functionality, let me know and I'll look into adding it. 190 | 191 | * It'd be nice to have another Rake task that will read your database and generate a new `db_leftovers.rb` file, to help people migrating existing projects that already have lots of tables. 192 | 193 | 194 | 195 | Tests 196 | ----- 197 | 198 | Db\_leftovers has three kinds of RSpec tests: tests that run against a mock database, a Postgres database, and a MySQL database. The tests look at `spec/config/database.yml` to see if you've created a test database for Postgres and/or MySQL, and only run those tests if an entry is found there. You can look at `spec/config/database.yml.sample` to get a sense of what is expected and for instructions on setting up your own test databases. If you're contributing a patch, please make sure you've run these tests! 199 | 200 | 201 | 202 | Contributing to db\_leftovers 203 | ----------------------------- 204 | 205 | * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet. 206 | * Check out the issue tracker to make sure someone hasn't already requested and/or contributed it. 207 | * Fork the project. 208 | * Start a feature/bugfix branch. 209 | * Commit and push until you are happy with your contribution. 210 | * Make be sure to add tests for it. This is important so I don't break it in a future version unintentionally. 211 | * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, that is fine, but please isolate that change to its own commit so I can cherry-pick around it. 212 | 213 | Commands for building/releasing/installing: 214 | 215 | * `rake build` 216 | * `rake install` 217 | * `rake release` 218 | 219 | Copyright 220 | --------- 221 | 222 | Copyright (c) 2012 Paul A. Jungwirth. 223 | See LICENSE.txt for further details. 224 | 225 | -------------------------------------------------------------------------------- /spec/db_leftovers_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rails' 2 | require File.expand_path(File.dirname(__FILE__) + '/spec_helper') 3 | 4 | describe DBLeftovers do 5 | 6 | before do 7 | @db = DBLeftovers::MockDatabaseInterface.new 8 | end 9 | 10 | it "should allow an empty definition" do 11 | @db.starts_with 12 | DBLeftovers::Definition.define :db_interface => @db do 13 | end 14 | @db.sqls.should be_empty 15 | end 16 | 17 | it "should allow an empty table definition" do 18 | @db.starts_with 19 | DBLeftovers::Definition.define :db_interface => @db do 20 | table :books do 21 | end 22 | end 23 | @db.sqls.should be_empty 24 | end 25 | 26 | it "should create indexes on an empty database" do 27 | @db.starts_with 28 | DBLeftovers::Definition.define :db_interface => @db do 29 | index :books, :shelf_id 30 | index :books, :publisher_id, :where => 'published' 31 | end 32 | @db.sqls.size.should == 2 33 | @db.should have_seen_sql <<-EOQ 34 | CREATE INDEX index_books_on_shelf_id 35 | ON books 36 | (shelf_id) 37 | EOQ 38 | @db.should have_seen_sql <<-EOQ 39 | CREATE INDEX index_books_on_publisher_id 40 | ON books 41 | (publisher_id) 42 | WHERE published 43 | EOQ 44 | end 45 | 46 | 47 | 48 | it "should create table-prefixed indexes on an empty database" do 49 | @db.starts_with 50 | DBLeftovers::Definition.define :db_interface => @db do 51 | table :books do 52 | index :shelf_id 53 | index :publisher_id, :where => 'published' 54 | end 55 | end 56 | @db.sqls.size.should == 2 57 | @db.should have_seen_sql <<-EOQ 58 | CREATE INDEX index_books_on_shelf_id 59 | ON books 60 | (shelf_id) 61 | EOQ 62 | @db.should have_seen_sql <<-EOQ 63 | CREATE INDEX index_books_on_publisher_id 64 | ON books 65 | (publisher_id) 66 | WHERE published 67 | EOQ 68 | end 69 | 70 | 71 | 72 | it "should create foreign keys on an empty database" do 73 | @db.starts_with 74 | DBLeftovers::Definition.define :db_interface => @db do 75 | foreign_key :books, :shelf_id, :shelves, :deferrable => :deferred 76 | foreign_key :books, :publisher_id, :publishers, :id, :on_delete => :set_null 77 | foreign_key :books, :author_id, :authors, :id, :on_delete => :cascade, :deferrable => :immediate 78 | end 79 | @db.sqls.should have(3).items 80 | @db.should have_seen_sql <<-EOQ 81 | ALTER TABLE books 82 | ADD CONSTRAINT fk_books_shelf_id 83 | FOREIGN KEY (shelf_id) 84 | REFERENCES shelves (id) 85 | DEFERRABLE INITIALLY DEFERRED 86 | EOQ 87 | @db.should have_seen_sql <<-EOQ 88 | ALTER TABLE books 89 | ADD CONSTRAINT fk_books_publisher_id 90 | FOREIGN KEY (publisher_id) 91 | REFERENCES publishers (id) 92 | ON DELETE SET NULL 93 | EOQ 94 | @db.should have_seen_sql <<-EOQ 95 | ALTER TABLE books 96 | ADD CONSTRAINT fk_books_author_id 97 | FOREIGN KEY (author_id) 98 | REFERENCES authors (id) 99 | ON DELETE CASCADE 100 | DEFERRABLE INITIALLY IMMEDIATE 101 | EOQ 102 | end 103 | 104 | 105 | 106 | it "should create table-prefixed foreign keys on an empty database" do 107 | @db.starts_with 108 | DBLeftovers::Definition.define :db_interface => @db do 109 | table :books do 110 | foreign_key :shelf_id, :shelves, :deferrable => :deferred 111 | foreign_key :publisher_id, :publishers, :id, :on_delete => :set_null 112 | foreign_key :author_id, :authors, :id, :on_delete => :cascade, :deferrable => :immediate 113 | end 114 | end 115 | @db.sqls.should have(3).items 116 | @db.should have_seen_sql <<-EOQ 117 | ALTER TABLE books 118 | ADD CONSTRAINT fk_books_shelf_id 119 | FOREIGN KEY (shelf_id) 120 | REFERENCES shelves (id) 121 | DEFERRABLE INITIALLY DEFERRED 122 | EOQ 123 | @db.should have_seen_sql <<-EOQ 124 | ALTER TABLE books 125 | ADD CONSTRAINT fk_books_publisher_id 126 | FOREIGN KEY (publisher_id) 127 | REFERENCES publishers (id) 128 | ON DELETE SET NULL 129 | EOQ 130 | @db.should have_seen_sql <<-EOQ 131 | ALTER TABLE books 132 | ADD CONSTRAINT fk_books_author_id 133 | FOREIGN KEY (author_id) 134 | REFERENCES authors (id) 135 | ON DELETE CASCADE 136 | DEFERRABLE INITIALLY IMMEDIATE 137 | EOQ 138 | end 139 | 140 | it "should create foreign keys with optional params inferred" do 141 | @db.starts_with 142 | DBLeftovers::Definition.define :db_interface => @db do 143 | foreign_key :books, :shelves 144 | foreign_key :books, :publishers, :on_delete => :set_null 145 | foreign_key :books, :publication_country_id, :countries 146 | foreign_key :books, :co_author_id, :authors, :on_delete => :cascade 147 | end 148 | @db.sqls.should have(4).items 149 | @db.should have_seen_sql <<-EOQ 150 | ALTER TABLE books ADD CONSTRAINT fk_books_shelf_id FOREIGN KEY (shelf_id) REFERENCES shelves (id) 151 | EOQ 152 | @db.should have_seen_sql <<-EOQ 153 | ALTER TABLE books ADD CONSTRAINT fk_books_publisher_id FOREIGN KEY (publisher_id) REFERENCES publishers (id) ON DELETE SET NULL 154 | EOQ 155 | @db.should have_seen_sql <<-EOQ 156 | ALTER TABLE books ADD CONSTRAINT fk_books_publication_country_id 157 | FOREIGN KEY (publication_country_id) REFERENCES countries (id) 158 | EOQ 159 | @db.should have_seen_sql <<-EOQ 160 | ALTER TABLE books ADD CONSTRAINT fk_books_co_author_id 161 | FOREIGN KEY (co_author_id) REFERENCES authors (id) ON DELETE CASCADE 162 | EOQ 163 | end 164 | 165 | it "should create foreign keys with optional params inferred and table block" do 166 | @db.starts_with 167 | DBLeftovers::Definition.define :db_interface => @db do 168 | table :books do 169 | foreign_key :shelves 170 | foreign_key :publishers 171 | foreign_key :publication_country_id, :countries 172 | foreign_key :co_author_id, :authors, :on_delete => :cascade 173 | end 174 | end 175 | @db.sqls.should have(4).items 176 | end 177 | 178 | it "should not create indexes when they already exist" do 179 | @db.starts_with([ 180 | DBLeftovers::Index.new(:books, :shelf_id), 181 | DBLeftovers::Index.new(:books, :publisher_id, :where => 'published') 182 | ]) 183 | DBLeftovers::Definition.define :db_interface => @db do 184 | index :books, :shelf_id 185 | index :books, :publisher_id, :where => 'published' 186 | end 187 | @db.sqls.should have(0).items 188 | end 189 | 190 | 191 | 192 | it "should create indexes when they have been redefined" do 193 | @db.starts_with([ 194 | DBLeftovers::Index.new(:books, :shelf_id), 195 | DBLeftovers::Index.new(:books, :publisher_id, :where => 'published'), 196 | DBLeftovers::Index.new(:books, :isbn, :unique => true) 197 | ]) 198 | DBLeftovers::Definition.define :db_interface => @db do 199 | index :books, :shelf_id, :where => 'isbn IS NOT NULL' 200 | index :books, :publisher_id 201 | index :books, :isbn 202 | end 203 | @db.sqls.should have(6).items 204 | @db.sqls[0].should =~ /DROP INDEX index_books_on_shelf_id/ 205 | @db.sqls[1].should =~ /CREATE\s+INDEX index_books_on_shelf_id/ 206 | end 207 | 208 | 209 | 210 | it "should not create table-prefixed indexes when they already exist" do 211 | @db.starts_with([ 212 | DBLeftovers::Index.new(:books, :shelf_id), 213 | DBLeftovers::Index.new(:books, :publisher_id, :where => 'published') 214 | ]) 215 | DBLeftovers::Definition.define :db_interface => @db do 216 | table :books do 217 | index :shelf_id 218 | index :publisher_id, :where => 'published' 219 | end 220 | end 221 | @db.sqls.should have(0).items 222 | end 223 | 224 | 225 | 226 | 227 | it "should not create foreign keys when they already exist" do 228 | @db.starts_with([], [ 229 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id') 230 | ]) 231 | DBLeftovers::Definition.define :db_interface => @db do 232 | foreign_key :books, :shelf_id, :shelves 233 | end 234 | @db.sqls.should have(0).items 235 | end 236 | 237 | 238 | 239 | it "should not create table-prefixed foreign keys when they already exist" do 240 | @db.starts_with([], [ 241 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id') 242 | ]) 243 | DBLeftovers::Definition.define :db_interface => @db do 244 | table :books do 245 | foreign_key :shelf_id, :shelves 246 | end 247 | end 248 | @db.sqls.should have(0).items 249 | end 250 | 251 | 252 | 253 | it "should drop indexes when they are removed from the definition" do 254 | @db.starts_with([ 255 | DBLeftovers::Index.new(:books, :shelf_id), 256 | DBLeftovers::Index.new(:books, :publisher_id, :where => 'published') 257 | ]) 258 | DBLeftovers::Definition.define :db_interface => @db do 259 | index :books, :shelf_id 260 | end 261 | @db.sqls.should have(1).item 262 | @db.should have_seen_sql <<-EOQ 263 | DROP INDEX index_books_on_publisher_id 264 | EOQ 265 | end 266 | 267 | 268 | 269 | it "should drop foreign keys when they are removed from the definition" do 270 | @db.starts_with([], [ 271 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id'), 272 | DBLeftovers::ForeignKey.new('books', 'author_id', 'authors', 'id') 273 | ]) 274 | DBLeftovers::Definition.define :db_interface => @db do 275 | foreign_key :books, :shelf_id, :shelves 276 | end 277 | @db.sqls.should have(1).item 278 | @db.should have_seen_sql <<-EOQ 279 | ALTER TABLE books DROP CONSTRAINT fk_books_author_id 280 | EOQ 281 | end 282 | 283 | 284 | 285 | it "should create foreign keys when they have been redefined" do 286 | @db.starts_with([], [ 287 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id'), 288 | DBLeftovers::ForeignKey.new('books', 'author_id', 'authors', 'id') 289 | ]) 290 | DBLeftovers::Definition.define :db_interface => @db do 291 | table :books do 292 | foreign_key :shelf_id, :shelves, :id, :on_delete => :cascade 293 | foreign_key :author_id, :authors, :id, :on_delete => :set_null 294 | end 295 | end 296 | @db.sqls.should have(4).items 297 | @db.sqls[0].should =~ /ALTER TABLE books DROP CONSTRAINT fk_books_shelf_id/ 298 | @db.sqls[1].should =~ /ALTER TABLE books ADD CONSTRAINT fk_books_shelf_id/ 299 | @db.sqls[2].should =~ /ALTER TABLE books DROP CONSTRAINT fk_books_author_id/ 300 | @db.sqls[3].should =~ /ALTER TABLE books ADD CONSTRAINT fk_books_author_id/ 301 | end 302 | 303 | 304 | 305 | it "should support creating multi-column indexes" do 306 | @db.starts_with 307 | DBLeftovers::Definition.define :db_interface => @db do 308 | index :books, [:year, :title] 309 | end 310 | @db.sqls.should have(1).item 311 | @db.should have_seen_sql <<-EOQ 312 | CREATE INDEX index_books_on_year_and_title 313 | ON books 314 | (year, title) 315 | EOQ 316 | end 317 | 318 | 319 | 320 | it "should support dropping multi-column indexes" do 321 | @db.starts_with([ 322 | DBLeftovers::Index.new(:books, [:year, :title]) 323 | ]) 324 | DBLeftovers::Definition.define :db_interface => @db do 325 | end 326 | @db.sqls.should have(1).item 327 | @db.should have_seen_sql <<-EOQ 328 | DROP INDEX index_books_on_year_and_title 329 | EOQ 330 | end 331 | 332 | 333 | 334 | it "should allow mixing indexes and foreign keys in the same table" do 335 | @db.starts_with 336 | DBLeftovers::Definition.define :db_interface => @db do 337 | table :books do 338 | index :author_id 339 | foreign_key :author_id, :authors, :id 340 | end 341 | end 342 | @db.sqls.should have(2).items 343 | @db.should have_seen_sql <<-EOQ 344 | CREATE INDEX index_books_on_author_id 345 | ON books 346 | (author_id) 347 | EOQ 348 | @db.should have_seen_sql <<-EOQ 349 | ALTER TABLE books 350 | ADD CONSTRAINT fk_books_author_id 351 | FOREIGN KEY (author_id) 352 | REFERENCES authors (id) 353 | EOQ 354 | end 355 | 356 | 357 | 358 | it "should allow separating indexes and foreign keys from the same table" do 359 | @db.starts_with 360 | DBLeftovers::Definition.define :db_interface => @db do 361 | table :books do 362 | index :author_id 363 | end 364 | table :books do 365 | foreign_key :author_id, :authors, :id 366 | end 367 | end 368 | @db.sqls.should have(2).items 369 | @db.should have_seen_sql <<-EOQ 370 | CREATE INDEX index_books_on_author_id 371 | ON books 372 | (author_id) 373 | EOQ 374 | @db.should have_seen_sql <<-EOQ 375 | ALTER TABLE books 376 | ADD CONSTRAINT fk_books_author_id 377 | FOREIGN KEY (author_id) 378 | REFERENCES authors (id) 379 | EOQ 380 | end 381 | 382 | it "should reject invalid foreign key options" do 383 | lambda { 384 | DBLeftovers::Definition.define :db_interface => @db do 385 | foreign_key :books, :author_id, :authors, :id, :icky => :boo_boo 386 | end 387 | }.should raise_error(RuntimeError, "Unknown option: icky") 388 | end 389 | 390 | it "should reject invalid foreign key on_delete values" do 391 | lambda { 392 | DBLeftovers::Definition.define :db_interface => @db do 393 | foreign_key :books, :author_id, :authors, :id, :on_delete => :panic 394 | end 395 | }.should raise_error(RuntimeError, "Unknown on_delete option: panic") 396 | end 397 | 398 | it "should give good a error message if you use the old :set_null option" do 399 | lambda { 400 | DBLeftovers::Definition.define :db_interface => @db do 401 | foreign_key :books, :author_id, :authors, :id, :set_null => true 402 | end 403 | }.should raise_error(RuntimeError, "`:set_null => true` should now be `:on_delete => :set_null`") 404 | end 405 | 406 | it "should give good a error message if you use the old :cascade option" do 407 | lambda { 408 | DBLeftovers::Definition.define :db_interface => @db do 409 | foreign_key :books, :author_id, :authors, :id, :cascade => true 410 | end 411 | }.should raise_error(RuntimeError, "`:cascade => true` should now be `:on_delete => :cascade`") 412 | end 413 | 414 | it "should create CHECK constraints on an empty database" do 415 | @db.starts_with([], [], []) 416 | DBLeftovers::Definition.define :db_interface => @db do 417 | check :books, :books_have_positive_pages, 'pages_count > 0' 418 | end 419 | @db.sqls.should have(1).item 420 | @db.should have_seen_sql <<-EOQ 421 | ALTER TABLE books ADD CONSTRAINT books_have_positive_pages CHECK (pages_count > 0) 422 | EOQ 423 | end 424 | 425 | it "should create CHECK constraints inside a table block" do 426 | @db.starts_with([], [], []) 427 | DBLeftovers::Definition.define :db_interface => @db do 428 | table :books do 429 | check :books_have_positive_pages, 'pages_count > 0' 430 | end 431 | end 432 | @db.sqls.should have(1).item 433 | @db.should have_seen_sql <<-EOQ 434 | ALTER TABLE books ADD CONSTRAINT books_have_positive_pages CHECK (pages_count > 0) 435 | EOQ 436 | end 437 | 438 | it "should remove obsolete CHECK constraints" do 439 | @db.starts_with([], [], [ 440 | DBLeftovers::Constraint.new(:books_have_positive_pages, :books, 'pages_count > 0') 441 | ]) 442 | DBLeftovers::Definition.define :db_interface => @db do 443 | end 444 | @db.sqls.should have(1).item 445 | @db.should have_seen_sql <<-EOQ 446 | ALTER TABLE books DROP CONSTRAINT books_have_positive_pages 447 | EOQ 448 | end 449 | 450 | it "should drop and re-create changed CHECK constraints" do 451 | @db.starts_with([], [], [ 452 | DBLeftovers::Constraint.new(:books_have_positive_pages, :books, 'pages_count > 0') 453 | ]) 454 | DBLeftovers::Definition.define :db_interface => @db do 455 | check :books, :books_have_positive_pages, 'pages_count > 12' 456 | end 457 | @db.sqls.should have(2).items 458 | @db.should have_seen_sql <<-EOQ 459 | ALTER TABLE books DROP CONSTRAINT books_have_positive_pages 460 | EOQ 461 | @db.should have_seen_sql <<-EOQ 462 | ALTER TABLE books ADD CONSTRAINT books_have_positive_pages CHECK (pages_count > 12) 463 | EOQ 464 | end 465 | 466 | it "should not try to change anything on an ignored table" do 467 | @db.starts_with([ 468 | DBLeftovers::Index.new(:books, :shelf_id), 469 | ], [ 470 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id'), 471 | DBLeftovers::ForeignKey.new('books', 'author_id', 'authors', 'id') 472 | ], [ 473 | DBLeftovers::Constraint.new(:books_have_positive_pages, :books, 'pages_count > 0') 474 | ]) 475 | DBLeftovers::Definition.define :db_interface => @db do 476 | ignore :books 477 | end 478 | @db.sqls.should have(0).items 479 | end 480 | 481 | it "should accept multiple ignored tables" do 482 | @db.starts_with([ 483 | DBLeftovers::Index.new(:books, :shelf_id), 484 | DBLeftovers::Index.new(:authors, :last_name), 485 | ], [ 486 | DBLeftovers::ForeignKey.new('books', 'shelf_id', 'shelves', 'id'), 487 | DBLeftovers::ForeignKey.new('books', 'author_id', 'authors', 'id') 488 | ], [ 489 | DBLeftovers::Constraint.new(:books_have_positive_pages, :books, 'pages_count > 0') 490 | ]) 491 | DBLeftovers::Definition.define :db_interface => @db do 492 | ignore :books, :authors 493 | end 494 | @db.sqls.should have(0).items 495 | end 496 | 497 | it "should ignore schema_migrations and delayed_jobs by default" do 498 | @db.starts_with([ 499 | DBLeftovers::Index.new(:schema_migrations, :foo), 500 | DBLeftovers::Index.new(:delayed_jobs, :bar), 501 | ], [], []) 502 | DBLeftovers::Definition.define :db_interface => @db do 503 | end 504 | @db.sqls.should have(0).items 505 | end 506 | 507 | it "should accept `USING` for indexes" do 508 | @db.starts_with([], [], []) 509 | DBLeftovers::Definition.define :db_interface => @db do 510 | index :libraries, :lonlat, using: 'gist' 511 | end 512 | @db.sqls.should have(1).item 513 | @db.should have_seen_sql <<-EOQ 514 | CREATE INDEX index_libraries_on_lonlat ON libraries USING gist (lonlat) 515 | EOQ 516 | end 517 | 518 | end 519 | --------------------------------------------------------------------------------