├── .rspec ├── .gitignore ├── lib ├── yaml_db │ ├── version.rb │ ├── rake_tasks.rb │ ├── csv_db.rb │ └── serialization_helper.rb ├── tasks │ └── yaml_db_tasks.rake └── yaml_db.rb ├── Gemfile ├── Rakefile ├── spec ├── yaml_db │ ├── utils_spec.rb │ ├── integration_spec.rb │ ├── load_spec.rb │ ├── serialization_helper_base_spec.rb │ ├── dump_spec.rb │ ├── serialization_helper_utils_spec.rb │ ├── rake_tasks_spec.rb │ ├── serialization_helper_load_spec.rb │ └── serialization_helper_dump_spec.rb ├── tasks │ └── yaml_db_tasks_spec.rb └── spec_helper.rb ├── CHANGELOG.md ├── yaml_db.gemspec ├── README.md └── .github └── workflows └── ci.yml /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --require spec_helper 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | /.bundle/ 3 | /pkg/ 4 | Gemfile.lock 5 | -------------------------------------------------------------------------------- /lib/yaml_db/version.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | VERSION = "0.7.0" 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'rails', ENV['RAILS_VERSION'] 6 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | require "rspec/core/rake_task" 3 | require "bundler/gem_tasks" 4 | 5 | RSpec::Core::RakeTask.new(:spec) 6 | 7 | task :default => :spec 8 | 9 | -------------------------------------------------------------------------------- /spec/yaml_db/utils_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | RSpec.describe Utils do 3 | 4 | it "turns an array with one record into a yaml chunk" do 5 | expect(Utils.chunk_records([ %w(a b) ])).to eq(< 'sqlite3', 6 | :database => ':memory:' 7 | ) 8 | 9 | # define a dummy User model 10 | class User < ActiveRecord::Base 11 | end 12 | 13 | # create the users table 14 | ActiveRecord::Schema.define do 15 | self.verbose = false 16 | 17 | create_table :users, :force => true do |t| 18 | t.string :username 19 | end 20 | end 21 | 22 | # add some users 23 | User.create([ 24 | {:username => 'alice'}, 25 | {:username => 'bob'} 26 | ]) 27 | 28 | 29 | RSpec.describe "with real ActiveRecord," do 30 | 31 | it "contains two users" do 32 | expect(User.count).to eq(2) 33 | end 34 | 35 | it "dumps the user records" do 36 | @io = StringIO.new 37 | YamlDb::Dump.dump_table_records(@io, 'users') 38 | @io.rewind 39 | expect(@io.read).to eq(< [ "db:schema:dump", "db:data:dump" ]) 4 | 5 | desc "Load schema and data from db/schema.rb and db/data.yml" 6 | task(:load => [ "db:schema:load", "db:data:load" ]) 7 | 8 | namespace :data do 9 | desc "Dump contents of database to db/data.extension (defaults to yaml)" 10 | task :dump => :environment do 11 | YamlDb::RakeTasks.data_dump_task 12 | end 13 | 14 | desc "Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml)" 15 | task :dump_dir => :environment do 16 | YamlDb::RakeTasks.data_dump_dir_task 17 | end 18 | 19 | desc "Load contents of db/data.extension (defaults to yaml) into database" 20 | task :load => :environment do 21 | YamlDb::RakeTasks.data_load_task 22 | end 23 | 24 | desc "Load contents of db/data_dir into database" 25 | task :load_dir => :environment do 26 | YamlDb::RakeTasks.data_load_dir_task 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/yaml_db/rake_tasks.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | module RakeTasks 3 | def self.data_dump_task 4 | SerializationHelper::Base.new(helper).dump(db_dump_data_file(helper.extension)) 5 | end 6 | 7 | def self.data_dump_dir_task 8 | dir = ENV['dir'] || default_dir_name 9 | SerializationHelper::Base.new(helper).dump_to_dir(dump_dir("/#{dir}")) 10 | end 11 | 12 | def self.data_load_task 13 | SerializationHelper::Base.new(helper).load(db_dump_data_file(helper.extension)) 14 | end 15 | 16 | def self.data_load_dir_task 17 | dir = ENV['dir'] || 'base' 18 | SerializationHelper::Base.new(helper).load_from_dir(dump_dir("/#{dir}")) 19 | end 20 | 21 | private 22 | 23 | def self.default_dir_name 24 | Time.now.strftime('%FT%H%M%S') 25 | end 26 | 27 | def self.db_dump_data_file(extension = 'yml') 28 | "#{dump_dir}/data.#{extension}" 29 | end 30 | 31 | def self.dump_dir(dir = '') 32 | "#{Rails.root}/db#{dir}" 33 | end 34 | 35 | def self.helper 36 | format_class = ENV['class'] || 'YamlDb::Helper' 37 | format_class.constantize 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /spec/yaml_db/load_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | RSpec.describe Load do 3 | 4 | before do 5 | allow(SerializationHelper::Utils).to receive(:quote_table).with('mytable').and_return('mytable') 6 | 7 | allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection').as_null_object) 8 | allow(ActiveRecord::Base.connection).to receive(:transaction).and_yield 9 | end 10 | 11 | before(:each) do 12 | @io = StringIO.new 13 | end 14 | 15 | it "calls load structure for each document in the file" do 16 | expect(YAML).to receive(:load_stream).with(@io).and_yield({ 'mytable' => { 17 | 'columns' => [ 'a', 'b' ], 18 | 'records' => [[1, 2], [3, 4]] 19 | } } ) 20 | expect(Load).to receive(:load_table).with('mytable', { 'columns' => [ 'a', 'b' ], 'records' => [[1, 2], [3, 4]] },true) 21 | Load.load(@io) 22 | end 23 | 24 | it "calls load structure when the document in the file contains no records" do 25 | expect(YAML).to receive(:load_stream).with(@io).and_yield({ 'mytable' => nil }) 26 | expect(Load).not_to receive(:load_table) 27 | Load.load(@io) 28 | end 29 | 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /yaml_db.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'yaml_db/version' 5 | 6 | Gem::Specification.new do |s| 7 | s.platform = Gem::Platform::RUBY 8 | s.name = "yaml_db" 9 | s.version = YamlDb::VERSION 10 | s.authors = ["Adam Wiggins", "Orion Henry"] 11 | s.summary = %q{yaml_db allows export/import of database into/from yaml files} 12 | s.description = "\nYamlDb is a database-independent format for dumping and restoring data. It complements the database-independent schema format found in db/schema.rb. The data is saved into db/data.yml.\nThis can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps. Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.\nAny database that has an ActiveRecord adapter should work.\n" 13 | s.homepage = "https://github.com/yamldb/yaml_db" 14 | s.license = "MIT" 15 | 16 | s.extra_rdoc_files = ["README.md"] 17 | s.files = Dir['README.md', 'lib/**/*'] 18 | s.require_paths = ["lib"] 19 | 20 | s.required_ruby_version = ">= 1.8.7" 21 | 22 | s.add_runtime_dependency "rails", ">= 3.0" 23 | s.add_runtime_dependency "rake", ">= 0.8.7" 24 | 25 | s.add_development_dependency "bundler", "~> 1.14" 26 | s.add_development_dependency "rspec", "~> 3.0" 27 | s.add_development_dependency "sqlite3", "~> 1.3.4" 28 | end 29 | -------------------------------------------------------------------------------- /lib/yaml_db.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'yaml' 3 | require 'active_record' 4 | require 'rails/railtie' 5 | require 'yaml_db/rake_tasks' 6 | require 'yaml_db/version' 7 | require 'yaml_db/serialization_helper' 8 | 9 | module YamlDb 10 | module Helper 11 | def self.loader 12 | Load 13 | end 14 | 15 | def self.dumper 16 | Dump 17 | end 18 | 19 | def self.extension 20 | "yml" 21 | end 22 | end 23 | 24 | 25 | module Utils 26 | def self.chunk_records(records) 27 | yaml = [ records ].to_yaml 28 | yaml.sub!(/---\s\n|---\n/, '') 29 | yaml.sub!('- - -', ' - -') 30 | yaml 31 | end 32 | 33 | end 34 | 35 | class Dump < SerializationHelper::Dump 36 | 37 | def self.dump_table_columns(io, table) 38 | io.write("\n") 39 | io.write({ table => { 'columns' => table_column_names(table) } }.to_yaml) 40 | end 41 | 42 | def self.dump_table_records(io, table) 43 | table_record_header(io) 44 | 45 | column_names = table_column_names(table) 46 | 47 | each_table_page(table) do |records| 48 | rows = SerializationHelper::Utils.unhash_records(records.to_a, column_names) 49 | io.write(Utils.chunk_records(rows)) 50 | end 51 | end 52 | 53 | def self.table_record_header(io) 54 | io.write(" records: \n") 55 | end 56 | 57 | end 58 | 59 | class Load < SerializationHelper::Load 60 | def self.load_documents(io, truncate = true) 61 | YAML.load_stream(io) do |ydoc| 62 | ydoc.keys.each do |table_name| 63 | next if ydoc[table_name].nil? 64 | load_table(table_name, ydoc[table_name], truncate) 65 | end 66 | end 67 | end 68 | end 69 | 70 | class Railtie < Rails::Railtie 71 | rake_tasks do 72 | load File.expand_path('../tasks/yaml_db_tasks.rake', 73 | __FILE__) 74 | end 75 | end 76 | 77 | end 78 | -------------------------------------------------------------------------------- /spec/tasks/yaml_db_tasks_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | 3 | RSpec.describe 'Rake tasks' do 4 | before do 5 | Rake::Application.new.rake_require('tasks/yaml_db_tasks') 6 | Rake::Task.define_task(:environment) 7 | end 8 | 9 | subject { Rake::Task[self.class.description] } 10 | 11 | describe 'db:dump' do 12 | it 'depends on db:schema:dump and db:data:dump' do 13 | expect(subject.prerequisites).to eq(['db:schema:dump', 'db:data:dump']) 14 | end 15 | end 16 | 17 | describe 'db:load' do 18 | it 'depends on db:schema:load and db:data:load' do 19 | expect(subject.prerequisites).to eq(['db:schema:load', 'db:data:load']) 20 | end 21 | end 22 | 23 | describe 'db:data:dump' do 24 | it 'loads the environment' do 25 | expect(subject.prerequisites).to eq(['environment']) 26 | end 27 | 28 | it 'invokes the correct task' do 29 | expect(YamlDb::RakeTasks).to receive(:data_dump_task).once.with(no_args) 30 | subject.invoke 31 | end 32 | end 33 | 34 | describe 'db:data:dump_dir' do 35 | it 'loads the environment' do 36 | expect(subject.prerequisites).to eq(['environment']) 37 | end 38 | 39 | it 'invokes the correct task' do 40 | expect(YamlDb::RakeTasks).to receive(:data_dump_dir_task).once.with(no_args) 41 | subject.invoke 42 | end 43 | end 44 | 45 | describe 'db:data:load' do 46 | it 'loads the environment' do 47 | expect(subject.prerequisites).to eq(['environment']) 48 | end 49 | 50 | it 'invokes the correct task' do 51 | expect(YamlDb::RakeTasks).to receive(:data_load_task).once.with(no_args) 52 | subject.invoke 53 | end 54 | end 55 | 56 | describe 'db:data:load_dir' do 57 | it 'loads the environment' do 58 | expect(subject.prerequisites).to eq(['environment']) 59 | end 60 | 61 | it 'invokes the correct task' do 62 | expect(YamlDb::RakeTasks).to receive(:data_load_dir_task).once.with(no_args) 63 | subject.invoke 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/yaml_db/serialization_helper_base_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | module SerializationHelper 3 | RSpec.describe Base do 4 | def prestub_active_record 5 | end 6 | 7 | before do 8 | @io = StringIO.new 9 | allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection')) 10 | allow(ActiveRecord::Base.connection).to receive(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ]) 11 | end 12 | 13 | def stub_helper! 14 | @helper = double("MyHelper") 15 | @dumper = double("MyDumper"); 16 | @loader = double("MyLoader"); 17 | allow(@helper).to receive(:dumper).and_return(@dumper) 18 | allow(@helper).to receive(:loader).and_return(@loader) 19 | allow(@helper).to receive(:extension).and_return("yml") 20 | allow(@dumper).to receive(:tables).and_return([ActiveRecord::Base.connection.tables[0]]) 21 | allow(@dumper).to receive(:before_table).and_return(nil) 22 | allow(@dumper).to receive(:after_table).and_return(nil) 23 | end 24 | 25 | context "for multi-file dumps" do 26 | before do 27 | expect(File).to receive(:open).once.with("dir_name/mytable.yml", "w").and_yield(@io) 28 | expect(Dir).to receive(:mkdir).once.with("dir_name") 29 | stub_helper! 30 | expect(@dumper).to receive(:dump_table).once.with(@io, "mytable") 31 | end 32 | 33 | it "creates the number of files that there are tables" do 34 | Base.new(@helper).dump_to_dir "dir_name" 35 | end 36 | 37 | end 38 | 39 | context "for multi-file loads" do 40 | before do 41 | stub_helper! 42 | expect(@loader).to receive(:load).once.with(@io, true) 43 | expect(File).to receive(:new).once.with("dir_name/mytable.yml", "r").and_return(@io) 44 | allow(Dir).to receive(:entries).and_return(["mytable.yml"]) 45 | end 46 | 47 | it "inserts into the number of tables that there are files" do 48 | Base.new(@helper).load_from_dir "dir_name" 49 | end 50 | end 51 | 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/yaml_db/csv_db.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | module CsvDb 3 | module Helper 4 | def self.loader 5 | Load 6 | end 7 | 8 | def self.dumper 9 | Dump 10 | end 11 | 12 | def self.extension 13 | "csv" 14 | end 15 | end 16 | 17 | class Load < SerializationHelper::Load 18 | def self.load_documents(io, truncate = true) 19 | tables = {} 20 | curr_table = nil 21 | io.each do |line| 22 | if /BEGIN_CSV_TABLE_DECLARATION(.+)END_CSV_TABLE_DECLARATION/ =~ line 23 | curr_table = $1 24 | tables[curr_table] = {} 25 | else 26 | if tables[curr_table]["columns"] 27 | tables[curr_table]["records"] << FasterCSV.parse(line)[0] 28 | else 29 | tables[curr_table]["columns"] = FasterCSV.parse(line)[0] 30 | tables[curr_table]["records"] = [] 31 | end 32 | end 33 | end 34 | 35 | tables.each_pair do |table_name, contents| 36 | load_table(table_name, contents, truncate) 37 | end 38 | end 39 | end 40 | 41 | class Dump < SerializationHelper::Dump 42 | 43 | def self.before_table(io,table) 44 | io.write "BEGIN_CSV_TABLE_DECLARATION#{table}END_CSV_TABLE_DECLARATION\n" 45 | end 46 | 47 | def self.dump(io) 48 | tables.each do |table| 49 | before_table(io, table) 50 | dump_table(io, table) 51 | after_table(io, table) 52 | end 53 | end 54 | 55 | def self.after_table(io,table) 56 | io.write "" 57 | end 58 | 59 | def self.dump_table_columns(io, table) 60 | io.write(table_column_names(table).to_csv) 61 | end 62 | 63 | def self.dump_table_records(io, table) 64 | 65 | column_names = table_column_names(table) 66 | 67 | each_table_page(table) do |records| 68 | rows = SerializationHelper::Utils.unhash_records(records, column_names) 69 | records.each do |record| 70 | io.write(record.to_csv) 71 | end 72 | end 73 | end 74 | 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/yaml_db/dump_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | RSpec.describe Dump do 3 | 4 | before do 5 | allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection').as_null_object) 6 | allow(ActiveRecord::Base.connection).to receive(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ]) 7 | allow(ActiveRecord::Base.connection).to receive(:columns).with('mytable').and_return([ double('a',:name => 'a', :type => :string), double('b', :name => 'b', :type => :string) ]) 8 | allow(ActiveRecord::Base.connection).to receive(:select_one).and_return({"count"=>"2"}) 9 | allow(ActiveRecord::Base.connection).to receive(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]) 10 | allow(Utils).to receive(:quote_table).with('mytable').and_return('mytable') 11 | end 12 | 13 | before(:each) do 14 | allow(File).to receive(:open).with('dump.yml', 'w').and_yield(StringIO.new) 15 | @io = StringIO.new 16 | end 17 | 18 | it "returns a formatted string" do 19 | Dump.table_record_header(@io) 20 | @io.rewind 21 | expect(@io.read).to eq(" records: \n") 22 | end 23 | 24 | it "returns a yaml string that contains a table header and column names" do 25 | allow(Dump).to receive(:table_column_names).with('mytable').and_return([ 'a', 'b' ]) 26 | Dump.dump_table_columns(@io, 'mytable') 27 | @io.rewind 28 | if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('1.9.3') 29 | expect(@io.read).to eq(< 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 }] 54 | ) 55 | Dump.dump_table_records(@io, 'mytable') 56 | @io.rewind 57 | expect(@io.read).to eq(< Dump contents of Rails database to db/data.yml 26 | rake db:data:load -> Load contents of db/data.yml into the database 27 | 28 | Further, there are tasks db:dump and db:load which do the entire database (the equivalent of running db:schema:dump followed by db:data:load). Also, there are other tasks recently added that allow the export of the database contents to/from multiple files (each one named after the table being dumped or loaded). 29 | 30 | rake db:data:dump_dir -> Dump contents of database to curr_dir_name/tablename.extension (defaults to yaml) 31 | rake db:data:load_dir -> Load contents of db/#{dir} into database (where dir is ENV['dir'] || 'base') 32 | 33 | In addition, we have plugins whereby you can export your database to/from various formats. We only deal with yaml and csv right now, but you can easily write tools for your own formats (such as Excel or XML). To use another format, just load setting the "class" parameter to the class you are using. This defaults to "YamlDb::Helper" which is a refactoring of the old yaml_db code. We'll shorten this to use class nicknames in a little bit. 34 | 35 | ## Examples 36 | 37 | One common use would be to switch your data from one database backend to another. For example, let's say you wanted to switch from SQLite to MySQL. You might execute the following steps: 38 | 39 | 1. rake db:dump 40 | 41 | 2. Edit config/database.yml and change your adapter to mysql, set up database params 42 | 43 | 3. mysqladmin create [database name] 44 | 45 | 4. rake db:load 46 | 47 | ## Credits 48 | 49 | Created by Orion Henry and Adam Wiggins. Major updates by Ricardo Chimal Jr. and Nate Kidwell. 50 | -------------------------------------------------------------------------------- /spec/yaml_db/serialization_helper_utils_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | module SerializationHelper 3 | RSpec.describe Utils do 4 | 5 | before do 6 | allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection').as_null_object) 7 | end 8 | 9 | it "returns an array of hash values using an array of ordered keys" do 10 | expect(Utils.unhash({ 'a' => 1, 'b' => 2 }, [ 'b', 'a' ])).to eq([ 2, 1 ]) 11 | end 12 | 13 | it "unhashes each hash to an array using an array of ordered keys" do 14 | expect(Utils.unhash_records([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ], [ 'b', 'a' ])).to eq([ [ 2, 1 ], [ 4, 3 ] ]) 15 | end 16 | 17 | it "returns true if it is a boolean type" do 18 | expect(Utils.is_boolean(true)).to be true 19 | expect(Utils.is_boolean('true')).to be false 20 | end 21 | 22 | it "returns an array of boolean columns" do 23 | allow(ActiveRecord::Base.connection).to receive(:columns).with('mytable').and_return([ double('a',:name => 'a',:type => :string), double('b', :name => 'b',:type => :boolean) ]) 24 | expect(Utils.boolean_columns('mytable')).to eq(['b']) 25 | end 26 | 27 | it "quotes the table name" do 28 | expect(ActiveRecord::Base.connection).to receive(:quote_table_name).with('values').and_return('`values`') 29 | expect(Utils.quote_table('values')).to eq('`values`') 30 | end 31 | 32 | it "converts ruby booleans to true and false" do 33 | expect(Utils.convert_boolean(true)).to be true 34 | expect(Utils.convert_boolean(false)).to be false 35 | end 36 | 37 | it "converts ruby strings t and f to true and false" do 38 | expect(Utils.convert_boolean('t')).to be true 39 | expect(Utils.convert_boolean('f')).to be false 40 | end 41 | 42 | it "converts ruby strings 1 and 0 to true and false" do 43 | expect(Utils.convert_boolean('1')).to be true 44 | expect(Utils.convert_boolean('0')).to be false 45 | end 46 | 47 | it "converts ruby integers 1 and 0 to true and false" do 48 | expect(Utils.convert_boolean(1)).to be true 49 | expect(Utils.convert_boolean(0)).to be false 50 | end 51 | 52 | describe ".quote_column" do 53 | it "quotes the column name" do 54 | allow(ActiveRecord::Base.connection).to receive(:quote_column_name).with('id').and_return('`id`') 55 | expect(Utils.quote_column('id')).to eq('`id`') 56 | end 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'yaml_db' 2 | 3 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 4 | RSpec.configure do |config| 5 | config.expect_with :rspec do |expectations| 6 | # This option will default to `true` in RSpec 4. It makes the `description` 7 | # and `failure_message` of custom matchers include text for helper methods 8 | # defined using `chain`, e.g.: 9 | # be_bigger_than(2).and_smaller_than(4).description 10 | # # => "be bigger than 2 and smaller than 4" 11 | # ...rather than: 12 | # # => "be bigger than 2" 13 | #expectations.include_chain_clauses_in_custom_matcher_descriptions = true 14 | end 15 | 16 | config.mock_with :rspec do |mocks| 17 | # Prevents you from mocking or stubbing a method that does not exist on 18 | # a real object. This is generally recommended, and will default to 19 | # `true` in RSpec 4. 20 | #mocks.verify_partial_doubles = true 21 | end 22 | 23 | # These two settings work together to allow you to limit a spec run 24 | # to individual examples or groups you care about by tagging them with 25 | # `:focus` metadata. When nothing is tagged with `:focus`, all examples 26 | # get run. 27 | config.filter_run :focus 28 | config.run_all_when_everything_filtered = true 29 | 30 | # Limits the available syntax to the non-monkey patched syntax that is recommended. 31 | # For more details, see: 32 | # - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax 33 | # - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 34 | # - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching 35 | config.disable_monkey_patching! 36 | 37 | # This setting enables warnings. It's recommended, but in some cases may 38 | # be too noisy due to issues in dependencies. 39 | config.warnings = true 40 | 41 | # Many RSpec users commonly either run the entire suite or an individual 42 | # file, and it's useful to allow more verbose output when running an 43 | # individual spec file. 44 | if config.files_to_run.one? 45 | # Use the documentation formatter for detailed output, 46 | # unless a formatter has already been configured 47 | # (e.g. via a command-line flag). 48 | config.default_formatter = 'doc' 49 | end 50 | 51 | # Print the 10 slowest examples and example groups at the 52 | # end of the spec run, to help surface which specs are running 53 | # particularly slow. 54 | #config.profile_examples = 10 55 | 56 | # Run specs in random order to surface order dependencies. If you find an 57 | # order dependency and want to debug it, you can fix the order by providing 58 | # the seed, which is printed after each run. 59 | # --seed 1234 60 | config.order = :random 61 | 62 | # Seed global randomization in this process using the `--seed` CLI option. 63 | # Setting this allows you to use `--seed` to deterministically reproduce 64 | # test failures related to randomization by passing the same `--seed` value 65 | # as the one that triggered the failure. 66 | Kernel.srand config.seed 67 | end 68 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: CI 3 | on: 4 | - push 5 | - pull_request 6 | jobs: 7 | spec: 8 | runs-on: ubuntu-20.04 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | ruby_version: 13 | - "1.9.3" 14 | - "2.0.0" 15 | - "2.1" 16 | - "2.2" 17 | - "2.3" 18 | rails_version: 19 | - "3.0" 20 | - "3.1" 21 | - "3.2" 22 | - "4.0" 23 | - "4.1" 24 | - "4.2" 25 | include: 26 | - ruby_version: "2.4" 27 | rails_version: "4.2" 28 | - ruby_version: "2.5" 29 | rails_version: "4.2" 30 | - ruby_version: "2.2" 31 | rails_version: "5.0" 32 | - ruby_version: "2.3" 33 | rails_version: "5.0" 34 | - ruby_version: "2.4" 35 | rails_version: "5.0" 36 | - ruby_version: "2.5" 37 | rails_version: "5.0" 38 | - ruby_version: "2.2" 39 | rails_version: "5.1" 40 | - ruby_version: "2.3" 41 | rails_version: "5.1" 42 | - ruby_version: "2.4" 43 | rails_version: "5.1" 44 | - ruby_version: "2.5" 45 | rails_version: "5.1" 46 | - ruby_version: "2.2" 47 | rails_version: "5.2" 48 | - ruby_version: "2.3" 49 | rails_version: "5.2" 50 | - ruby_version: "2.4" 51 | rails_version: "5.2" 52 | - ruby_version: "2.5" 53 | rails_version: "5.2" 54 | steps: 55 | - uses: actions/checkout@v2 56 | - uses: ruby/setup-ruby@v1 57 | with: 58 | ruby-version: ${{ matrix.ruby_version }} 59 | bundler: 1 60 | bundler-cache: true 61 | cache-version: rails${{ matrix.rails_version }} 62 | env: 63 | RAILS_VERSION: "~> ${{ matrix.rails_version }}.0" 64 | - run: gem update --no-rdoc --no-ri --system 2.7.11 65 | if: ${{ matrix.ruby_version == '1.9.3' }} 66 | - run: bundle exec rake 67 | 68 | spec-ruby1_8_7: 69 | runs-on: ubuntu-20.04 70 | strategy: 71 | # fail-fast: false 72 | matrix: 73 | rails_version: 74 | - "3.0" 75 | - "3.1" 76 | - "3.2" 77 | steps: 78 | - uses: actions/checkout@v2 79 | - uses: docker://silvioq/ruby-1.8.7 80 | with: 81 | entrypoint: /bin/bash 82 | args: > 83 | -x -c "gem install bundler -v '~> 1.17' 84 | && bundle install 85 | && bundle exec rake" 86 | env: 87 | RAILS_VERSION: "~> ${{ matrix.rails_version }}.0" 88 | 89 | spec-ruby1_9_2: 90 | runs-on: ubuntu-20.04 91 | strategy: 92 | fail-fast: false 93 | matrix: 94 | rails_version: 95 | - "3.0" 96 | - "3.1" 97 | - "3.2" 98 | steps: 99 | - uses: actions/checkout@v2 100 | - run: wget https://rubygems.org/rubygems/rubygems-2.7.11.tgz 101 | - run: tar -xzf rubygems-2.7.11.tgz 102 | - uses: docker://allenwei/ruby-1.9.2 103 | with: 104 | entrypoint: /bin/bash 105 | args: > 106 | -x -c "unset GEM_HOME 107 | && cd rubygems-2.7.11 108 | && ruby setup.rb --no-document --silent 109 | && cd .. 110 | && gem install --no-document bundler -v '~> 1.17' 111 | && bundle install 112 | && bundle exec rake" 113 | env: 114 | RAILS_VERSION: "~> ${{ matrix.rails_version }}.0" 115 | -------------------------------------------------------------------------------- /spec/yaml_db/rake_tasks_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | RSpec.describe RakeTasks do 3 | before do 4 | @serializer = instance_double(SerializationHelper::Base) 5 | allow(SerializationHelper::Base).to receive(:new).and_return(@serializer) 6 | allow(Rails).to receive(:root).and_return('/root') 7 | allow(Time).to receive(:now).and_return(Time.parse('2007-08-09 12:34:56')) 8 | stub_const('UserSpecifiedHelper', Class.new) 9 | allow(UserSpecifiedHelper).to receive(:extension).and_return('ext') 10 | end 11 | 12 | describe '.data_dump_task' do 13 | it 'dumps to a file' do 14 | expect(SerializationHelper::Base).to receive(:new).once.with(Helper) 15 | expect(@serializer).to receive(:dump).once.with('/root/db/data.yml') 16 | RakeTasks.data_dump_task 17 | end 18 | 19 | it 'dumps to a file using a user-specified format class' do 20 | stub_const('ENV', 'class' => 'UserSpecifiedHelper') 21 | expect(SerializationHelper::Base).to receive(:new).once.with(UserSpecifiedHelper) 22 | expect(@serializer).to receive(:dump).once.with('/root/db/data.ext') 23 | RakeTasks.data_dump_task 24 | end 25 | end 26 | 27 | describe '.data_dump_dir_task' do 28 | it 'dumps to a directory' do 29 | expect(SerializationHelper::Base).to receive(:new).once.with(Helper) 30 | expect(@serializer).to receive(:dump_to_dir).once.with('/root/db/2007-08-09T123456') 31 | RakeTasks.data_dump_dir_task 32 | end 33 | 34 | it 'dumps to a directory using a user-specified format class' do 35 | stub_const('ENV', 'class' => 'UserSpecifiedHelper') 36 | expect(SerializationHelper::Base).to receive(:new).once.with(UserSpecifiedHelper) 37 | expect(@serializer).to receive(:dump_to_dir).once.with('/root/db/2007-08-09T123456') 38 | RakeTasks.data_dump_dir_task 39 | end 40 | 41 | it 'dumps to a user-specified directory' do 42 | stub_const('ENV', 'dir' => 'user_dir') 43 | expect(SerializationHelper::Base).to receive(:new).once.with(Helper) 44 | expect(@serializer).to receive(:dump_to_dir).once.with('/root/db/user_dir') 45 | RakeTasks.data_dump_dir_task 46 | end 47 | end 48 | 49 | describe '.data_load_task' do 50 | it 'loads a file' do 51 | expect(SerializationHelper::Base).to receive(:new).once.with(Helper) 52 | expect(@serializer).to receive(:load).once.with('/root/db/data.yml') 53 | RakeTasks.data_load_task 54 | end 55 | 56 | it 'loads a file using a user-specified format class' do 57 | stub_const('ENV', 'class' => 'UserSpecifiedHelper') 58 | expect(SerializationHelper::Base).to receive(:new).once.with(UserSpecifiedHelper) 59 | expect(@serializer).to receive(:load).once.with('/root/db/data.ext') 60 | RakeTasks.data_load_task 61 | end 62 | end 63 | 64 | describe '.data_load_dir_task' do 65 | it 'loads a directory' do 66 | expect(SerializationHelper::Base).to receive(:new).once.with(Helper) 67 | expect(@serializer).to receive(:load_from_dir).once.with('/root/db/base') 68 | RakeTasks.data_load_dir_task 69 | end 70 | 71 | it 'loads a directory using a user-specified format class' do 72 | stub_const('ENV', 'class' => 'UserSpecifiedHelper') 73 | expect(SerializationHelper::Base).to receive(:new).once.with(UserSpecifiedHelper) 74 | expect(@serializer).to receive(:load_from_dir).once.with('/root/db/base') 75 | RakeTasks.data_load_dir_task 76 | end 77 | 78 | it 'loads a user-specified directory' do 79 | stub_const('ENV', 'dir' => 'user_dir') 80 | expect(SerializationHelper::Base).to receive(:new).once.with(Helper) 81 | expect(@serializer).to receive(:load_from_dir).once.with('/root/db/user_dir') 82 | RakeTasks.data_load_dir_task 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/yaml_db/serialization_helper_load_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | module SerializationHelper 3 | RSpec.describe Load do 4 | 5 | before do 6 | allow(Utils).to receive(:quote_table).with('mytable').and_return('mytable') 7 | 8 | allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection').as_null_object) 9 | allow(ActiveRecord::Base.connection).to receive(:transaction).and_yield 10 | @io = StringIO.new 11 | end 12 | 13 | it "truncates the table" do 14 | allow(ActiveRecord::Base.connection).to receive(:execute).with("TRUNCATE mytable").and_return(true) 15 | expect(ActiveRecord::Base.connection).not_to receive(:execute).with("DELETE FROM mytable") 16 | Load.truncate_table('mytable') 17 | end 18 | 19 | it "deletes the table if truncate throws an exception" do 20 | expect(ActiveRecord::Base.connection).to receive(:execute).with("TRUNCATE mytable").and_raise() 21 | expect(ActiveRecord::Base.connection).to receive(:execute).with("DELETE FROM mytable").and_return(true) 22 | Load.truncate_table('mytable') 23 | end 24 | 25 | it "calls reset pk sequence if the connection adapter is postgres" do 26 | expect(ActiveRecord::Base.connection).to receive(:respond_to?).with(:reset_pk_sequence!).and_return(true) 27 | expect(ActiveRecord::Base.connection).to receive(:reset_pk_sequence!).with('mytable') 28 | Load.reset_pk_sequence!('mytable') 29 | end 30 | 31 | it "does not call reset pk sequence for other adapters" do 32 | expect(ActiveRecord::Base.connection).to receive(:respond_to?).with(:reset_pk_sequence!).and_return(false) 33 | expect(ActiveRecord::Base.connection).not_to receive(:reset_pk_sequence!) 34 | Load.reset_pk_sequence!('mytable') 35 | end 36 | 37 | it "inserts records into a table" do 38 | allow(ActiveRecord::Base.connection).to receive(:quote_column_name).with('a').and_return('a') 39 | allow(ActiveRecord::Base.connection).to receive(:quote_column_name).with('b').and_return('b') 40 | allow(ActiveRecord::Base.connection).to receive(:quote).with(1).and_return("'1'") 41 | allow(ActiveRecord::Base.connection).to receive(:quote).with(2).and_return("'2'") 42 | allow(ActiveRecord::Base.connection).to receive(:quote).with(3).and_return("'3'") 43 | allow(ActiveRecord::Base.connection).to receive(:quote).with(4).and_return("'4'") 44 | expect(ActiveRecord::Base.connection).to receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('1','2')") 45 | expect(ActiveRecord::Base.connection).to receive(:execute).with("INSERT INTO mytable (a,b) VALUES ('3','4')") 46 | 47 | Load.load_records('mytable', ['a', 'b'], [[1, 2], [3, 4]]) 48 | end 49 | 50 | it "quotes column names that correspond to sql keywords" do 51 | allow(ActiveRecord::Base.connection).to receive(:quote_column_name).with('a').and_return('a') 52 | allow(ActiveRecord::Base.connection).to receive(:quote_column_name).with('count').and_return('"count"') 53 | allow(ActiveRecord::Base.connection).to receive(:quote).with(1).and_return("'1'") 54 | allow(ActiveRecord::Base.connection).to receive(:quote).with(2).and_return("'2'") 55 | allow(ActiveRecord::Base.connection).to receive(:quote).with(3).and_return("'3'") 56 | allow(ActiveRecord::Base.connection).to receive(:quote).with(4).and_return("'4'") 57 | expect(ActiveRecord::Base.connection).to receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('1','2')") 58 | expect(ActiveRecord::Base.connection).to receive(:execute).with("INSERT INTO mytable (a,\"count\") VALUES ('3','4')") 59 | 60 | Load.load_records('mytable', ['a', 'count'], [[1, 2], [3, 4]]) 61 | end 62 | 63 | it "truncates the table and then loads the records into the table" do 64 | expect(Load).to receive(:truncate_table).with('mytable') 65 | expect(Load).to receive(:load_records).with('mytable', ['a', 'b'], [[1, 2], [3, 4]]) 66 | expect(Load).to receive(:reset_pk_sequence!).with('mytable') 67 | 68 | Load.load_table('mytable', { 'columns' => [ 'a', 'b' ], 'records' => [[1, 2], [3, 4]] }) 69 | end 70 | 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/yaml_db/serialization_helper_dump_spec.rb: -------------------------------------------------------------------------------- 1 | module YamlDb 2 | module SerializationHelper 3 | RSpec.describe Dump do 4 | 5 | before do 6 | allow(ActiveRecord::Base).to receive(:connection).and_return(double('connection').as_null_object) 7 | allow(ActiveRecord::Base.connection).to receive(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ]) 8 | allow(ActiveRecord::Base.connection).to receive(:columns).with('mytable').and_return([ double('a', :name => 'a', :type => :string), double('b', :name => 'b', :type => :string) ]) 9 | allow(ActiveRecord::Base.connection).to receive(:select_one).and_return({"count"=>"2"}) 10 | allow(ActiveRecord::Base.connection).to receive(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]) 11 | allow(Utils).to receive(:quote_table).with('mytable').and_return('mytable') 12 | end 13 | 14 | before(:each) do 15 | allow(File).to receive(:open).with('dump.yml', 'w').and_yield(StringIO.new) 16 | @io = StringIO.new 17 | end 18 | 19 | it "returns a list of column names" do 20 | expect(Dump.table_column_names('mytable')).to eq([ 'a', 'b' ]) 21 | end 22 | 23 | it "returns the total number of records in a table" do 24 | expect(Dump.table_record_count('mytable')).to eq(2) 25 | end 26 | 27 | describe ".each_table_page" do 28 | before do 29 | allow(Dump).to receive(:sort_keys) 30 | end 31 | 32 | it "returns all records from the database and returns them when there is only 1 page" do 33 | Dump.each_table_page('mytable') do |records| 34 | expect(records).to eq([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]) 35 | end 36 | end 37 | 38 | it "paginates records from the database and returns them" do 39 | allow(ActiveRecord::Base.connection).to receive(:select_all).and_return([ { 'a' => 1, 'b' => 2 } ], [ { 'a' => 3, 'b' => 4 } ]) 40 | 41 | records = [ ] 42 | Dump.each_table_page('mytable', 1) do |page| 43 | expect(page.size).to eq(1) 44 | records.concat(page) 45 | end 46 | 47 | expect(records).to eq([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]) 48 | end 49 | end 50 | 51 | it "dumps a table's contents to yaml" do 52 | expect(Dump).to receive(:dump_table_columns) 53 | expect(Dump).to receive(:dump_table_records) 54 | Dump.dump_table(@io, 'mytable') 55 | end 56 | 57 | it "does not dump a table's contents when the record count is zero" do 58 | allow(Dump).to receive(:table_record_count).with('mytable').and_return(0) 59 | expect(Dump).not_to receive(:dump_table_columns) 60 | expect(Dump).not_to receive(:dump_table_records) 61 | Dump.dump_table(@io, 'mytable') 62 | end 63 | 64 | describe ".tables" do 65 | it "returns a list of tables without the rails schema table" do 66 | expect(Dump.tables).to eq(['mytable']) 67 | end 68 | 69 | it "returns the list of tables in a consistent (sorted) order" do 70 | allow(ActiveRecord::Base.connection).to receive(:tables).and_return(%w(z y x)) 71 | expect(Dump.tables).to eq(%w(x y z)) 72 | end 73 | end 74 | 75 | describe ".sort_keys" do 76 | before do 77 | allow(Utils).to receive(:quote_column) { |column| column } 78 | end 79 | 80 | it "returns the first column as sort key" do 81 | expect(Dump.sort_keys('mytable')).to eq(['a']) 82 | end 83 | 84 | it "returns the combined ids as sort key if the table looks like a HABTM" do 85 | allow(ActiveRecord::Base.connection).to receive(:columns).with('mytable').and_return([ 86 | double('a_id', :name => 'a_id', :type => :string), 87 | double('b_id', :name => 'b_id', :type => :string) 88 | ]) 89 | 90 | expect(Dump.sort_keys('mytable')).to eq(['a_id', 'b_id']) 91 | end 92 | 93 | it "quotes the column name" do 94 | allow(Utils).to receive(:quote_column).with('a').and_return('`a`') 95 | expect(Dump.sort_keys('mytable')).to eq(['`a`']) 96 | end 97 | end 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/yaml_db/serialization_helper.rb: -------------------------------------------------------------------------------- 1 | require 'active_support/core_ext/kernel/reporting' 2 | 3 | module YamlDb 4 | module SerializationHelper 5 | 6 | class Base 7 | attr_reader :extension 8 | 9 | def initialize(helper) 10 | @dumper = helper.dumper 11 | @loader = helper.loader 12 | @extension = helper.extension 13 | end 14 | 15 | def dump(filename) 16 | disable_logger 17 | File.open(filename, "w") do |file| 18 | @dumper.dump(file) 19 | end 20 | reenable_logger 21 | end 22 | 23 | def dump_to_dir(dirname) 24 | Dir.mkdir(dirname) 25 | tables = @dumper.tables 26 | tables.each do |table| 27 | File.open("#{dirname}/#{table}.#{@extension}", "w") do |io| 28 | @dumper.before_table(io, table) 29 | @dumper.dump_table io, table 30 | @dumper.after_table(io, table) 31 | end 32 | end 33 | end 34 | 35 | def load(filename, truncate = true) 36 | disable_logger 37 | @loader.load(File.new(filename, "r"), truncate) 38 | reenable_logger 39 | end 40 | 41 | def load_from_dir(dirname, truncate = true) 42 | Dir.entries(dirname).each do |filename| 43 | if filename =~ /^[.]/ 44 | next 45 | end 46 | @loader.load(File.new("#{dirname}/#{filename}", "r"), truncate) 47 | end 48 | end 49 | 50 | def disable_logger 51 | @@old_logger = ActiveRecord::Base.logger 52 | ActiveRecord::Base.logger = nil 53 | end 54 | 55 | def reenable_logger 56 | ActiveRecord::Base.logger = @@old_logger 57 | end 58 | end 59 | 60 | class Load 61 | def self.load(io, truncate = true) 62 | ActiveRecord::Base.connection.transaction do 63 | load_documents(io, truncate) 64 | end 65 | end 66 | 67 | def self.truncate_table(table) 68 | begin 69 | ActiveRecord::Base.connection.execute("TRUNCATE #{Utils.quote_table(table)}") 70 | rescue Exception 71 | ActiveRecord::Base.connection.execute("DELETE FROM #{Utils.quote_table(table)}") 72 | end 73 | end 74 | 75 | def self.load_table(table, data, truncate = true) 76 | column_names = data['columns'] 77 | if truncate 78 | truncate_table(table) 79 | end 80 | load_records(table, column_names, data['records']) 81 | reset_pk_sequence!(table) 82 | end 83 | 84 | def self.load_records(table, column_names, records) 85 | if column_names.nil? 86 | return 87 | end 88 | quoted_column_names = column_names.map { |column| ActiveRecord::Base.connection.quote_column_name(column) }.join(',') 89 | quoted_table_name = Utils.quote_table(table) 90 | records.each do |record| 91 | quoted_values = record.map{|c| ActiveRecord::Base.connection.quote(c)}.join(',') 92 | ActiveRecord::Base.connection.execute("INSERT INTO #{quoted_table_name} (#{quoted_column_names}) VALUES (#{quoted_values})") 93 | end 94 | end 95 | 96 | def self.reset_pk_sequence!(table_name) 97 | if ActiveRecord::Base.connection.respond_to?(:reset_pk_sequence!) 98 | ActiveRecord::Base.connection.reset_pk_sequence!(table_name) 99 | end 100 | end 101 | 102 | end 103 | 104 | module Utils 105 | 106 | def self.unhash(hash, keys) 107 | keys.map { |key| hash[key] } 108 | end 109 | 110 | def self.unhash_records(records, keys) 111 | records.each_with_index do |record, index| 112 | records[index] = unhash(record, keys) 113 | end 114 | 115 | records 116 | end 117 | 118 | def self.convert_booleans(records, columns) 119 | records.each do |record| 120 | columns.each do |column| 121 | next if is_boolean(record[column]) 122 | record[column] = convert_boolean(record[column]) 123 | end 124 | end 125 | records 126 | end 127 | 128 | def self.convert_boolean(value) 129 | ['t', '1', true, 1].include?(value) 130 | end 131 | 132 | def self.boolean_columns(table) 133 | columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } } 134 | columns.map { |c| c.name } 135 | end 136 | 137 | def self.is_boolean(value) 138 | value.kind_of?(TrueClass) or value.kind_of?(FalseClass) 139 | end 140 | 141 | def self.quote_table(table) 142 | ActiveRecord::Base.connection.quote_table_name(table) 143 | end 144 | 145 | def self.quote_column(column) 146 | ActiveRecord::Base.connection.quote_column_name(column) 147 | end 148 | end 149 | 150 | class Dump 151 | def self.before_table(io, table) 152 | 153 | end 154 | 155 | def self.dump(io) 156 | tables.each do |table| 157 | before_table(io, table) 158 | dump_table(io, table) 159 | after_table(io, table) 160 | end 161 | end 162 | 163 | def self.after_table(io, table) 164 | 165 | end 166 | 167 | def self.tables 168 | ActiveRecord::Base.connection.tables.reject { |table| ['schema_info', 'schema_migrations'].include?(table) }.sort 169 | end 170 | 171 | def self.dump_table(io, table) 172 | return if table_record_count(table).zero? 173 | 174 | dump_table_columns(io, table) 175 | dump_table_records(io, table) 176 | end 177 | 178 | def self.table_column_names(table) 179 | ActiveRecord::Base.connection.columns(table).map { |c| c.name } 180 | end 181 | 182 | 183 | def self.each_table_page(table, records_per_page=1000) 184 | total_count = table_record_count(table) 185 | pages = (total_count.to_f / records_per_page).ceil - 1 186 | keys = sort_keys(table) 187 | boolean_columns = Utils.boolean_columns(table) 188 | 189 | (0..pages).to_a.each do |page| 190 | query = Arel::Table.new(table).order(*keys).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*')) 191 | records = ActiveRecord::Base.connection.select_all(query.to_sql) 192 | records = Utils.convert_booleans(records, boolean_columns) 193 | yield records 194 | end 195 | end 196 | 197 | def self.table_record_count(table) 198 | ActiveRecord::Base.connection.select_one("SELECT COUNT(*) FROM #{Utils.quote_table(table)}").values.first.to_i 199 | end 200 | 201 | # Return the first column as sort key unless the table looks like a 202 | # standard has_and_belongs_to_many join table, in which case add the second "ID column" 203 | def self.sort_keys(table) 204 | first_column, second_column = table_column_names(table) 205 | 206 | if [first_column, second_column].all? { |name| name =~ /_id$/ } 207 | [Utils.quote_column(first_column), Utils.quote_column(second_column)] 208 | else 209 | [Utils.quote_column(first_column)] 210 | end 211 | end 212 | end 213 | end 214 | end 215 | --------------------------------------------------------------------------------