├── VERSION ├── test ├── adapters │ ├── mysql.rb │ ├── mysql2.rb │ ├── postgis.rb │ ├── sqlite3.rb │ ├── jdbcmysql.rb │ ├── postgresql.rb │ ├── spatialite.rb │ ├── mysql2spatial.rb │ ├── mysqlspatial.rb │ └── seamless_database_pool.rb ├── models │ ├── book.rb │ ├── widget.rb │ ├── group.rb │ └── topic.rb ├── postgis │ └── import_test.rb ├── postgresql │ └── import_test.rb ├── mysql │ └── import_test.rb ├── mysql2 │ └── import_test.rb ├── jdbcmysql │ └── import_test.rb ├── mysqlspatial │ └── import_test.rb ├── mysqlspatial2 │ └── import_test.rb ├── schema │ ├── version.rb │ ├── mysql_schema.rb │ └── generic_schema.rb ├── support │ ├── factories.rb │ ├── postgresql │ │ └── import_examples.rb │ ├── generate.rb │ ├── active_support │ │ └── test_case_extensions.rb │ └── mysql │ │ ├── assertions.rb │ │ └── import_examples.rb ├── database.yml.sample ├── synchronize_test.rb ├── test_helper.rb ├── active_record │ └── connection_adapter_test.rb └── import_test.rb ├── benchmarks ├── models │ ├── test_innodb.rb │ ├── test_memory.rb │ └── test_myisam.rb ├── lib │ ├── float.rb │ ├── output_to_csv.rb │ ├── mysql_benchmark.rb │ ├── output_to_html.rb │ ├── cli_parser.rb │ └── base.rb ├── boot.rb ├── schema │ └── mysql_schema.rb ├── README └── benchmark.rb ├── lib ├── activerecord-import │ ├── adapters │ │ ├── mysql2_adapter.rb │ │ ├── sqlite3_adapter.rb │ │ ├── postgresql_adapter.rb │ │ ├── mysql_adapter.rb │ │ └── abstract_adapter.rb │ ├── active_record │ │ └── adapters │ │ │ ├── mysql_adapter.rb │ │ │ ├── jdbcmysql_adapter.rb │ │ │ ├── mysql2_adapter.rb │ │ │ ├── sqlite3_adapter.rb │ │ │ ├── postgresql_adapter.rb │ │ │ ├── seamless_database_pool_adapter.rb │ │ │ └── abstract_adapter.rb │ ├── mysql.rb │ ├── mysql2.rb │ ├── sqlite3.rb │ ├── postgresql.rb │ ├── base.rb │ ├── synchronize.rb │ └── import.rb └── activerecord-import.rb ├── .gitignore ├── README.markdown ├── Gemfile ├── Gemfile.lock ├── Rakefile └── activerecord-import.gemspec /VERSION: -------------------------------------------------------------------------------- 1 | 0.3.0 -------------------------------------------------------------------------------- /test/adapters/mysql.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "mysql" -------------------------------------------------------------------------------- /test/adapters/mysql2.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "mysql2" 2 | -------------------------------------------------------------------------------- /test/adapters/postgis.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "postgis" -------------------------------------------------------------------------------- /test/adapters/sqlite3.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "sqlite3" -------------------------------------------------------------------------------- /test/adapters/jdbcmysql.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "jdbcmysql" 2 | -------------------------------------------------------------------------------- /test/adapters/postgresql.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "postgresql" -------------------------------------------------------------------------------- /test/adapters/spatialite.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "spatialite" -------------------------------------------------------------------------------- /test/adapters/mysql2spatial.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "mysql2spatial" -------------------------------------------------------------------------------- /test/adapters/mysqlspatial.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "mysqlspatial" -------------------------------------------------------------------------------- /test/adapters/seamless_database_pool.rb: -------------------------------------------------------------------------------- 1 | ENV["ARE_DB"] = "seamless_database_pool" 2 | -------------------------------------------------------------------------------- /test/models/book.rb: -------------------------------------------------------------------------------- 1 | class Book < ActiveRecord::Base 2 | belongs_to :topic 3 | end 4 | -------------------------------------------------------------------------------- /test/models/widget.rb: -------------------------------------------------------------------------------- 1 | class Widget < ActiveRecord::Base 2 | self.primary_key = :w_id 3 | end -------------------------------------------------------------------------------- /test/models/group.rb: -------------------------------------------------------------------------------- 1 | class Group < ActiveRecord::Base 2 | self.table_name = 'group' 3 | end 4 | -------------------------------------------------------------------------------- /benchmarks/models/test_innodb.rb: -------------------------------------------------------------------------------- 1 | class TestInnoDb < ActiveRecord::Base 2 | set_table_name 'test_innodb' 3 | end 4 | -------------------------------------------------------------------------------- /benchmarks/models/test_memory.rb: -------------------------------------------------------------------------------- 1 | class TestMemory < ActiveRecord::Base 2 | set_table_name 'test_memory' 3 | end 4 | -------------------------------------------------------------------------------- /benchmarks/models/test_myisam.rb: -------------------------------------------------------------------------------- 1 | class TestMyISAM < ActiveRecord::Base 2 | set_table_name 'test_myisam' 3 | end 4 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/mysql2_adapter.rb: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + "/mysql_adapter" 2 | 3 | module ActiveRecord::Import::Mysql2Adapter 4 | include ActiveRecord::Import::MysqlAdapter 5 | end -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/sqlite3_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::SQLite3Adapter 2 | def next_value_for_sequence(sequence_name) 3 | %{nextval('#{sequence_name}')} 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /test/postgis/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') 3 | 4 | should_support_postgresql_import_functionality -------------------------------------------------------------------------------- /test/postgresql/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | require File.expand_path(File.dirname(__FILE__) + '/../support/postgresql/import_examples') 3 | 4 | should_support_postgresql_import_functionality -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/postgresql_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::PostgreSQLAdapter 2 | include ActiveRecord::Import::ImportSupport 3 | 4 | def next_value_for_sequence(sequence_name) 5 | %{nextval('#{sequence_name}')} 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/mysql_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/mysql_adapter" 2 | require "activerecord-import/adapters/mysql_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::MysqlAdapter 5 | include ActiveRecord::Import::MysqlAdapter 6 | end 7 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/mysql_adapter" 2 | require "activerecord-import/adapters/mysql_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::MysqlAdapter 5 | include ActiveRecord::Import::MysqlAdapter 6 | end 7 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/mysql2_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/mysql2_adapter" 2 | require "activerecord-import/adapters/mysql2_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::Mysql2Adapter 5 | include ActiveRecord::Import::Mysql2Adapter 6 | end 7 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/sqlite3_adapter" 2 | require "activerecord-import/adapters/sqlite3_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::SQLite3Adapter 5 | include ActiveRecord::Import::SQLite3Adapter 6 | end 7 | 8 | -------------------------------------------------------------------------------- /test/mysql/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality -------------------------------------------------------------------------------- /test/mysql2/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/postgresql_adapter.rb: -------------------------------------------------------------------------------- 1 | require "active_record/connection_adapters/postgresql_adapter" 2 | require "activerecord-import/adapters/postgresql_adapter" 3 | 4 | class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter 5 | include ActiveRecord::Import::PostgreSQLAdapter 6 | end 7 | 8 | -------------------------------------------------------------------------------- /test/jdbcmysql/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality -------------------------------------------------------------------------------- /test/models/topic.rb: -------------------------------------------------------------------------------- 1 | class Topic < ActiveRecord::Base 2 | validates_presence_of :author_name 3 | has_many :books 4 | belongs_to :parent, :class_name => "Topic" 5 | 6 | composed_of :description, :mapping => [ %w(title title), %w(author_name author_name)], :allow_nil => true, :class_name => "TopicDescription" 7 | end 8 | -------------------------------------------------------------------------------- /test/mysqlspatial/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality -------------------------------------------------------------------------------- /test/mysqlspatial2/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/assertions') 4 | require File.expand_path(File.dirname(__FILE__) + '/../support/mysql/import_examples') 5 | 6 | should_support_mysql_import_functionality -------------------------------------------------------------------------------- /test/schema/version.rb: -------------------------------------------------------------------------------- 1 | class SchemaInfo < ActiveRecord::Base 2 | if respond_to?(:table_name=) 3 | self.table_name = 'schema_info' 4 | else 5 | # this is becoming deprecated in ActiveRecord but not all adapters supported it 6 | # at this time 7 | set_table_name 'schema_info' 8 | end 9 | VERSION = 12 10 | end 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## MAC OS 2 | .DS_Store 3 | 4 | ## TEXTMATE 5 | *.tmproj 6 | tmtags 7 | 8 | ## EMACS 9 | *~ 10 | \#* 11 | .\#* 12 | 13 | ## VIM 14 | *.swp 15 | 16 | ## PROJECT::GENERAL 17 | coverage 18 | rdoc 19 | pkg 20 | 21 | ## PROJECT::SPECIFIC 22 | log/*.log 23 | test.db 24 | test/database.yml 25 | 26 | .bundle/ 27 | .redcar/ 28 | .rvmrc 29 | docsite/ -------------------------------------------------------------------------------- /benchmarks/lib/float.rb: -------------------------------------------------------------------------------- 1 | # Taken from http://www.programmingishard.com/posts/show/128 2 | # Posted by rbates 3 | class Float 4 | def round_to(x) 5 | (self * 10**x).round.to_f / 10**x 6 | end 7 | 8 | def ceil_to(x) 9 | (self * 10**x).ceil.to_f / 10**x 10 | end 11 | 12 | def floor_to(x) 13 | (self * 10**x).floor.to_f / 10**x 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb: -------------------------------------------------------------------------------- 1 | require "seamless_database_pool" 2 | require "active_record/connection_adapters/seamless_database_pool_adapter" 3 | require "activerecord-import/adapters/mysql_adapter" 4 | 5 | class ActiveRecord::ConnectionAdapters::SeamlessDatabasePoolAdapter 6 | include ActiveRecord::Import::MysqlAdapter 7 | end 8 | -------------------------------------------------------------------------------- /lib/activerecord-import/active_record/adapters/abstract_adapter.rb: -------------------------------------------------------------------------------- 1 | require "activerecord-import/adapters/abstract_adapter" 2 | 3 | module ActiveRecord # :nodoc: 4 | module ConnectionAdapters # :nodoc: 5 | class AbstractAdapter # :nodoc: 6 | extend ActiveRecord::Import::AbstractAdapter::ClassMethods 7 | include ActiveRecord::Import::AbstractAdapter::InstanceMethods 8 | end 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /lib/activerecord-import/mysql.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require File.expand_path(File.join(File.dirname(__FILE__), "/../activerecord-import")) 8 | 9 | -------------------------------------------------------------------------------- /lib/activerecord-import/mysql2.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require File.expand_path(File.join(File.dirname(__FILE__), "/../activerecord-import")) 8 | 9 | -------------------------------------------------------------------------------- /lib/activerecord-import/sqlite3.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require File.expand_path(File.join(File.dirname(__FILE__), "/../activerecord-import")) 8 | 9 | -------------------------------------------------------------------------------- /lib/activerecord-import/postgresql.rb: -------------------------------------------------------------------------------- 1 | warn <<-MSG 2 | [DEPRECATION] loading activerecord-import via 'require "activerecord-import/"' 3 | is deprecated. Update to autorequire using 'require "activerecord-import"'. See 4 | http://github.com/zdennis/activerecord-import/wiki/Requiring for more information 5 | MSG 6 | 7 | require File.expand_path(File.join(File.dirname(__FILE__), "/../activerecord-import")) 8 | 9 | -------------------------------------------------------------------------------- /test/support/factories.rb: -------------------------------------------------------------------------------- 1 | Factory.define :group do |m| 2 | m.sequence(:order) { |n| "Order #{n}" } 3 | end 4 | 5 | Factory.define :invalid_topic, :class => "Topic" do |m| 6 | m.sequence(:title){ |n| "Title #{n}"} 7 | m.author_name nil 8 | end 9 | 10 | Factory.define :topic do |m| 11 | m.sequence(:title){ |n| "Title #{n}"} 12 | m.sequence(:author_name){ |n| "Author #{n}"} 13 | end 14 | 15 | Factory.define :widget do |m| 16 | m.sequence(:w_id){ |n| n} 17 | end 18 | -------------------------------------------------------------------------------- /benchmarks/lib/output_to_csv.rb: -------------------------------------------------------------------------------- 1 | require 'fastercsv' 2 | 3 | module OutputToCSV 4 | def self.output_results( filename, results ) 5 | FasterCSV.open( filename, 'w' ) do |csv| 6 | # Iterate over each result set, which contains many results 7 | results.each do |result_set| 8 | columns, times = [], [] 9 | result_set.each do |result| 10 | columns << result.description 11 | times << result.tms.real 12 | end 13 | csv << columns 14 | csv << times 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /benchmarks/boot.rb: -------------------------------------------------------------------------------- 1 | begin ; require 'rubygems' ; rescue LoadError ; end 2 | require 'active_record' # ActiveRecord loads the Benchmark library automatically 3 | require 'active_record/version' 4 | require 'fastercsv' 5 | require 'fileutils' 6 | require 'logger' 7 | 8 | # Files are loaded alphabetically. If this is a problem then manually specify the files 9 | # that need to be loaded here. 10 | Dir[ File.join( File.dirname( __FILE__ ), 'lib', '*.rb' ) ].sort.each{ |f| require f } 11 | 12 | ActiveRecord::Base.logger = Logger.new STDOUT 13 | 14 | 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # activerecord-import 2 | 3 | activerecord-import is a library for bulk inserting data using ActiveRecord. 4 | 5 | For more information on activerecord-import please see its wiki: https://github.com/zdennis/activerecord-import/wiki 6 | 7 | # License 8 | 9 | This is licensed under the ruby license. 10 | 11 | # Author 12 | 13 | Zach Dennis (zach.dennis@gmail.com) 14 | 15 | # Contributors 16 | 17 | * Blythe Dunham 18 | * Gabe da Silveira 19 | * Henry Work 20 | * James Herdman 21 | * Marcus Crafter 22 | * Thibaud Guillaume-Gentil 23 | * Mark Van Holstyn 24 | * Victor Costan 25 | -------------------------------------------------------------------------------- /benchmarks/schema/mysql_schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | create_table :test_myisam, :options=>'ENGINE=MyISAM', :force=>true do |t| 3 | t.column :my_name, :string, :null=>false 4 | t.column :description, :string 5 | end 6 | 7 | create_table :test_innodb, :options=>'ENGINE=InnoDb', :force=>true do |t| 8 | t.column :my_name, :string, :null=>false 9 | t.column :description, :string 10 | end 11 | 12 | create_table :test_memory, :options=>'ENGINE=Memory', :force=>true do |t| 13 | t.column :my_name, :string, :null=>false 14 | t.column :description, :string 15 | end 16 | end -------------------------------------------------------------------------------- /test/schema/mysql_schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | 3 | create_table :books, :options=>'ENGINE=MyISAM', :force=>true do |t| 4 | t.column :title, :string, :null=>false 5 | t.column :publisher, :string, :null=>false, :default => 'Default Publisher' 6 | t.column :author_name, :string, :null=>false 7 | t.column :created_at, :datetime 8 | t.column :created_on, :datetime 9 | t.column :updated_at, :datetime 10 | t.column :updated_on, :datetime 11 | t.column :publish_date, :date 12 | t.column :topic_id, :integer 13 | t.column :for_sale, :boolean, :default => true 14 | end 15 | execute "ALTER TABLE books ADD FULLTEXT( `title`, `publisher`, `author_name` )" 16 | 17 | end 18 | -------------------------------------------------------------------------------- /test/support/postgresql/import_examples.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | def should_support_postgresql_import_functionality 3 | describe "#supports_imports?" do 4 | it "should support import" do 5 | assert ActiveRecord::Base.supports_import? 6 | end 7 | end 8 | 9 | describe "#import" do 10 | it "should import with a single insert" do 11 | # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics 12 | assert_difference "Topic.count", +10 do 13 | result = Topic.import Build(3, :topics) 14 | assert_equal 1, result.num_inserts 15 | 16 | result = Topic.import Build(7, :topics) 17 | assert_equal 1, result.num_inserts 18 | end 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/activerecord-import.rb: -------------------------------------------------------------------------------- 1 | class ActiveRecord::Base 2 | class << self 3 | def establish_connection_with_activerecord_import(*args) 4 | establish_connection_without_activerecord_import(*args) 5 | ActiveSupport.run_load_hooks(:active_record_connection_established, connection_pool) 6 | end 7 | alias_method_chain :establish_connection, :activerecord_import 8 | end 9 | end 10 | 11 | ActiveSupport.on_load(:active_record_connection_established) do |connection_pool| 12 | if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool) 13 | require File.join File.dirname(__FILE__), "activerecord-import/base" 14 | end 15 | ActiveRecord::Import.load_from_connection_pool connection_pool 16 | end 17 | -------------------------------------------------------------------------------- /benchmarks/lib/mysql_benchmark.rb: -------------------------------------------------------------------------------- 1 | class MysqlBenchmark < BenchmarkBase 2 | 3 | def benchmark_all( array_of_cols_and_vals ) 4 | methods = self.methods.find_all { |m| m =~ /benchmark_/ } 5 | methods.delete_if{ |m| m =~ /benchmark_(all|model)/ } 6 | methods.each { |method| self.send( method, array_of_cols_and_vals ) } 7 | end 8 | 9 | def benchmark_myisam( array_of_cols_and_vals ) 10 | bm_model( TestMyISAM, array_of_cols_and_vals ) 11 | end 12 | 13 | def benchmark_innodb( array_of_cols_and_vals ) 14 | bm_model( TestInnoDb, array_of_cols_and_vals ) 15 | end 16 | 17 | def benchmark_memory( array_of_cols_and_vals ) 18 | bm_model( TestMemory, array_of_cols_and_vals ) 19 | end 20 | 21 | end 22 | 23 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source :gemcutter 2 | 3 | gem "activerecord", "~> 3.0" 4 | 5 | group :development do 6 | gem "rake" 7 | gem "jeweler", ">= 1.4.0" 8 | end 9 | 10 | group :test do 11 | # Database Adapters 12 | platforms :ruby do 13 | gem "mysql", "~> 2.8.1" 14 | gem "mysql2", "~> 0.3.0" 15 | gem "pg", "~> 0.9" 16 | gem "sqlite3-ruby", "~> 1.3.1" 17 | gem "seamless_database_pool", "~> 1.0.11" 18 | end 19 | 20 | platforms :jruby do 21 | gem "jdbc-mysql" 22 | gem "activerecord-jdbcmysql-adapter" 23 | end 24 | 25 | # Support libs 26 | gem "factory_girl", "~> 1.3.3" 27 | gem "delorean", "~> 0.2.0" 28 | 29 | # Debugging 30 | platforms :mri_18 do 31 | gem "ruby-debug", "= 0.10.4" 32 | end 33 | 34 | platforms :jruby do 35 | gem "ruby-debug-base", "= 0.10.4" 36 | gem "ruby-debug", "= 0.10.4" 37 | end 38 | 39 | platforms :mri_19 do 40 | gem "debugger" 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /test/database.yml.sample: -------------------------------------------------------------------------------- 1 | common: &common 2 | username: root 3 | password: 4 | encoding: utf8 5 | host: localhost 6 | database: activerecord_import_test 7 | 8 | mysql: 9 | <<: *common 10 | adapter: mysql 11 | 12 | mysql2: 13 | <<: *common 14 | adapter: mysql2 15 | 16 | mysqlspatial: 17 | <<: *mysql 18 | 19 | mysqlspatial2: 20 | <<: *mysql2 21 | 22 | seamless_database_pool: 23 | <<: *common 24 | adapter: seamless_database_pool 25 | pool_adapter: mysql2 26 | master: 27 | host: localhost 28 | 29 | postgresql: 30 | <<: *common 31 | username: postgres 32 | adapter: postgresql 33 | min_messages: warning 34 | 35 | postgis: 36 | <<: *postgresql 37 | 38 | oracle: 39 | <<: *common 40 | adapter: oracle 41 | min_messages: debug 42 | 43 | sqlite: 44 | adapter: sqlite 45 | dbfile: test.db 46 | 47 | sqlite3: 48 | adapter: sqlite3 49 | database: test.db 50 | 51 | spatialite: 52 | <<: *sqlite3 53 | -------------------------------------------------------------------------------- /test/support/generate.rb: -------------------------------------------------------------------------------- 1 | class ActiveSupport::TestCase 2 | def Build(*args) 3 | n = args.shift if args.first.is_a?(Numeric) 4 | factory = args.shift 5 | factory_girl_args = args.shift || {} 6 | 7 | if n 8 | Array.new.tap do |collection| 9 | n.times.each { collection << Factory.build(factory.to_s.singularize.to_sym, factory_girl_args) } 10 | end 11 | else 12 | Factory.build(factory.to_s.singularize.to_sym, factory_girl_args) 13 | end 14 | end 15 | 16 | def Generate(*args) 17 | n = args.shift if args.first.is_a?(Numeric) 18 | factory = args.shift 19 | factory_girl_args = args.shift || {} 20 | 21 | if n 22 | Array.new.tap do |collection| 23 | n.times.each { collection << Factory.create(factory.to_s.singularize.to_sym, factory_girl_args) } 24 | end 25 | else 26 | Factory.create(factory.to_s.singularize.to_sym, factory_girl_args) 27 | end 28 | end 29 | end -------------------------------------------------------------------------------- /lib/activerecord-import/base.rb: -------------------------------------------------------------------------------- 1 | require "pathname" 2 | require "active_record" 3 | require "active_record/version" 4 | 5 | module ActiveRecord::Import 6 | AdapterPath = File.join File.expand_path(File.dirname(__FILE__)), "/active_record/adapters" 7 | 8 | def self.base_adapter(adapter) 9 | case adapter 10 | when 'mysqlspatial' then 'mysql' 11 | when 'mysql2spatial' then 'mysql2' 12 | when 'spatialite' then 'sqlite3' 13 | when 'postgis' then 'postgresql' 14 | else adapter 15 | end 16 | end 17 | 18 | # Loads the import functionality for a specific database adapter 19 | def self.require_adapter(adapter) 20 | require File.join(AdapterPath,"/abstract_adapter") 21 | require File.join(AdapterPath,"/#{base_adapter(adapter)}_adapter") 22 | end 23 | 24 | # Loads the import functionality for the passed in ActiveRecord connection 25 | def self.load_from_connection_pool(connection_pool) 26 | require_adapter connection_pool.spec.config[:adapter] 27 | end 28 | end 29 | 30 | 31 | this_dir = Pathname.new File.dirname(__FILE__) 32 | require this_dir.join("import").to_s 33 | require this_dir.join("active_record/adapters/abstract_adapter").to_s 34 | require this_dir.join("synchronize").to_s -------------------------------------------------------------------------------- /test/synchronize_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | 3 | describe ".synchronize" do 4 | let(:topics){ Generate(3, :topics) } 5 | let(:titles){ %w(one two three) } 6 | 7 | setup do 8 | # update records outside of ActiveRecord knowing about it 9 | Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[0]}_haha' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" ) 10 | Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[1]}_haha' WHERE id=#{topics[1].id}", "Updating record 2 without ActiveRecord" ) 11 | Topic.connection.execute( "UPDATE #{Topic.table_name} SET title='#{titles[2]}_haha' WHERE id=#{topics[2].id}", "Updating record 3 without ActiveRecord" ) 12 | end 13 | 14 | it "reloads data for the specified records" do 15 | Book.synchronize topics 16 | 17 | actual_titles = topics.map(&:title) 18 | assert_equal "#{titles[0]}_haha", actual_titles[0], "the first record was not correctly updated" 19 | assert_equal "#{titles[1]}_haha", actual_titles[1], "the second record was not correctly updated" 20 | assert_equal "#{titles[2]}_haha", actual_titles[2], "the third record was not correctly updated" 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /benchmarks/README: -------------------------------------------------------------------------------- 1 | To run the benchmarks, from within the benchmarks run: 2 | ruby benchmark.rb [options] 3 | 4 | The following options are supported: 5 | --adapter [String] The database adapter to use. IE: mysql, postgresql, oracle 6 | 7 | --do-not-delete By default all records in the benchmark tables will be deleted at the end of the benchmark. This flag indicates not to delete the benchmark data. 8 | --num [Integer] The number of objects to benchmark. (Required!) 9 | --table-type [String] The table type to test. This can be used multiple times. By default it is all table types. 10 | --to-csv [String] Print results in a CSV file format 11 | --to-html [String] Print results in HTML format (String filename must be supplied) 12 | 13 | See "ruby benchmark.rb -h" for the complete listing of options. 14 | 15 | EXAMPLES 16 | -------- 17 | To output to html format: 18 | ruby benchmark.rb --adapter=mysql --to-html=results.html 19 | 20 | To output to csv format: 21 | ruby benchmark.rb --adapter=mysql --to-csv=results.csv 22 | 23 | LIMITATIONS 24 | ----------- 25 | Currently MySQL is the only supported adapter to benchmark. 26 | 27 | AUTHOR 28 | ------ 29 | Zach Dennis 30 | zach.dennis@gmail.com 31 | http://www.continuousthinking.com 32 | 33 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require 'pathname' 2 | test_dir = Pathname.new File.dirname(__FILE__) 3 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 4 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 5 | 6 | require "fileutils" 7 | require "rubygems" 8 | 9 | ENV["RAILS_ENV"] = "test" 10 | 11 | require "bundler" 12 | Bundler.setup 13 | 14 | require "logger" 15 | require 'test/unit' 16 | require "active_record" 17 | require "active_record/fixtures" 18 | require "active_support/test_case" 19 | 20 | require "delorean" 21 | require "ruby-debug" 22 | 23 | adapter = ENV["ARE_DB"] || "sqlite3" 24 | 25 | FileUtils.mkdir_p 'log' 26 | ActiveRecord::Base.logger = Logger.new("log/test.log") 27 | ActiveRecord::Base.logger.level = Logger::DEBUG 28 | ActiveRecord::Base.configurations["test"] = YAML.load(test_dir.join("database.yml").open)[adapter] 29 | 30 | require "activerecord-import" 31 | ActiveRecord::Base.establish_connection "test" 32 | 33 | ActiveSupport::Notifications.subscribe(/active_record.sql/) do |event, _, _, _, hsh| 34 | ActiveRecord::Base.logger.info hsh[:sql] 35 | end 36 | 37 | require "factory_girl" 38 | Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each{ |file| require file } 39 | 40 | # Load base/generic schema 41 | require test_dir.join("schema/version") 42 | require test_dir.join("schema/generic_schema") 43 | adapter_schema = test_dir.join("schema/#{adapter}_schema.rb") 44 | require adapter_schema if File.exists?(adapter_schema) 45 | 46 | Dir[File.dirname(__FILE__) + "/models/*.rb"].each{ |file| require file } 47 | 48 | # Prevent this deprecation warning from breaking the tests. 49 | module Rake::DeprecatedObjectDSL 50 | remove_method :import 51 | end 52 | -------------------------------------------------------------------------------- /test/support/active_support/test_case_extensions.rb: -------------------------------------------------------------------------------- 1 | class ActiveSupport::TestCase 2 | include ActiveRecord::TestFixtures 3 | self.use_transactional_fixtures = true 4 | 5 | class << self 6 | def assertion(name, &block) 7 | mc = class << self ; self ; end 8 | mc.class_eval do 9 | define_method(name) do 10 | it(name, &block) 11 | end 12 | end 13 | end 14 | 15 | def asssertion_group(name, &block) 16 | mc = class << self ; self ; end 17 | mc.class_eval do 18 | define_method(name, &block) 19 | end 20 | end 21 | 22 | def macro(name, &block) 23 | class_eval do 24 | define_method(name, &block) 25 | end 26 | end 27 | 28 | def describe(description, toplevel=nil, &blk) 29 | text = toplevel ? description : "#{name} #{description}" 30 | klass = Class.new(self) 31 | 32 | klass.class_eval <<-RUBY_EVAL 33 | def self.name 34 | "#{text}" 35 | end 36 | RUBY_EVAL 37 | 38 | # do not inherit test methods from the superclass 39 | klass.class_eval do 40 | instance_methods.grep(/^test.+/) do |method| 41 | undef_method method 42 | end 43 | end 44 | 45 | klass.instance_eval &blk 46 | end 47 | alias_method :context, :describe 48 | 49 | def let(name, &blk) 50 | values = {} 51 | define_method(name) do 52 | return values[name] if values.has_key?(name) 53 | values[name] = instance_eval(&blk) 54 | end 55 | end 56 | 57 | def it(description, &blk) 58 | define_method("test: #{name} #{description}", &blk) 59 | end 60 | end 61 | 62 | end 63 | 64 | def describe(description, &blk) 65 | ActiveSupport::TestCase.describe(description, true, &blk) 66 | end 67 | 68 | -------------------------------------------------------------------------------- /benchmarks/lib/output_to_html.rb: -------------------------------------------------------------------------------- 1 | require 'erb' 2 | 3 | module OutputToHTML 4 | 5 | TEMPLATE_HEADER =<<"EOT" 6 |
7 | All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed 8 | before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against 9 | the very first column (which is always the default ActiveRecord::Base.create method. 10 |
11 | EOT 12 | 13 | TEMPLATE =<<"EOT" 14 | 28 | 29 | 30 | <% columns.each do |col| %> 31 | 32 | <% end %> 33 | 34 | 35 | <% times.each do |time| %> 36 | 37 | <% end %> 38 | 39 | 40 |
<%= col %>
<%= time %>
 
41 | EOT 42 | 43 | def self.output_results( filename, results ) 44 | html = '' 45 | results.each do |result_set| 46 | columns, times = [], [] 47 | result_set.each do |result| 48 | columns << result.description 49 | if result.failed 50 | times << "failed" 51 | else 52 | time = result.tms.real.round_to( 3 ) 53 | speedup = ( result_set.first.tms.real / result.tms.real ).round 54 | 55 | if result == result_set.first 56 | times << "#{time}" 57 | else 58 | times << "#{time} (#{speedup}x speedup)" 59 | end 60 | end 61 | end 62 | 63 | template = ERB.new( TEMPLATE, 0, "%<>") 64 | html << template.result( binding ) 65 | end 66 | 67 | File.open( filename, 'w' ){ |file| file.write( TEMPLATE_HEADER + html ) } 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/mysql_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::MysqlAdapter 2 | include ActiveRecord::Import::ImportSupport 3 | include ActiveRecord::Import::OnDuplicateKeyUpdateSupport 4 | 5 | # Returns the maximum number of bytes that the server will allow 6 | # in a single packet 7 | def max_allowed_packet # :nodoc: 8 | @max_allowed_packet ||= begin 9 | result = execute( "SHOW VARIABLES like 'max_allowed_packet';" ) 10 | # original Mysql gem responds to #fetch_row while Mysql2 responds to #first 11 | val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1] 12 | val.to_i 13 | end 14 | end 15 | 16 | # Returns a generated ON DUPLICATE KEY UPDATE statement given the passed 17 | # in +args+. 18 | def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc: 19 | sql = ' ON DUPLICATE KEY UPDATE ' 20 | arg = args.first 21 | if arg.is_a?( Array ) 22 | sql << sql_for_on_duplicate_key_update_as_array( table_name, arg ) 23 | elsif arg.is_a?( Hash ) 24 | sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg ) 25 | elsif arg.is_a?( String ) 26 | sql << arg 27 | else 28 | raise ArgumentError.new( "Expected Array or Hash" ) 29 | end 30 | sql 31 | end 32 | 33 | def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc: 34 | results = arr.map do |column| 35 | qc = quote_column_name( column ) 36 | "#{table_name}.#{qc}=VALUES(#{qc})" 37 | end 38 | results.join( ',' ) 39 | end 40 | 41 | def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc: 42 | sql = ' ON DUPLICATE KEY UPDATE ' 43 | results = hsh.map do |column1, column2| 44 | qc1 = quote_column_name( column1 ) 45 | qc2 = quote_column_name( column2 ) 46 | "#{table_name}.#{qc1}=VALUES( #{qc2} )" 47 | end 48 | results.join( ',') 49 | end 50 | 51 | #return true if the statement is a duplicate key record error 52 | def duplicate_key_update_error?(exception)# :nodoc: 53 | exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry') 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /benchmarks/benchmark.rb: -------------------------------------------------------------------------------- 1 | require "pathname" 2 | this_dir = Pathname.new File.dirname(__FILE__) 3 | require this_dir.join('boot') 4 | 5 | # Parse the options passed in via the command line 6 | options = BenchmarkOptionParser.parse( ARGV ) 7 | 8 | # The support directory where we use to load our connections and models for the 9 | # benchmarks. 10 | SUPPORT_DIR = this_dir.join('../test') 11 | 12 | # Load the database adapter 13 | adapter = options.adapter 14 | 15 | # load the library 16 | LIB_DIR = this_dir.join("../lib") 17 | require LIB_DIR.join("activerecord-import/#{adapter}") 18 | 19 | ActiveRecord::Base.logger = Logger.new("log/test.log") 20 | ActiveRecord::Base.logger.level = Logger::DEBUG 21 | ActiveRecord::Base.configurations["test"] = YAML.load(SUPPORT_DIR.join("database.yml").open)[adapter] 22 | ActiveRecord::Base.establish_connection "test" 23 | 24 | ActiveSupport::Notifications.subscribe(/active_record.sql/) do |event, _, _, _, hsh| 25 | ActiveRecord::Base.logger.info hsh[:sql] 26 | end 27 | 28 | adapter_schema = SUPPORT_DIR.join("schema/#{adapter}_schema.rb") 29 | require adapter_schema if File.exists?(adapter_schema) 30 | Dir[this_dir.join("models/*.rb")].each{ |file| require file } 31 | 32 | # Load databse specific benchmarks 33 | require File.join( File.dirname( __FILE__ ), 'lib', "#{adapter}_benchmark" ) 34 | 35 | # TODO implement method/table-type selection 36 | table_types = nil 37 | if options.benchmark_all_types 38 | table_types = [ "all" ] 39 | else 40 | table_types = options.table_types.keys 41 | end 42 | puts 43 | 44 | letter = options.adapter[0].chr 45 | clazz_str = letter.upcase + options.adapter[1..-1].downcase 46 | clazz = Object.const_get( clazz_str + "Benchmark" ) 47 | 48 | benchmarks = [] 49 | options.number_of_objects.each do |num| 50 | benchmarks << (benchmark = clazz.new) 51 | benchmark.send( "benchmark", table_types, num ) 52 | end 53 | 54 | options.outputs.each do |output| 55 | format = output.format.downcase 56 | output_module = Object.const_get( "OutputTo#{format.upcase}" ) 57 | benchmarks.each do |benchmark| 58 | output_module.output_results( output.filename, benchmark.results ) 59 | end 60 | end 61 | 62 | puts 63 | puts "Done with benchmark!" 64 | 65 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | activemodel (3.2.7) 5 | activesupport (= 3.2.7) 6 | builder (~> 3.0.0) 7 | activerecord (3.2.7) 8 | activemodel (= 3.2.7) 9 | activesupport (= 3.2.7) 10 | arel (~> 3.0.2) 11 | tzinfo (~> 0.3.29) 12 | activerecord-jdbc-adapter (1.2.2) 13 | activerecord-jdbcmysql-adapter (1.2.2) 14 | activerecord-jdbc-adapter (~> 1.2.2) 15 | jdbc-mysql (~> 5.1.0) 16 | activesupport (3.2.7) 17 | i18n (~> 0.6) 18 | multi_json (~> 1.0) 19 | arel (3.0.2) 20 | builder (3.0.0) 21 | chronic (0.7.0) 22 | columnize (0.3.6) 23 | debugger (1.2.0) 24 | columnize (>= 0.3.1) 25 | debugger-linecache (~> 1.1.1) 26 | debugger-ruby_core_source (~> 1.1.3) 27 | debugger-linecache (1.1.2) 28 | debugger-ruby_core_source (>= 1.1.1) 29 | debugger-ruby_core_source (1.1.3) 30 | delorean (0.2.1) 31 | chronic 32 | factory_girl (1.3.3) 33 | git (1.2.5) 34 | i18n (0.6.0) 35 | jdbc-mysql (5.1.13) 36 | jeweler (1.8.4) 37 | bundler (~> 1.0) 38 | git (>= 1.2.5) 39 | rake 40 | rdoc 41 | json (1.7.4) 42 | json (1.7.4-java) 43 | linecache (0.46) 44 | rbx-require-relative (> 0.0.4) 45 | multi_json (1.3.6) 46 | mysql (2.8.1) 47 | mysql2 (0.3.11) 48 | pg (0.14.0) 49 | rake (0.9.2.2) 50 | rbx-require-relative (0.0.9) 51 | rdoc (3.12) 52 | json (~> 1.4) 53 | ruby-debug (0.10.4) 54 | columnize (>= 0.1) 55 | ruby-debug-base (~> 0.10.4.0) 56 | ruby-debug-base (0.10.4) 57 | linecache (>= 0.3) 58 | ruby-debug-base (0.10.4-java) 59 | seamless_database_pool (1.0.11) 60 | activerecord (>= 2.2.2) 61 | sqlite3 (1.3.6) 62 | sqlite3-ruby (1.3.3) 63 | sqlite3 (>= 1.3.3) 64 | tzinfo (0.3.33) 65 | 66 | PLATFORMS 67 | java 68 | ruby 69 | 70 | DEPENDENCIES 71 | activerecord (~> 3.0) 72 | activerecord-jdbcmysql-adapter 73 | debugger 74 | delorean (~> 0.2.0) 75 | factory_girl (~> 1.3.3) 76 | jdbc-mysql 77 | jeweler (>= 1.4.0) 78 | mysql (~> 2.8.1) 79 | mysql2 (~> 0.3.0) 80 | pg (~> 0.9) 81 | rake 82 | ruby-debug (= 0.10.4) 83 | ruby-debug-base (= 0.10.4) 84 | seamless_database_pool (~> 1.0.11) 85 | sqlite3-ruby (~> 1.3.1) 86 | -------------------------------------------------------------------------------- /test/support/mysql/assertions.rb: -------------------------------------------------------------------------------- 1 | class ActiveSupport::TestCase 2 | module MySQLAssertions 3 | def self.extended(klass) 4 | klass.instance_eval do 5 | assertion(:should_not_update_created_at_on_timestamp_columns) do 6 | Delorean.time_travel_to("5 minutes from now") do 7 | perform_import 8 | assert_equal @topic.created_at.to_i, updated_topic.created_at.to_i 9 | assert_equal @topic.created_on.to_i, updated_topic.created_on.to_i 10 | end 11 | end 12 | 13 | assertion(:should_update_updated_at_on_timestamp_columns) do 14 | time = Chronic.parse("5 minutes from now") 15 | Delorean.time_travel_to(time) do 16 | perform_import 17 | assert_equal time.to_i, updated_topic.updated_at.to_i 18 | assert_equal time.to_i, updated_topic.updated_on.to_i 19 | end 20 | end 21 | 22 | assertion(:should_not_update_timestamps) do 23 | Delorean.time_travel_to("5 minutes from now") do 24 | perform_import :timestamps => false 25 | assert_equal @topic.created_at.to_i, updated_topic.created_at.to_i 26 | assert_equal @topic.created_on.to_i, updated_topic.created_on.to_i 27 | assert_equal @topic.updated_at.to_i, updated_topic.updated_at.to_i 28 | assert_equal @topic.updated_on.to_i, updated_topic.updated_on.to_i 29 | end 30 | end 31 | 32 | assertion(:should_not_update_fields_not_mentioned) do 33 | assert_equal "John Doe", updated_topic.author_name 34 | end 35 | 36 | assertion(:should_update_fields_mentioned) do 37 | perform_import 38 | assert_equal "Book - 2nd Edition", updated_topic.title 39 | assert_equal "johndoe@example.com", updated_topic.author_email_address 40 | end 41 | 42 | assertion(:should_update_fields_mentioned_with_hash_mappings) do 43 | perform_import 44 | assert_equal "johndoe@example.com", updated_topic.title 45 | assert_equal "Book - 2nd Edition", updated_topic.author_email_address 46 | end 47 | 48 | assertion(:should_update_foreign_keys) do 49 | perform_import 50 | assert_equal 57, updated_topic.parent_id 51 | end 52 | end 53 | end 54 | end 55 | end -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler" 2 | Bundler.setup 3 | 4 | require 'rake' 5 | require 'rake/testtask' 6 | 7 | begin 8 | require 'jeweler' 9 | Jeweler::Tasks.new do |gem| 10 | gem.name = "activerecord-import" 11 | gem.summary = %Q{Bulk-loading extension for ActiveRecord} 12 | gem.description = %Q{Extraction of the ActiveRecord::Base#import functionality from ar-extensions for Rails 3 and beyond} 13 | gem.email = "zach.dennis@gmail.com" 14 | gem.homepage = "http://github.com/zdennis/activerecord-import" 15 | gem.authors = ["Zach Dennis"] 16 | gem.files = FileList["VERSION", "Rakefile", "README*", "lib/**/*"] 17 | 18 | bundler = Bundler.load 19 | bundler.dependencies_for(:default).each do |dependency| 20 | gem.add_dependency dependency.name, *dependency.requirements_list 21 | end 22 | 23 | # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings 24 | end 25 | Jeweler::GemcutterTasks.new 26 | rescue LoadError 27 | puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler" 28 | end 29 | 30 | namespace :display do 31 | task :notice do 32 | puts 33 | puts "To run tests you must supply the adapter, see rake -T for more information." 34 | puts 35 | end 36 | end 37 | task :default => ["display:notice"] 38 | 39 | ADAPTERS = %w(mysql mysql2 jdbcmysql postgresql sqlite3 seamless_database_pool mysqlspatial mysql2spatial spatialite postgis) 40 | ADAPTERS.each do |adapter| 41 | namespace :test do 42 | desc "Runs #{adapter} database tests." 43 | Rake::TestTask.new(adapter) do |t| 44 | # FactoryGirl has an issue with warnings, so turn off, so noisy 45 | # t.warning = true 46 | t.test_files = FileList["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/active_record/*_test.rb", "test/#{adapter}/**/*_test.rb"] 47 | end 48 | task adapter 49 | end 50 | end 51 | 52 | begin 53 | require 'rcov/rcovtask' 54 | adapter = ENV['ARE_DB'] 55 | Rcov::RcovTask.new do |test| 56 | test.libs << 'test' 57 | test.pattern = ["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/#{adapter}/**/*_test.rb"] 58 | test.verbose = true 59 | end 60 | rescue LoadError 61 | task :rcov do 62 | abort "RCov is not available. In order to run rcov, you must: sudo gem install rcov" 63 | end 64 | end 65 | 66 | require 'rdoc/task' 67 | Rake::RDocTask.new do |rdoc| 68 | version = File.exist?('VERSION') ? File.read('VERSION') : "" 69 | 70 | rdoc.rdoc_dir = 'rdoc' 71 | rdoc.title = "activerecord-import #{version}" 72 | rdoc.rdoc_files.include('README*') 73 | rdoc.rdoc_files.include('lib/**/*.rb') 74 | end 75 | -------------------------------------------------------------------------------- /lib/activerecord-import/synchronize.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord # :nodoc: 2 | class Base # :nodoc: 3 | 4 | # Synchronizes the passed in ActiveRecord instances with data 5 | # from the database. This is like calling reload on an individual 6 | # ActiveRecord instance but it is intended for use on multiple instances. 7 | # 8 | # This uses one query for all instance updates and then updates existing 9 | # instances rather sending one query for each instance 10 | # 11 | # == Examples 12 | # # Synchronizing existing models by matching on the primary key field 13 | # posts = Post.find_by_author("Zach") 14 | # <.. out of system changes occur to change author name from Zach to Zachary..> 15 | # Post.synchronize posts 16 | # posts.first.author # => "Zachary" instead of Zach 17 | # 18 | # # Synchronizing using custom key fields 19 | # posts = Post.find_by_author("Zach") 20 | # <.. out of system changes occur to change the address of author 'Zach' to 1245 Foo Ln ..> 21 | # Post.synchronize posts, [:name] # queries on the :name column and not the :id column 22 | # posts.first.address # => "1245 Foo Ln" instead of whatever it was 23 | # 24 | def self.synchronize(instances, keys=[self.primary_key]) 25 | return if instances.empty? 26 | 27 | conditions = {} 28 | order = "" 29 | 30 | key_values = keys.map { |key| instances.map(&"#{key}".to_sym) } 31 | keys.zip(key_values).each { |key, values| conditions[key] = values } 32 | order = keys.map{ |key| "#{key} ASC" }.join(",") 33 | 34 | klass = instances.first.class 35 | 36 | fresh_instances = klass.find( :all, :conditions=>conditions, :order=>order ) 37 | instances.each do |instance| 38 | matched_instance = fresh_instances.detect do |fresh_instance| 39 | keys.all?{ |key| fresh_instance.send(key) == instance.send(key) } 40 | end 41 | 42 | if matched_instance 43 | instance.clear_aggregation_cache 44 | instance.clear_association_cache 45 | instance.instance_variable_set '@attributes', matched_instance.attributes 46 | # Since the instance now accurately reflects the record in 47 | # the database, ensure that instance.persisted? is true. 48 | instance.instance_variable_set '@new_record', false 49 | instance.instance_variable_set '@destroyed', false 50 | end 51 | end 52 | end 53 | 54 | # See ActiveRecord::ConnectionAdapters::AbstractAdapter.synchronize 55 | def synchronize(instances, key=[ActiveRecord::Base.primary_key]) 56 | self.class.synchronize(instances, key) 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /test/active_record/connection_adapter_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path(File.dirname(__FILE__) + '/../test_helper') 2 | 3 | describe "ActiveRecord::ConnectionAdapter::AbstractAdapter" do 4 | context "#get_insert_value_sets - computing insert value sets" do 5 | let(:adapter){ ActiveRecord::ConnectionAdapters::AbstractAdapter } 6 | let(:base_sql){ "INSERT INTO atable (a,b,c)" } 7 | let(:values){ [ "(1,2,3)", "(2,3,4)", "(3,4,5)" ] } 8 | 9 | context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do 10 | it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do 11 | value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 33 12 | assert_equal 3, value_sets.size 13 | end 14 | end 15 | 16 | context "when the max allowed bytes is 40 and the base SQL is 26 bytes" do 17 | it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do 18 | value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 40 19 | assert_equal 3, value_sets.size 20 | end 21 | end 22 | 23 | context "when the max allowed bytes is 41 and the base SQL is 26 bytes" do 24 | it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do 25 | value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 41 26 | assert_equal 2, value_sets.size 27 | end 28 | end 29 | 30 | context "when the max allowed bytes is 48 and the base SQL is 26 bytes" do 31 | it "should return 2 value sets when given 2 value sets of 7 bytes a piece" do 32 | value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 48 33 | assert_equal 2, value_sets.size 34 | end 35 | end 36 | 37 | context "when the max allowed bytes is 49 and the base SQL is 26 bytes" do 38 | it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do 39 | value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 49 40 | assert_equal 1, value_sets.size 41 | end 42 | end 43 | 44 | context "when the max allowed bytes is 999999 and the base SQL is 26 bytes" do 45 | it "should return 1 value sets when given 1 value sets of 7 bytes a piece" do 46 | value_sets = adapter.get_insert_value_sets values, base_sql.size, max_allowed_bytes = 999999 47 | assert_equal 1, value_sets.size 48 | end 49 | end 50 | end 51 | 52 | end 53 | 54 | describe "ActiveRecord::Import DB-specific adapter class" do 55 | context "when ActiveRecord::Import is in use" do 56 | it "should appear in the AR connection adapter class's ancestors" do 57 | connection = ActiveRecord::Base.connection 58 | import_class_name = 'ActiveRecord::Import::' + connection.class.name.demodulize 59 | assert_includes connection.class.ancestors, import_class_name.constantize 60 | end 61 | end 62 | end -------------------------------------------------------------------------------- /activerecord-import.gemspec: -------------------------------------------------------------------------------- 1 | # Generated by jeweler 2 | # DO NOT EDIT THIS FILE DIRECTLY 3 | # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec' 4 | # -*- encoding: utf-8 -*- 5 | 6 | Gem::Specification.new do |s| 7 | s.name = "activerecord-import" 8 | s.version = "0.2.10" 9 | 10 | s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= 11 | s.authors = ["Zach Dennis"] 12 | s.date = "2012-08-30" 13 | s.description = "Extraction of the ActiveRecord::Base#import functionality from ar-extensions for Rails 3 and beyond" 14 | s.email = "zach.dennis@gmail.com" 15 | s.extra_rdoc_files = [ 16 | "README.markdown" 17 | ] 18 | s.files = [ 19 | "README.markdown", 20 | "Rakefile", 21 | "VERSION", 22 | "lib/activerecord-import.rb", 23 | "lib/activerecord-import/active_record/adapters/abstract_adapter.rb", 24 | "lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb", 25 | "lib/activerecord-import/active_record/adapters/mysql2_adapter.rb", 26 | "lib/activerecord-import/active_record/adapters/mysql_adapter.rb", 27 | "lib/activerecord-import/active_record/adapters/postgresql_adapter.rb", 28 | "lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb", 29 | "lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb", 30 | "lib/activerecord-import/adapters/abstract_adapter.rb", 31 | "lib/activerecord-import/adapters/mysql_adapter.rb", 32 | "lib/activerecord-import/adapters/postgresql_adapter.rb", 33 | "lib/activerecord-import/adapters/sqlite3_adapter.rb", 34 | "lib/activerecord-import/base.rb", 35 | "lib/activerecord-import/import.rb", 36 | "lib/activerecord-import/mysql.rb", 37 | "lib/activerecord-import/mysql2.rb", 38 | "lib/activerecord-import/postgresql.rb", 39 | "lib/activerecord-import/sqlite3.rb", 40 | "lib/activerecord-import/synchronize.rb" 41 | ] 42 | s.homepage = "http://github.com/zdennis/activerecord-import" 43 | s.require_paths = ["lib"] 44 | s.rubygems_version = "1.8.24" 45 | s.summary = "Bulk-loading extension for ActiveRecord" 46 | 47 | if s.respond_to? :specification_version then 48 | s.specification_version = 3 49 | 50 | if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then 51 | s.add_runtime_dependency(%q, ["~> 3.0"]) 52 | s.add_development_dependency(%q, [">= 0"]) 53 | s.add_development_dependency(%q, [">= 1.4.0"]) 54 | s.add_runtime_dependency(%q, ["~> 3.0"]) 55 | else 56 | s.add_dependency(%q, ["~> 3.0"]) 57 | s.add_dependency(%q, [">= 0"]) 58 | s.add_dependency(%q, [">= 1.4.0"]) 59 | s.add_dependency(%q, ["~> 3.0"]) 60 | end 61 | else 62 | s.add_dependency(%q, ["~> 3.0"]) 63 | s.add_dependency(%q, [">= 0"]) 64 | s.add_dependency(%q, [">= 1.4.0"]) 65 | s.add_dependency(%q, ["~> 3.0"]) 66 | end 67 | end 68 | 69 | -------------------------------------------------------------------------------- /test/schema/generic_schema.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Schema.define do 2 | 3 | create_table :schema_info, :force=>true do |t| 4 | t.column :version, :integer, :unique=>true 5 | end 6 | SchemaInfo.create :version=>SchemaInfo::VERSION 7 | 8 | create_table :group, :force => true do |t| 9 | t.column :order, :string 10 | t.timestamps 11 | end 12 | 13 | create_table :topics, :force=>true do |t| 14 | t.column :title, :string, :null => false 15 | t.column :author_name, :string 16 | t.column :author_email_address, :string 17 | t.column :written_on, :datetime 18 | t.column :bonus_time, :time 19 | t.column :last_read, :datetime 20 | t.column :content, :text 21 | t.column :approved, :boolean, :default=>'1' 22 | t.column :replies_count, :integer 23 | t.column :parent_id, :integer 24 | t.column :type, :string 25 | t.column :created_at, :datetime 26 | t.column :created_on, :datetime 27 | t.column :updated_at, :datetime 28 | t.column :updated_on, :datetime 29 | end 30 | 31 | create_table :projects, :force=>true do |t| 32 | t.column :name, :string 33 | t.column :type, :string 34 | end 35 | 36 | create_table :developers, :force=>true do |t| 37 | t.column :name, :string 38 | t.column :salary, :integer, :default=>'70000' 39 | t.column :created_at, :datetime 40 | t.column :team_id, :integer 41 | t.column :updated_at, :datetime 42 | end 43 | 44 | create_table :addresses, :force=>true do |t| 45 | t.column :address, :string 46 | t.column :city, :string 47 | t.column :state, :string 48 | t.column :zip, :string 49 | t.column :developer_id, :integer 50 | end 51 | 52 | create_table :teams, :force=>true do |t| 53 | t.column :name, :string 54 | end 55 | 56 | create_table :books, :force=>true do |t| 57 | t.column :title, :string, :null=>false 58 | t.column :publisher, :string, :null=>false, :default => 'Default Publisher' 59 | t.column :author_name, :string, :null=>false 60 | t.column :created_at, :datetime 61 | t.column :created_on, :datetime 62 | t.column :updated_at, :datetime 63 | t.column :updated_on, :datetime 64 | t.column :publish_date, :date 65 | t.column :topic_id, :integer 66 | t.column :for_sale, :boolean, :default => true 67 | end 68 | 69 | create_table :languages, :force=>true do |t| 70 | t.column :name, :string 71 | t.column :developer_id, :integer 72 | end 73 | 74 | create_table :shopping_carts, :force=>true do |t| 75 | t.column :name, :string, :null => true 76 | t.column :created_at, :datetime 77 | t.column :updated_at, :datetime 78 | end 79 | 80 | create_table :cart_items, :force => true do |t| 81 | t.column :shopping_cart_id, :string, :null => false 82 | t.column :book_id, :string, :null => false 83 | t.column :copies, :integer, :default => 1 84 | t.column :created_at, :datetime 85 | t.column :updated_at, :datetime 86 | end 87 | 88 | add_index :cart_items, [:shopping_cart_id, :book_id], :unique => true, :name => 'uk_shopping_cart_books' 89 | 90 | create_table :animals, :force => true do |t| 91 | t.column :name, :string, :null => false 92 | t.column :size, :string, :default => nil 93 | t.column :created_at, :datetime 94 | t.column :updated_at, :datetime 95 | end 96 | 97 | add_index :animals, [:name], :unique => true, :name => 'uk_animals' 98 | 99 | create_table :widgets, :id => false, :force => true do |t| 100 | t.integer :w_id 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /benchmarks/lib/cli_parser.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | require 'ostruct' 3 | 4 | # 5 | # == PARAMETERS 6 | # * a - database adapter. ie: mysql, postgresql, oracle, etc. 7 | # * n - number of objects to test with. ie: 1, 100, 1000, etc. 8 | # * t - the table types to test. ie: myisam, innodb, memory, temporary, etc. 9 | # 10 | module BenchmarkOptionParser 11 | BANNER = "Usage: ruby #{$0} [options]\nSee ruby #{$0} -h for more options." 12 | 13 | def self.print_banner 14 | puts BANNER 15 | end 16 | 17 | def self.print_banner! 18 | print_banner 19 | exit 20 | end 21 | 22 | def self.print_options( options ) 23 | puts "Benchmarking the following options:" 24 | puts " Database adapter: #{options.adapter}" 25 | puts " Number of objects: #{options.number_of_objects}" 26 | puts " Table types:" 27 | print_valid_table_types( options, :prefix=>" " ) 28 | end 29 | 30 | # TODO IMPLEMENT THIS 31 | def self.print_valid_table_types( options, hsh={:prefix=>''} ) 32 | if options.table_types.keys.size > 0 33 | options.table_types.keys.sort.each{ |type| puts hsh[:prefix].to_s + type.to_s } 34 | else 35 | puts 'No table types defined.' 36 | end 37 | end 38 | 39 | def self.parse( args ) 40 | options = OpenStruct.new( 41 | :table_types => {}, 42 | :delete_on_finish => true, 43 | :number_of_objects => [], 44 | :outputs => [] ) 45 | 46 | opts = OptionParser.new do |opts| 47 | opts.banner = BANNER 48 | 49 | # parse the database adapter 50 | opts.on( "a", "--adapter [String]", 51 | "The database adapter to use. IE: mysql, postgresql, oracle" ) do |arg| 52 | options.adapter = arg 53 | end 54 | 55 | # parse do_not_delete flag 56 | opts.on( "d", "--do-not-delete", 57 | "By default all records in the benchmark tables will be deleted at the end of the benchmark. " + 58 | "This flag indicates not to delete the benchmark data." ) do |arg| 59 | options.delete_on_finish = false 60 | end 61 | 62 | # parse the number of row objects to test 63 | opts.on( "n", "--num [Integer]", 64 | "The number of objects to benchmark." ) do |arg| 65 | options.number_of_objects << arg.to_i 66 | end 67 | 68 | # parse the table types to test 69 | opts.on( "t", "--table-type [String]", 70 | "The table type to test. This can be used multiple times." ) do |arg| 71 | if arg =~ /^all$/ 72 | options.table_types['all'] = options.benchmark_all_types = true 73 | else 74 | options.table_types[arg] = true 75 | end 76 | end 77 | 78 | # print results in CSV format 79 | opts.on( "--to-csv [String]", "Print results in a CSV file format" ) do |filename| 80 | options.outputs << OpenStruct.new( :format=>'csv', :filename=>filename) 81 | end 82 | 83 | # print results in HTML format 84 | opts.on( "--to-html [String]", "Print results in HTML format" ) do |filename| 85 | options.outputs << OpenStruct.new( :format=>'html', :filename=>filename ) 86 | end 87 | end #end opt.parse! 88 | 89 | begin 90 | opts.parse!( args ) 91 | if options.table_types.size == 0 92 | options.table_types['all'] = options.benchmark_all_types = true 93 | end 94 | rescue Exception => ex 95 | print_banner! 96 | end 97 | 98 | print_options( options ) 99 | 100 | options 101 | end 102 | 103 | end 104 | -------------------------------------------------------------------------------- /lib/activerecord-import/adapters/abstract_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveRecord::Import::AbstractAdapter 2 | NO_MAX_PACKET = 0 3 | QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from. 4 | 5 | module ClassMethods 6 | def get_insert_value_sets( values, sql_size, max_bytes ) # :nodoc: 7 | value_sets = [] 8 | arr, current_arr_values_size, current_size = [], 0, 0 9 | values.each_with_index do |val,i| 10 | comma_bytes = arr.size 11 | sql_size_thus_far = sql_size + current_size + val.bytesize + comma_bytes 12 | if NO_MAX_PACKET == max_bytes or sql_size_thus_far <= max_bytes 13 | current_size += val.bytesize 14 | arr << val 15 | else 16 | value_sets << arr 17 | arr = [ val ] 18 | current_size = val.bytesize 19 | end 20 | 21 | # if we're on the last iteration push whatever we have in arr to value_sets 22 | value_sets << arr if i == (values.size-1) 23 | end 24 | [ *value_sets ] 25 | end 26 | end 27 | 28 | module InstanceMethods 29 | def next_value_for_sequence(sequence_name) 30 | %{#{sequence_name}.nextval} 31 | end 32 | 33 | # +sql+ can be a single string or an array. If it is an array all 34 | # elements that are in position >= 1 will be appended to the final SQL. 35 | def insert_many( sql, values, *args ) # :nodoc: 36 | # the number of inserts default 37 | number_of_inserts = 0 38 | 39 | base_sql,post_sql = if sql.is_a?( String ) 40 | [ sql, '' ] 41 | elsif sql.is_a?( Array ) 42 | [ sql.shift, sql.join( ' ' ) ] 43 | end 44 | 45 | sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size 46 | 47 | # the number of bytes the requested insert statement values will take up 48 | values_in_bytes = values.sum {|value| value.bytesize } 49 | 50 | # the number of bytes (commas) it will take to comma separate our values 51 | comma_separated_bytes = values.size-1 52 | 53 | # the total number of bytes required if this statement is one statement 54 | total_bytes = sql_size + values_in_bytes + comma_separated_bytes 55 | 56 | max = max_allowed_packet 57 | 58 | # if we can insert it all as one statement 59 | if NO_MAX_PACKET == max or total_bytes < max 60 | number_of_inserts += 1 61 | sql2insert = base_sql + values.join( ',' ) + post_sql 62 | insert( sql2insert, *args ) 63 | else 64 | value_sets = self.class.get_insert_value_sets( values, sql_size, max ) 65 | value_sets.each do |values| 66 | number_of_inserts += 1 67 | sql2insert = base_sql + values.join( ',' ) + post_sql 68 | insert( sql2insert, *args ) 69 | end 70 | end 71 | 72 | number_of_inserts 73 | end 74 | 75 | def pre_sql_statements(options) 76 | sql = [] 77 | sql << options[:pre_sql] if options[:pre_sql] 78 | sql << options[:command] if options[:command] 79 | sql << "IGNORE" if options[:ignore] 80 | 81 | #add keywords like IGNORE or DELAYED 82 | if options[:keywords].is_a?(Array) 83 | sql.concat(options[:keywords]) 84 | elsif options[:keywords] 85 | sql << options[:keywords].to_s 86 | end 87 | 88 | sql 89 | end 90 | 91 | # Synchronizes the passed in ActiveRecord instances with the records in 92 | # the database by calling +reload+ on each instance. 93 | def after_import_synchronize( instances ) 94 | instances.each { |e| e.reload } 95 | end 96 | 97 | # Returns an array of post SQL statements given the passed in options. 98 | def post_sql_statements( table_name, options ) # :nodoc: 99 | post_sql_statements = [] 100 | if options[:on_duplicate_key_update] 101 | post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] ) 102 | end 103 | 104 | #custom user post_sql 105 | post_sql_statements << options[:post_sql] if options[:post_sql] 106 | 107 | #with rollup 108 | post_sql_statements << rollup_sql if options[:rollup] 109 | 110 | post_sql_statements 111 | end 112 | 113 | # Returns the maximum number of bytes that the server will allow 114 | # in a single packet 115 | def max_allowed_packet 116 | NO_MAX_PACKET 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /benchmarks/lib/base.rb: -------------------------------------------------------------------------------- 1 | class BenchmarkBase 2 | 3 | attr_reader :results 4 | 5 | # The main benchmark method dispatcher. This dispatches the benchmarks 6 | # to actual benchmark_xxxx methods. 7 | # 8 | # == PARAMETERS 9 | # * table_types - an array of table types to benchmark 10 | # * num - the number of record insertions to test 11 | def benchmark( table_types, num ) 12 | array_of_cols_and_vals = build_array_of_cols_and_vals( num ) 13 | table_types.each do |table_type| 14 | self.send( "benchmark_#{table_type}", array_of_cols_and_vals ) 15 | end 16 | end 17 | 18 | # Returns an OpenStruct which contains two attritues, +description+ and +tms+ after performing an 19 | # actual benchmark. 20 | # 21 | # == PARAMETERS 22 | # * description - the description of the block that is getting benchmarked 23 | # * blk - the block of code to benchmark 24 | # 25 | # == RETURNS 26 | # An OpenStruct object with the following attributes: 27 | # * description - the description of the benchmark ran 28 | # * tms - a Benchmark::Tms containing the results of the benchmark 29 | def bm( description, &blk ) 30 | tms = nil 31 | puts "Benchmarking #{description}" 32 | 33 | Benchmark.bm { |x| tms = x.report { blk.call } } 34 | delete_all 35 | failed = false 36 | 37 | OpenStruct.new :description=>description, :tms=>tms, :failed=>failed 38 | end 39 | 40 | # Given a model class (ie: Topic), and an array of columns and value sets 41 | # this will perform all of the benchmarks necessary for this library. 42 | # 43 | # == PARAMETERS 44 | # * model_clazz - the model class to benchmark (ie: Topic) 45 | # * array_of_cols_and_vals - an array of column identifiers and value sets 46 | # 47 | # == RETURNS 48 | # returns true 49 | def bm_model( model_clazz, array_of_cols_and_vals ) 50 | puts 51 | puts "------ Benchmarking #{model_clazz.name} -------" 52 | 53 | cols,vals = array_of_cols_and_vals 54 | num_inserts = vals.size 55 | 56 | # add a new result group for this particular benchmark 57 | group = [] 58 | @results << group 59 | 60 | description = "#{model_clazz.name}.create (#{num_inserts} records)" 61 | group << bm( description ) { 62 | vals.each do |values| 63 | model_clazz.create create_hash_for_cols_and_vals( cols, values ) 64 | end } 65 | 66 | description = "#{model_clazz.name}.import(column, values) for #{num_inserts} records with validations" 67 | group << bm( description ) { model_clazz.import cols, vals, :validate=>true } 68 | 69 | description = "#{model_clazz.name}.import(columns, values) for #{num_inserts} records without validations" 70 | group << bm( description ) { model_clazz.import cols, vals, :validate=>false } 71 | 72 | models = [] 73 | array_of_attrs = [] 74 | 75 | vals.each do |arr| 76 | array_of_attrs << (attrs={}) 77 | arr.each_with_index { |value, i| attrs[cols[i]] = value } 78 | end 79 | array_of_attrs.each{ |attrs| models << model_clazz.new(attrs) } 80 | 81 | description = "#{model_clazz.name}.import(models) for #{num_inserts} records with validations" 82 | group << bm( description ) { model_clazz.import models, :validate=>true } 83 | 84 | description = "#{model_clazz.name}.import(models) for #{num_inserts} records without validations" 85 | group << bm( description ) { model_clazz.import models, :validate=>false } 86 | 87 | true 88 | end 89 | 90 | # Returns a two element array composing of an array of columns and an array of 91 | # value sets given the passed +num+. 92 | # 93 | # === What is a value set? 94 | # A value set is an array of arrays. Each child array represents an array of value sets 95 | # for a given row of data. 96 | # 97 | # For example, say we wanted to represent an insertion of two records: 98 | # column_names = [ 'id', 'name', 'description' ] 99 | # record1 = [ 1, 'John Doe', 'A plumber' ] 100 | # record2 = [ 2, 'John Smith', 'A painter' ] 101 | # value_set [ record1, record2 ] 102 | # 103 | # == PARAMETER 104 | # * num - the number of records to create 105 | def build_array_of_cols_and_vals( num ) 106 | cols = [ :my_name, :description ] 107 | value_sets = [] 108 | num.times { |i| value_sets << [ "My Name #{i}", "My Description #{i}" ] } 109 | [ cols, value_sets ] 110 | end 111 | 112 | # Returns a hash of column identifier to value mappings giving the passed in 113 | # value array. 114 | # 115 | # Example: 116 | # cols = [ 'id', 'name', 'description' ] 117 | # values = [ 1, 'John Doe', 'A plumber' ] 118 | # hsh = create_hash_for_cols_and_vals( cols, values ) 119 | # # hsh => { 'id'=>1, 'name'=>'John Doe', 'description'=>'A plumber' } 120 | def create_hash_for_cols_and_vals( cols, vals ) 121 | h = {} 122 | cols.zip( vals ){ |col,val| h[col] = val } 123 | h 124 | end 125 | 126 | # Deletes all records from all ActiveRecord subclasses 127 | def delete_all 128 | ActiveRecord::Base.send( :subclasses ).each do |subclass| 129 | subclass.delete_all if subclass.respond_to? :delete_all 130 | end 131 | end 132 | 133 | def initialize # :nodoc: 134 | @results = [] 135 | end 136 | 137 | end 138 | -------------------------------------------------------------------------------- /test/support/mysql/import_examples.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | def should_support_mysql_import_functionality 3 | 4 | describe "building insert value sets" do 5 | it "should properly build insert value set based on max packet allowed" do 6 | values = [ 7 | "('1','2','3')", 8 | "('4','5','6')", 9 | "('7','8','9')" ] 10 | 11 | adapter = ActiveRecord::Base.connection.class 12 | values_size_in_bytes = values.sum {|value| value.bytesize } 13 | base_sql_size_in_bytes = 15 14 | max_bytes = 30 15 | 16 | value_sets = adapter.get_insert_value_sets( values, base_sql_size_in_bytes, max_bytes ) 17 | assert_equal 3, value_sets.size, 'Three value sets were expected!' 18 | 19 | # Each element in the value_sets array must be an array 20 | value_sets.each_with_index { |e,i| 21 | assert_kind_of Array, e, "Element #{i} was expected to be an Array!" } 22 | 23 | # Each element in the values array should have a 1:1 correlation to the elements 24 | # in the returned value_sets arrays 25 | assert_equal values[0], value_sets[0].first 26 | assert_equal values[1], value_sets[1].first 27 | assert_equal values[2], value_sets[2].first 28 | end 29 | 30 | context "data contains multi-byte chars" do 31 | it "should properly build insert value set based on max packet allowed" do 32 | # each accented e should be 2 bytes, so each entry is 6 bytes instead of 5 33 | values = [ 34 | "('é')", 35 | "('é')" ] 36 | 37 | adapter = ActiveRecord::Base.connection.class 38 | base_sql_size_in_bytes = 15 39 | max_bytes = 26 40 | 41 | values_size_in_bytes = values.sum {|value| value.bytesize } 42 | value_sets = adapter.get_insert_value_sets( values, base_sql_size_in_bytes, max_bytes ) 43 | 44 | assert_equal 2, value_sets.size, 'Two value sets were expected!' 45 | end 46 | end 47 | end 48 | 49 | describe "#import with :on_duplicate_key_update option (mysql specific functionality)" do 50 | extend ActiveSupport::TestCase::MySQLAssertions 51 | 52 | asssertion_group(:should_support_on_duplicate_key_update) do 53 | should_not_update_fields_not_mentioned 54 | should_update_foreign_keys 55 | should_not_update_created_at_on_timestamp_columns 56 | should_update_updated_at_on_timestamp_columns 57 | end 58 | 59 | macro(:perform_import){ raise "supply your own #perform_import in a context below" } 60 | macro(:updated_topic){ Topic.find(@topic) } 61 | 62 | context "given columns and values with :validation checks turned off" do 63 | let(:columns){ %w( id title author_name author_email_address parent_id ) } 64 | let(:values){ [ [ 99, "Book", "John Doe", "john@doe.com", 17 ] ] } 65 | let(:updated_values){ [ [ 99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57 ] ] } 66 | 67 | macro(:perform_import) do |*opts| 68 | Topic.import columns, updated_values, opts.extract_options!.merge(:on_duplicate_key_update => update_columns, :validate => false) 69 | end 70 | 71 | setup do 72 | Topic.import columns, values, :validate => false 73 | @topic = Topic.find 99 74 | end 75 | 76 | context "using string column names" do 77 | let(:update_columns){ [ "title", "author_email_address", "parent_id" ] } 78 | should_support_on_duplicate_key_update 79 | should_update_fields_mentioned 80 | end 81 | 82 | context "using symbol column names" do 83 | let(:update_columns){ [ :title, :author_email_address, :parent_id ] } 84 | should_support_on_duplicate_key_update 85 | should_update_fields_mentioned 86 | end 87 | 88 | context "using string hash map" do 89 | let(:update_columns){ { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } } 90 | should_support_on_duplicate_key_update 91 | should_update_fields_mentioned 92 | end 93 | 94 | context "using string hash map, but specifying column mismatches" do 95 | let(:update_columns){ { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } } 96 | should_support_on_duplicate_key_update 97 | should_update_fields_mentioned_with_hash_mappings 98 | end 99 | 100 | context "using symbol hash map" do 101 | let(:update_columns){ { :title => :title, :author_email_address => :author_email_address, :parent_id => :parent_id } } 102 | should_support_on_duplicate_key_update 103 | should_update_fields_mentioned 104 | end 105 | 106 | context "using symbol hash map, but specifying column mismatches" do 107 | let(:update_columns){ { :title => :author_email_address, :author_email_address => :title, :parent_id => :parent_id } } 108 | should_support_on_duplicate_key_update 109 | should_update_fields_mentioned_with_hash_mappings 110 | end 111 | end 112 | 113 | context "given array of model instances with :validation checks turned off" do 114 | macro(:perform_import) do |*opts| 115 | @topic.title = "Book - 2nd Edition" 116 | @topic.author_name = "Author Should Not Change" 117 | @topic.author_email_address = "johndoe@example.com" 118 | @topic.parent_id = 57 119 | Topic.import [@topic], opts.extract_options!.merge(:on_duplicate_key_update => update_columns, :validate => false) 120 | end 121 | 122 | setup do 123 | @topic = Generate(:topic, :id => 99, :author_name => "John Doe", :parent_id => 17) 124 | end 125 | 126 | context "using string column names" do 127 | let(:update_columns){ [ "title", "author_email_address", "parent_id" ] } 128 | should_support_on_duplicate_key_update 129 | should_update_fields_mentioned 130 | end 131 | 132 | context "using symbol column names" do 133 | let(:update_columns){ [ :title, :author_email_address, :parent_id ] } 134 | should_support_on_duplicate_key_update 135 | should_update_fields_mentioned 136 | end 137 | 138 | context "using string hash map" do 139 | let(:update_columns){ { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } } 140 | should_support_on_duplicate_key_update 141 | should_update_fields_mentioned 142 | end 143 | 144 | context "using string hash map, but specifying column mismatches" do 145 | let(:update_columns){ { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } } 146 | should_support_on_duplicate_key_update 147 | should_update_fields_mentioned_with_hash_mappings 148 | end 149 | 150 | context "using symbol hash map" do 151 | let(:update_columns){ { :title => :title, :author_email_address => :author_email_address, :parent_id => :parent_id } } 152 | should_support_on_duplicate_key_update 153 | should_update_fields_mentioned 154 | end 155 | 156 | context "using symbol hash map, but specifying column mismatches" do 157 | let(:update_columns){ { :title => :author_email_address, :author_email_address => :title, :parent_id => :parent_id } } 158 | should_support_on_duplicate_key_update 159 | should_update_fields_mentioned_with_hash_mappings 160 | end 161 | end 162 | 163 | end 164 | 165 | describe "#import with :synchronization option" do 166 | let(:topics){ Array.new } 167 | let(:values){ [ [topics.first.id, "Jerry Carter"], [topics.last.id, "Chad Fowler"] ]} 168 | let(:columns){ %W(id author_name) } 169 | 170 | setup do 171 | topics << Topic.create!(:title=>"LDAP", :author_name=>"Big Bird") 172 | topics << Topic.create!(:title=>"Rails Recipes", :author_name=>"Elmo") 173 | end 174 | 175 | it "synchronizes passed in ActiveRecord model instances with the data just imported" do 176 | columns2update = [ 'author_name' ] 177 | 178 | expected_count = Topic.count 179 | Topic.import( columns, values, 180 | :validate=>false, 181 | :on_duplicate_key_update=>columns2update, 182 | :synchronize=>topics ) 183 | 184 | assert_equal expected_count, Topic.count, "no new records should have been created!" 185 | assert_equal "Jerry Carter", topics.first.author_name, "wrong author!" 186 | assert_equal "Chad Fowler", topics.last.author_name, "wrong author!" 187 | end 188 | end 189 | 190 | end -------------------------------------------------------------------------------- /test/import_test.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../test_helper', __FILE__) 2 | 3 | describe "#import" do 4 | it "should return the number of inserts performed" do 5 | # see ActiveRecord::ConnectionAdapters::AbstractAdapter test for more specifics 6 | assert_difference "Topic.count", +10 do 7 | result = Topic.import Build(3, :topics) 8 | assert result.num_inserts > 0 9 | 10 | result = Topic.import Build(7, :topics) 11 | assert result.num_inserts > 0 12 | end 13 | end 14 | 15 | it "should not produce an error when importing empty arrays" do 16 | assert_nothing_raised do 17 | Topic.import [] 18 | Topic.import %w(title author_name), [] 19 | end 20 | end 21 | 22 | describe "with non-default ActiveRecord models" do 23 | context "that have a non-standard primary key (that is no sequence)" do 24 | it "should import models successfully" do 25 | assert_difference "Widget.count", +3 do 26 | Widget.import Build(3, :widgets) 27 | end 28 | end 29 | end 30 | end 31 | 32 | context "with :validation option" do 33 | let(:columns) { %w(title author_name) } 34 | let(:valid_values) { [[ "LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] } 35 | let(:invalid_values) { [[ "The RSpec Book", ""], ["Agile+UX", ""]] } 36 | 37 | context "with validation checks turned off" do 38 | it "should import valid data" do 39 | assert_difference "Topic.count", +2 do 40 | result = Topic.import columns, valid_values, :validate => false 41 | end 42 | end 43 | 44 | it "should import invalid data" do 45 | assert_difference "Topic.count", +2 do 46 | result = Topic.import columns, invalid_values, :validate => false 47 | end 48 | end 49 | 50 | it 'should raise a specific error if a column does not exist' do 51 | assert_raises ActiveRecord::Import::MissingColumnError do 52 | Topic.import ['foo'], [['bar']], :validate => false 53 | end 54 | end 55 | end 56 | 57 | context "with validation checks turned on" do 58 | it "should import valid data" do 59 | assert_difference "Topic.count", +2 do 60 | result = Topic.import columns, valid_values, :validate => true 61 | end 62 | end 63 | 64 | it "should not import invalid data" do 65 | assert_no_difference "Topic.count" do 66 | result = Topic.import columns, invalid_values, :validate => true 67 | end 68 | end 69 | 70 | it "should report the failed instances" do 71 | results = Topic.import columns, invalid_values, :validate => true 72 | assert_equal invalid_values.size, results.failed_instances.size 73 | results.failed_instances.each{ |e| assert_kind_of Topic, e } 74 | end 75 | 76 | it "should import valid data when mixed with invalid data" do 77 | assert_difference "Topic.count", +2 do 78 | result = Topic.import columns, valid_values + invalid_values, :validate => true 79 | end 80 | assert_equal 0, Topic.find_all_by_title(invalid_values.map(&:first)).count 81 | end 82 | end 83 | end 84 | 85 | context "with :all_or_none option" do 86 | let(:columns) { %w(title author_name) } 87 | let(:valid_values) { [[ "LDAP", "Jerry Carter"], ["Rails Recipes", "Chad Fowler"]] } 88 | let(:invalid_values) { [[ "The RSpec Book", ""], ["Agile+UX", ""]] } 89 | let(:mixed_values) { valid_values + invalid_values } 90 | 91 | context "with validation checks turned on" do 92 | it "should import valid data" do 93 | assert_difference "Topic.count", +2 do 94 | result = Topic.import columns, valid_values, :all_or_none => true 95 | end 96 | end 97 | 98 | it "should not import invalid data" do 99 | assert_no_difference "Topic.count" do 100 | result = Topic.import columns, invalid_values, :all_or_none => true 101 | end 102 | end 103 | 104 | it "should not import valid data when mixed with invalid data" do 105 | assert_no_difference "Topic.count" do 106 | result = Topic.import columns, mixed_values, :all_or_none => true 107 | end 108 | end 109 | 110 | it "should report the failed instances" do 111 | results = Topic.import columns, mixed_values, :all_or_none => true 112 | assert_equal invalid_values.size, results.failed_instances.size 113 | results.failed_instances.each { |e| assert_kind_of Topic, e } 114 | end 115 | 116 | it "should report the zero inserts" do 117 | results = Topic.import columns, mixed_values, :all_or_none => true 118 | assert_equal 0, results.num_inserts 119 | end 120 | end 121 | end 122 | 123 | context "with :synchronize option" do 124 | context "synchronizing on new records" do 125 | let(:new_topics) { Build(3, :topics) } 126 | 127 | it "doesn't reload any data (doesn't work)" do 128 | Topic.import new_topics, :synchronize => new_topics 129 | assert new_topics.all?(&:new_record?), "No record should have been reloaded" 130 | end 131 | end 132 | 133 | context "synchronizing on new records with explicit conditions" do 134 | let(:new_topics) { Build(3, :topics) } 135 | 136 | it "reloads data for existing in-memory instances" do 137 | Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] ) 138 | assert new_topics.all?(&:persisted?), "Records should have been reloaded" 139 | end 140 | end 141 | 142 | context "synchronizing on destroyed records with explicit conditions" do 143 | let(:new_topics) { Generate(3, :topics) } 144 | 145 | it "reloads data for existing in-memory instances" do 146 | new_topics.each &:destroy 147 | Topic.import(new_topics, :synchronize => new_topics, :synchronize_keys => [:title] ) 148 | assert new_topics.all?(&:persisted?), "Records should have been reloaded" 149 | end 150 | end 151 | end 152 | 153 | context "with an array of unsaved model instances" do 154 | let(:topic) { Build(:topic, :title => "The RSpec Book", :author_name => "David Chelimsky")} 155 | let(:topics) { Build(9, :topics) } 156 | let(:invalid_topics){ Build(7, :invalid_topics)} 157 | 158 | it "should import records based on those model's attributes" do 159 | assert_difference "Topic.count", +9 do 160 | result = Topic.import topics 161 | end 162 | 163 | Topic.import [topic] 164 | assert Topic.find_by_title_and_author_name("The RSpec Book", "David Chelimsky") 165 | end 166 | 167 | it "should not overwrite existing records" do 168 | topic = Generate(:topic, :title => "foobar") 169 | assert_no_difference "Topic.count" do 170 | begin 171 | Topic.transaction do 172 | topic.title = "baz" 173 | Topic.import [topic] 174 | end 175 | rescue Exception 176 | # PostgreSQL raises PgError due to key constraints 177 | # I don't know why ActiveRecord doesn't catch these. *sigh* 178 | end 179 | end 180 | assert_equal "foobar", topic.reload.title 181 | end 182 | 183 | context "with validation checks turned on" do 184 | it "should import valid models" do 185 | assert_difference "Topic.count", +9 do 186 | result = Topic.import topics, :validate => true 187 | end 188 | end 189 | 190 | it "should not import invalid models" do 191 | assert_no_difference "Topic.count" do 192 | result = Topic.import invalid_topics, :validate => true 193 | end 194 | end 195 | end 196 | 197 | context "with validation checks turned off" do 198 | it "should import invalid models" do 199 | assert_difference "Topic.count", +7 do 200 | result = Topic.import invalid_topics, :validate => false 201 | end 202 | end 203 | end 204 | end 205 | 206 | context "with an array of columns and an array of unsaved model instances" do 207 | let(:topics) { Build(2, :topics) } 208 | 209 | it "should import records populating the supplied columns with the corresponding model instance attributes" do 210 | assert_difference "Topic.count", +2 do 211 | result = Topic.import [:author_name, :title], topics 212 | end 213 | 214 | # imported topics should be findable by their imported attributes 215 | assert Topic.find_by_author_name(topics.first.author_name) 216 | assert Topic.find_by_author_name(topics.last.author_name) 217 | end 218 | 219 | it "should not populate fields for columns not imported" do 220 | topics.first.author_email_address = "zach.dennis@gmail.com" 221 | assert_difference "Topic.count", +2 do 222 | result = Topic.import [:author_name, :title], topics 223 | end 224 | 225 | assert !Topic.find_by_author_email_address("zach.dennis@gmail.com") 226 | end 227 | end 228 | 229 | context "with an array of columns and an array of values" do 230 | it "should import ids when specified" do 231 | Topic.import [:id, :author_name, :title], [[99, "Bob Jones", "Topic 99"]] 232 | assert_equal 99, Topic.last.id 233 | end 234 | end 235 | 236 | context "ActiveRecord timestamps" do 237 | context "when the timestamps columns are present" do 238 | setup do 239 | Delorean.time_travel_to("5 minutes ago") do 240 | assert_difference "Book.count", +1 do 241 | result = Book.import [:title, :author_name, :publisher], [["LDAP", "Big Bird", "Del Rey"]] 242 | end 243 | end 244 | @book = Book.last 245 | end 246 | 247 | it "should set the created_at column for new records" do 248 | assert_equal 5.minutes.ago.strftime("%H:%M"), @book.created_at.strftime("%H:%M") 249 | end 250 | 251 | it "should set the created_on column for new records" do 252 | assert_equal 5.minutes.ago.strftime("%H:%M"), @book.created_on.strftime("%H:%M") 253 | end 254 | 255 | it "should set the updated_at column for new records" do 256 | assert_equal 5.minutes.ago.strftime("%H:%M"), @book.updated_at.strftime("%H:%M") 257 | end 258 | 259 | it "should set the updated_on column for new records" do 260 | assert_equal 5.minutes.ago.strftime("%H:%M"), @book.updated_on.strftime("%H:%M") 261 | end 262 | end 263 | 264 | context "when a custom time zone is set" do 265 | setup do 266 | original_timezone = ActiveRecord::Base.default_timezone 267 | ActiveRecord::Base.default_timezone = :utc 268 | Delorean.time_travel_to("5 minutes ago") do 269 | assert_difference "Book.count", +1 do 270 | result = Book.import [:title, :author_name, :publisher], [["LDAP", "Big Bird", "Del Rey"]] 271 | end 272 | end 273 | ActiveRecord::Base.default_timezone = original_timezone 274 | @book = Book.last 275 | end 276 | 277 | it "should set the created_at and created_on timestamps for new records" do 278 | assert_equal 5.minutes.ago.utc.strftime("%H:%M"), @book.created_at.strftime("%H:%M") 279 | assert_equal 5.minutes.ago.utc.strftime("%H:%M"), @book.created_on.strftime("%H:%M") 280 | end 281 | 282 | it "should set the updated_at and updated_on timestamps for new records" do 283 | assert_equal 5.minutes.ago.utc.strftime("%H:%M"), @book.updated_at.strftime("%H:%M") 284 | assert_equal 5.minutes.ago.utc.strftime("%H:%M"), @book.updated_on.strftime("%H:%M") 285 | end 286 | end 287 | end 288 | 289 | context "importing with database reserved words" do 290 | let(:group) { Build(:group, :order => "superx") } 291 | 292 | it "should import just fine" do 293 | assert_difference "Group.count", +1 do 294 | result = Group.import [group] 295 | end 296 | assert_equal "superx", Group.first.order 297 | end 298 | end 299 | 300 | context "importing a datetime field" do 301 | it "should import a date with YYYY/MM/DD format just fine" do 302 | Topic.import [:author_name, :title, :last_read], [["Bob Jones", "Topic 2", "2010/05/14"]] 303 | assert_equal "2010/05/14".to_date, Topic.last.last_read.to_date 304 | end 305 | end 306 | 307 | context "importing through an association scope" do 308 | [ true, false ].each do |b| 309 | context "when validation is " + (b ? "enabled" : "disabled") do 310 | it "should automatically set the foreign key column" do 311 | books = [[ "David Chelimsky", "The RSpec Book" ], [ "Chad Fowler", "Rails Recipes" ]] 312 | topic = Factory.create :topic 313 | topic.books.import [ :author_name, :title ], books, :validate => b 314 | assert_equal 2, topic.books.count 315 | assert topic.books.all? { |b| b.topic_id == topic.id } 316 | end 317 | end 318 | end 319 | end 320 | end 321 | -------------------------------------------------------------------------------- /lib/activerecord-import/import.rb: -------------------------------------------------------------------------------- 1 | require "ostruct" 2 | 3 | module ActiveRecord::Import::ConnectionAdapters ; end 4 | 5 | module ActiveRecord::Import #:nodoc: 6 | class Result < Struct.new(:failed_instances, :num_inserts) 7 | end 8 | 9 | module ImportSupport #:nodoc: 10 | def supports_import? #:nodoc: 11 | true 12 | end 13 | end 14 | 15 | module OnDuplicateKeyUpdateSupport #:nodoc: 16 | def supports_on_duplicate_key_update? #:nodoc: 17 | true 18 | end 19 | end 20 | 21 | class MissingColumnError < StandardError 22 | def initialize(name, index) 23 | super "Missing column for value <#{name}> at index #{index}" 24 | end 25 | end 26 | end 27 | 28 | class ActiveRecord::Base 29 | class << self 30 | 31 | # use tz as set in ActiveRecord::Base 32 | tproc = lambda do 33 | ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now 34 | end 35 | 36 | AREXT_RAILS_COLUMNS = { 37 | :create => { "created_on" => tproc , 38 | "created_at" => tproc }, 39 | :update => { "updated_on" => tproc , 40 | "updated_at" => tproc } 41 | } 42 | AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys 43 | 44 | # Returns true if the current database connection adapter 45 | # supports import functionality, otherwise returns false. 46 | def supports_import? 47 | connection.supports_import? 48 | rescue NoMethodError 49 | false 50 | end 51 | 52 | # Returns true if the current database connection adapter 53 | # supports on duplicate key update functionality, otherwise 54 | # returns false. 55 | def supports_on_duplicate_key_update? 56 | connection.supports_on_duplicate_key_update? 57 | rescue NoMethodError 58 | false 59 | end 60 | 61 | # Imports a collection of values to the database. 62 | # 63 | # This is more efficient than using ActiveRecord::Base#create or 64 | # ActiveRecord::Base#save multiple times. This method works well if 65 | # you want to create more than one record at a time and do not care 66 | # about having ActiveRecord objects returned for each record 67 | # inserted. 68 | # 69 | # This can be used with or without validations. It does not utilize 70 | # the ActiveRecord::Callbacks during creation/modification while 71 | # performing the import. 72 | # 73 | # == Usage 74 | # Model.import array_of_models 75 | # Model.import column_names, array_of_values 76 | # Model.import column_names, array_of_values, options 77 | # 78 | # ==== Model.import array_of_models 79 | # 80 | # With this form you can call _import_ passing in an array of model 81 | # objects that you want updated. 82 | # 83 | # ==== Model.import column_names, array_of_values 84 | # 85 | # The first parameter +column_names+ is an array of symbols or 86 | # strings which specify the columns that you want to update. 87 | # 88 | # The second parameter, +array_of_values+, is an array of 89 | # arrays. Each subarray is a single set of values for a new 90 | # record. The order of values in each subarray should match up to 91 | # the order of the +column_names+. 92 | # 93 | # ==== Model.import column_names, array_of_values, options 94 | # 95 | # The first two parameters are the same as the above form. The third 96 | # parameter, +options+, is a hash. This is optional. Please see 97 | # below for what +options+ are available. 98 | # 99 | # == Options 100 | # * +validate+ - true|false, tells import whether or not to use \ 101 | # ActiveRecord validations. Validations are enforced by default. 102 | # * +on_duplicate_key_update+ - an Array or Hash, tells import to \ 103 | # use MySQL's ON DUPLICATE KEY UPDATE ability. See On Duplicate\ 104 | # Key Update below. 105 | # * +synchronize+ - an array of ActiveRecord instances for the model 106 | # that you are currently importing data into. This synchronizes 107 | # existing model instances in memory with updates from the import. 108 | # * +timestamps+ - true|false, tells import to not add timestamps \ 109 | # (if false) even if record timestamps is disabled in ActiveRecord::Base 110 | # 111 | # == Examples 112 | # class BlogPost < ActiveRecord::Base ; end 113 | # 114 | # # Example using array of model objects 115 | # posts = [ BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT', 116 | # BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT2', 117 | # BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT3' ] 118 | # BlogPost.import posts 119 | # 120 | # # Example using column_names and array_of_values 121 | # columns = [ :author_name, :title ] 122 | # values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ] 123 | # BlogPost.import columns, values 124 | # 125 | # # Example using column_names, array_of_value and options 126 | # columns = [ :author_name, :title ] 127 | # values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ] 128 | # BlogPost.import( columns, values, :validate => false ) 129 | # 130 | # # Example synchronizing existing instances in memory 131 | # post = BlogPost.find_by_author_name( 'zdennis' ) 132 | # puts post.author_name # => 'zdennis' 133 | # columns = [ :author_name, :title ] 134 | # values = [ [ 'yoda', 'test post' ] ] 135 | # BlogPost.import posts, :synchronize=>[ post ] 136 | # puts post.author_name # => 'yoda' 137 | # 138 | # # Example synchronizing unsaved/new instances in memory by using a uniqued imported field 139 | # posts = [BlogPost.new(:title => "Foo"), BlogPost.new(:title => "Bar")] 140 | # BlogPost.import posts, :synchronize => posts, :synchronize_keys => [:title] 141 | # puts posts.first.persisted? # => true 142 | # 143 | # == On Duplicate Key Update (MySQL only) 144 | # 145 | # The :on_duplicate_key_update option can be either an Array or a Hash. 146 | # 147 | # ==== Using an Array 148 | # 149 | # The :on_duplicate_key_update option can be an array of column 150 | # names. The column names are the only fields that are updated if 151 | # a duplicate record is found. Below is an example: 152 | # 153 | # BlogPost.import columns, values, :on_duplicate_key_update=>[ :date_modified, :content, :author ] 154 | # 155 | # ==== Using A Hash 156 | # 157 | # The :on_duplicate_key_update option can be a hash of column name 158 | # to model attribute name mappings. This gives you finer grained 159 | # control over what fields are updated with what attributes on your 160 | # model. Below is an example: 161 | # 162 | # BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title } 163 | # 164 | # = Returns 165 | # This returns an object which responds to +failed_instances+ and +num_inserts+. 166 | # * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed. 167 | # * num_inserts - the number of insert statements it took to import the data 168 | def import( *args ) 169 | options = { :validate=>true, :timestamps=>true } 170 | options.merge!( args.pop ) if args.last.is_a? Hash 171 | 172 | is_validating = options.delete( :validate ) 173 | 174 | # assume array of model objects 175 | if args.last.is_a?( Array ) and args.last.first.is_a? ActiveRecord::Base 176 | if args.length == 2 177 | models = args.last 178 | column_names = args.first 179 | else 180 | models = args.first 181 | column_names = self.column_names.dup 182 | end 183 | 184 | array_of_attributes = models.map do |model| 185 | # this next line breaks sqlite.so with a segmentation fault 186 | # if model.new_record? || options[:on_duplicate_key_update] 187 | column_names.map do |name| 188 | model.send( "#{name}_before_type_cast" ) 189 | end 190 | # end 191 | end 192 | # supports empty array 193 | elsif args.last.is_a?( Array ) and args.last.empty? 194 | return ActiveRecord::Import::Result.new([], 0) if args.last.empty? 195 | # supports 2-element array and array 196 | elsif args.size == 2 and args.first.is_a?( Array ) and args.last.is_a?( Array ) 197 | column_names, array_of_attributes = args 198 | else 199 | raise ArgumentError.new( "Invalid arguments!" ) 200 | end 201 | 202 | # dup the passed in array so we don't modify it unintentionally 203 | array_of_attributes = array_of_attributes.dup 204 | 205 | # Force the primary key col into the insert if it's not 206 | # on the list and we are using a sequence and stuff a nil 207 | # value for it into each row so the sequencer will fire later 208 | if !column_names.include?(primary_key) && sequence_name && connection.prefetch_primary_key? 209 | column_names << primary_key 210 | array_of_attributes.each { |a| a << nil } 211 | end 212 | 213 | # record timestamps unless disabled in ActiveRecord::Base 214 | if record_timestamps && options.delete( :timestamps ) 215 | add_special_rails_stamps column_names, array_of_attributes, options 216 | end 217 | 218 | return_obj = if is_validating 219 | import_with_validations( column_names, array_of_attributes, options ) 220 | else 221 | num_inserts = import_without_validations_or_callbacks( column_names, array_of_attributes, options ) 222 | ActiveRecord::Import::Result.new([], num_inserts) 223 | end 224 | 225 | if options[:synchronize] 226 | sync_keys = options[:synchronize_keys] || [self.primary_key] 227 | synchronize( options[:synchronize], sync_keys) 228 | end 229 | 230 | return_obj.num_inserts = 0 if return_obj.num_inserts.nil? 231 | return_obj 232 | end 233 | 234 | # TODO import_from_table needs to be implemented. 235 | def import_from_table( options ) # :nodoc: 236 | end 237 | 238 | # Imports the passed in +column_names+ and +array_of_attributes+ 239 | # given the passed in +options+ Hash with validations. Returns an 240 | # object with the methods +failed_instances+ and +num_inserts+. 241 | # +failed_instances+ is an array of instances that failed validations. 242 | # +num_inserts+ is the number of inserts it took to import the data. See 243 | # ActiveRecord::Base.import for more information on 244 | # +column_names+, +array_of_attributes+ and +options+. 245 | def import_with_validations( column_names, array_of_attributes, options={} ) 246 | failed_instances = [] 247 | 248 | # create instances for each of our column/value sets 249 | arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) 250 | 251 | # keep track of the instance and the position it is currently at. if this fails 252 | # validation we'll use the index to remove it from the array_of_attributes 253 | arr.each_with_index do |hsh,i| 254 | instance = new do |model| 255 | hsh.each_pair{ |k,v| model.send("#{k}=", v) } 256 | end 257 | if not instance.valid? 258 | array_of_attributes[ i ] = nil 259 | failed_instances << instance 260 | end 261 | end 262 | array_of_attributes.compact! 263 | 264 | num_inserts = if array_of_attributes.empty? || options[:all_or_none] && failed_instances.any? 265 | 0 266 | else 267 | import_without_validations_or_callbacks( column_names, array_of_attributes, options ) 268 | end 269 | ActiveRecord::Import::Result.new(failed_instances, num_inserts) 270 | end 271 | 272 | # Imports the passed in +column_names+ and +array_of_attributes+ 273 | # given the passed in +options+ Hash. This will return the number 274 | # of insert operations it took to create these records without 275 | # validations or callbacks. See ActiveRecord::Base.import for more 276 | # information on +column_names+, +array_of_attributes_ and 277 | # +options+. 278 | def import_without_validations_or_callbacks( column_names, array_of_attributes, options={} ) 279 | scope_columns, scope_values = scope_attributes.to_a.transpose 280 | 281 | unless scope_columns.blank? 282 | column_names.concat scope_columns 283 | array_of_attributes.each { |a| a.concat scope_values } 284 | end 285 | 286 | columns = column_names.each_with_index.map do |name, i| 287 | column = columns_hash[name.to_s] 288 | 289 | raise ActiveRecord::Import::MissingColumnError.new(name.to_s, i) if column.nil? 290 | 291 | column 292 | end 293 | 294 | columns_sql = "(#{column_names.map{|name| connection.quote_column_name(name) }.join(',')})" 295 | insert_sql = "INSERT #{options[:ignore] ? 'IGNORE ':''}INTO #{quoted_table_name} #{columns_sql} VALUES " 296 | values_sql = values_sql_for_columns_and_attributes(columns, array_of_attributes) 297 | if not supports_import? 298 | number_inserted = 0 299 | values_sql.each do |values| 300 | connection.execute(insert_sql + values) 301 | number_inserted += 1 302 | end 303 | else 304 | # generate the sql 305 | post_sql_statements = connection.post_sql_statements( quoted_table_name, options ) 306 | 307 | # perform the inserts 308 | number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten, 309 | values_sql, 310 | "#{self.class.name} Create Many Without Validations Or Callbacks" ) 311 | end 312 | number_inserted 313 | end 314 | 315 | private 316 | 317 | # Returns SQL the VALUES for an INSERT statement given the passed in +columns+ 318 | # and +array_of_attributes+. 319 | def values_sql_for_columns_and_attributes(columns, array_of_attributes) # :nodoc: 320 | # connection gets called a *lot* in this high intensity loop. 321 | # Reuse the same one w/in the loop, otherwise it would keep being re-retreived (= lots of time for large imports) 322 | connection_memo = connection 323 | array_of_attributes.map do |arr| 324 | my_values = arr.each_with_index.map do |val,j| 325 | column = columns[j] 326 | 327 | # be sure to query sequence_name *last*, only if cheaper tests fail, because it's costly 328 | if val.nil? && column.name == primary_key && !sequence_name.blank? 329 | connection_memo.next_value_for_sequence(sequence_name) 330 | else 331 | if serialized_attributes.include?(column.name) 332 | connection_memo.quote(serialized_attributes[column.name].dump(val), column) 333 | else 334 | connection_memo.quote(val, column) 335 | end 336 | end 337 | end 338 | "(#{my_values.join(',')})" 339 | end 340 | end 341 | 342 | def add_special_rails_stamps( column_names, array_of_attributes, options ) 343 | AREXT_RAILS_COLUMNS[:create].each_pair do |key, blk| 344 | if self.column_names.include?(key) 345 | value = blk.call 346 | if index=column_names.index(key) 347 | # replace every instance of the array of attributes with our value 348 | array_of_attributes.each{ |arr| arr[index] = value } 349 | else 350 | column_names << key 351 | array_of_attributes.each { |arr| arr << value } 352 | end 353 | end 354 | end 355 | 356 | AREXT_RAILS_COLUMNS[:update].each_pair do |key, blk| 357 | if self.column_names.include?(key) 358 | value = blk.call 359 | if index=column_names.index(key) 360 | # replace every instance of the array of attributes with our value 361 | array_of_attributes.each{ |arr| arr[index] = value } 362 | else 363 | column_names << key 364 | array_of_attributes.each { |arr| arr << value } 365 | end 366 | 367 | if supports_on_duplicate_key_update? 368 | if options[:on_duplicate_key_update] 369 | options[:on_duplicate_key_update] << key.to_sym if options[:on_duplicate_key_update].is_a?(Array) 370 | options[:on_duplicate_key_update][key.to_sym] = key.to_sym if options[:on_duplicate_key_update].is_a?(Hash) 371 | else 372 | options[:on_duplicate_key_update] = [ key.to_sym ] 373 | end 374 | end 375 | end 376 | end 377 | end 378 | 379 | # Returns an Array of Hashes for the passed in +column_names+ and +array_of_attributes+. 380 | def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc: 381 | array_of_attributes.map do |attributes| 382 | Hash[attributes.each_with_index.map {|attr, c| [column_names[c], attr] }] 383 | end 384 | end 385 | 386 | end 387 | end 388 | --------------------------------------------------------------------------------